diff --git a/.gitignore b/.gitignore index 6edcc386f..4b2fd6d29 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,6 @@ GeoLite2-City.mmdb allCountries.zip rest/.miredot-offline.json /extensions/salesforce-connector/test.properties +extensions/web-tracker/javascript/dist/* +extensions/web-tracker/javascript/snippet.min.* **/*.versionsBackup \ No newline at end of file diff --git a/.travis.settings.xml b/.travis.settings.xml new file mode 100644 index 000000000..a23393eea --- /dev/null +++ b/.travis.settings.xml @@ -0,0 +1,27 @@ + + + + + ${JFROG_USER} + ${JFROG_PASSWORD} + snapshots + + + + + + + + snapshots + maven + https://yotpo.jfrog.io/artifactory/maven + + + artifactory + + + + artifactory + + diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..c4057968e --- /dev/null +++ b/.travis.yml @@ -0,0 +1,33 @@ +group: edge + +services: + - docker + +cache: + directories: + - $HOME/.m2/ +before_cache: + rm -rf $HOME/.m2/repository/org/apache/unomi + +language: java + +stages: + - name: build + if: branch != releases + - name: build and deploy + if: branch = releases + +install: skip + +jobs: + include: + - stage: build + name: "Build Unomi" + jdk: openjdk8 + script: + - ./build.sh + - stage: build and deploy + name: "Build and deploy Unomi" + jdk: openjdk8 + script: + - ./buildAndDeploy.sh \ No newline at end of file diff --git a/NOTICE b/NOTICE index 26aa3d46e..273ee5879 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache Unomi -Copyright 2015-2019 The Apache Software Foundation +Copyright 2015-2020 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/NOTICE.template b/NOTICE.template index ea88cc630..602667b9f 100644 --- a/NOTICE.template +++ b/NOTICE.template @@ -1,5 +1,5 @@ Apache Unomi -Copyright 2016 The Apache Software Foundation +Copyright 2015-2020 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/api/pom.xml b/api/pom.xml index 50024f58f..855f7fc94 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -22,7 +22,7 @@ org.apache.unomi unomi-root - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-api @@ -37,7 +37,10 @@ 2.2.11 provided - + + org.apache.commons + commons-lang3 + diff --git a/api/src/main/java/org/apache/unomi/api/ContextRequest.java b/api/src/main/java/org/apache/unomi/api/ContextRequest.java index 6a6c79a9b..1a59dde1e 100644 --- a/api/src/main/java/org/apache/unomi/api/ContextRequest.java +++ b/api/src/main/java/org/apache/unomi/api/ContextRequest.java @@ -64,6 +64,7 @@ public class ContextRequest { private Profile profileOverrides; private Map sessionPropertiesOverrides; private String sessionId; + private String profileId; /** * Retrieves the source of the context request. @@ -242,4 +243,21 @@ public String getSessionId() { public void setSessionId(String sessionId) { this.sessionId = sessionId; } + + /** + * Retrieve the profileId passed along with the request. All events will be processed with this profileId as a + * default + * @return the identifier for the profile + */ + public String getProfileId() { + return profileId; + } + + /** + * Sets the profileId in the request. + * @param profileId an unique identifier for the profile + */ + public void setProfileId(String profileId) { + this.profileId = profileId; + } } diff --git a/api/src/main/java/org/apache/unomi/api/Event.java b/api/src/main/java/org/apache/unomi/api/Event.java index e7ff5a623..6b4e1e829 100644 --- a/api/src/main/java/org/apache/unomi/api/Event.java +++ b/api/src/main/java/org/apache/unomi/api/Event.java @@ -17,6 +17,7 @@ package org.apache.unomi.api; +import org.apache.commons.lang3.StringUtils; import org.apache.unomi.api.actions.ActionPostExecutor; import javax.xml.bind.annotation.XmlTransient; @@ -323,6 +324,30 @@ public Object getProperty(String name) { return properties.get(name); } + /** + * Retrieves the value of the nested property identified by the specified name. + * + * @param name the name of the property to be retrieved, splited in the nested properties with "." + * @return the value of the property identified by the specified name + */ + public Object getNestedProperty(String name) { + if (!name.contains(".")) { + return getProperty(name); + } + + Map properties = this.properties; + String[] propertyPath = StringUtils.substringBeforeLast(name, ".").split("\\."); + String propertyName = StringUtils.substringAfterLast(name, "."); + + for (String property: propertyPath) { + properties = (Map) properties.get(property); + if (properties == null) { + return null; + } + } + return properties.get(propertyName); + } + /** * Retrieves the properties. * diff --git a/api/src/main/java/org/apache/unomi/api/Item.java b/api/src/main/java/org/apache/unomi/api/Item.java index 72f0ae6ae..70818a6df 100644 --- a/api/src/main/java/org/apache/unomi/api/Item.java +++ b/api/src/main/java/org/apache/unomi/api/Item.java @@ -21,6 +21,7 @@ import org.slf4j.LoggerFactory; import java.io.Serializable; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -63,6 +64,7 @@ public static String getItemType(Class clazz) { protected String itemType; protected String scope; protected Long version; + protected Map systemMetadata = new HashMap<>(); public Item() { this.itemType = getItemType(this.getClass()); @@ -140,4 +142,12 @@ public Long getVersion() { public void setVersion(Long version) { this.version = version; } + + public Object getSystemMetadataMetadata(String key) { + return systemMetadata.get(key); + } + + public void setSystemMetadata(String key, Object value) { + systemMetadata.put(key, value); + } } diff --git a/api/src/main/java/org/apache/unomi/api/Profile.java b/api/src/main/java/org/apache/unomi/api/Profile.java index ba75da910..7115bd563 100644 --- a/api/src/main/java/org/apache/unomi/api/Profile.java +++ b/api/src/main/java/org/apache/unomi/api/Profile.java @@ -17,6 +17,7 @@ package org.apache.unomi.api; +import org.apache.commons.lang3.StringUtils; import org.apache.unomi.api.segments.Scoring; import org.apache.unomi.api.segments.Segment; @@ -94,6 +95,30 @@ public Object getProperty(String name) { return properties.get(name); } + /** + * Retrieves the value of the nested property identified by the specified name. + * + * @param name the name of the property to be retrieved, splited in the nested properties with "." + * @return the value of the property identified by the specified name + */ + public Object getNestedProperty(String name) { + if (!name.contains(".")) { + return getProperty(name); + } + + Map properties = this.properties; + String[] propertyPath = StringUtils.substringBeforeLast(name, ".").split("\\."); + String propertyName = StringUtils.substringAfterLast(name, "."); + + for (String property: propertyPath) { + properties = (Map) properties.get(property); + if (properties == null) { + return null; + } + } + return properties.get(propertyName); + } + /** * Retrieves a Map of all property name - value pairs for this profile. * diff --git a/api/src/main/java/org/apache/unomi/api/conditions/ConditionHook.java b/api/src/main/java/org/apache/unomi/api/conditions/ConditionHook.java new file mode 100644 index 000000000..b93ddb922 --- /dev/null +++ b/api/src/main/java/org/apache/unomi/api/conditions/ConditionHook.java @@ -0,0 +1,7 @@ +package org.apache.unomi.api.conditions; + + +public interface ConditionHook { + + void executeHook(Condition condition); +} diff --git a/api/src/main/java/org/apache/unomi/api/conditions/ConditionHookFactory.java b/api/src/main/java/org/apache/unomi/api/conditions/ConditionHookFactory.java new file mode 100644 index 000000000..a347e8576 --- /dev/null +++ b/api/src/main/java/org/apache/unomi/api/conditions/ConditionHookFactory.java @@ -0,0 +1,7 @@ +package org.apache.unomi.api.conditions; + + +public interface ConditionHookFactory { + + ConditionHook createConditionHook(); +} diff --git a/api/src/main/java/org/apache/unomi/api/query/Query.java b/api/src/main/java/org/apache/unomi/api/query/Query.java index ef74b727a..48e0c1a6b 100644 --- a/api/src/main/java/org/apache/unomi/api/query/Query.java +++ b/api/src/main/java/org/apache/unomi/api/query/Query.java @@ -33,6 +33,8 @@ public class Query implements Serializable { private String sortby; private Condition condition; private boolean forceRefresh; + private String scrollTimeValidity; + private String scrollIdentifier; /** * Instantiates a new Query. @@ -150,4 +152,21 @@ public boolean isForceRefresh() { public void setForceRefresh(boolean forceRefresh) { this.forceRefresh = forceRefresh; } + + public String getScrollIdentifier() { + return scrollIdentifier; + } + + public void setScrollIdentifier(String scrollIdentifier) { + this.scrollIdentifier = scrollIdentifier; + } + + public String getScrollTimeValidity() { + return scrollTimeValidity; + } + + public void setScrollTimeValidity(String scrollTimeValidity) { + this.scrollTimeValidity = scrollTimeValidity; + } + } diff --git a/api/src/main/java/org/apache/unomi/api/services/DefinitionsService.java b/api/src/main/java/org/apache/unomi/api/services/DefinitionsService.java index 816f8831a..983ef48a4 100644 --- a/api/src/main/java/org/apache/unomi/api/services/DefinitionsService.java +++ b/api/src/main/java/org/apache/unomi/api/services/DefinitionsService.java @@ -20,6 +20,7 @@ import org.apache.unomi.api.PluginType; import org.apache.unomi.api.PropertyMergeStrategyType; import org.apache.unomi.api.ValueType; +import org.apache.unomi.api.actions.Action; import org.apache.unomi.api.actions.ActionType; import org.apache.unomi.api.conditions.Condition; import org.apache.unomi.api.conditions.ConditionType; @@ -205,6 +206,21 @@ public interface DefinitionsService { */ boolean resolveConditionType(Condition rootCondition); + /** + * Resolves (if possible) the {@link ActionType} for the specified action + * + * @param action the action for which we want to resolve type + * @return {@code true} + */ + boolean resolveActionType(Action action); + + /** + * Resolves (if possible) the {@link ActionType}s for the specified action + * + * @param actions the actions for which we want to resolve type + * @return {@code true} + */ + boolean resolveActionTypes(List actions); /** * Forces a refresh of the definitions from the persistence service. Warning: this may seriously impact performance * so it is recommended to use this in specific cases such as for example in integration tests. diff --git a/build.sh b/build.sh new file mode 100755 index 000000000..9dc4e3ce1 --- /dev/null +++ b/build.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -ev + +#mvn -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn clean install -P integration-tests -Drat.skip=true --no-snapshot-updates +#(cd itests && mvn -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -P integration-tests -Dit.test=org.apache.unomi.itests.ProfileServiceWithoutOverwriteIT --no-snapshot-updates verify) +mvn -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn clean install -Drat.skip=true --no-snapshot-updates \ No newline at end of file diff --git a/buildAndDeploy.sh b/buildAndDeploy.sh new file mode 100755 index 000000000..093ef0ec0 --- /dev/null +++ b/buildAndDeploy.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -ev + +cp .travis.settings.xml $HOME/.m2/settings.xml +mvn -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn clean install deploy -DskipTests -P integration-tests -Drat.skip=true --no-snapshot-updates -DaltDeploymentRepository=snapshots::default::https://yotpo.jfrog.io/artifactory/maven \ No newline at end of file diff --git a/buildAndRunNoTests.sh b/buildAndRunNoTests.sh index 9ec4e3ed8..427e7a29d 100755 --- a/buildAndRunNoTests.sh +++ b/buildAndRunNoTests.sh @@ -23,7 +23,7 @@ PROGNAME=`basename "$0"` if [ -f "$DIRNAME/setenv.sh" ]; then . "$DIRNAME/setenv.sh" fi -mvn clean install -P \!integration-tests,\!performance-tests,rat -DskipTests +mvn clean install -P \!integration-tests,\!performance-tests,rat,\!run-tests -DskipTests if [ $? -ne 0 ] then exit 1; diff --git a/common/pom.xml b/common/pom.xml index 6e1460c0b..9c6442d82 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -22,7 +22,7 @@ org.apache.unomi unomi-root - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-common diff --git a/common/src/main/java/org/apache/unomi/common/SecureFilteringClassLoader.java b/common/src/main/java/org/apache/unomi/common/SecureFilteringClassLoader.java new file mode 100644 index 000000000..61b91bf60 --- /dev/null +++ b/common/src/main/java/org/apache/unomi/common/SecureFilteringClassLoader.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.unomi.common; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** + * A class loader that uses a whitelist and a black list of classes that it will allow to resolve. This is useful for providing proper + * sandboxing to scripting engine such as MVEL, OGNL or Groovy. + */ +public class SecureFilteringClassLoader extends ClassLoader { + + private Set allowedClasses = null; + private Set forbiddenClasses = null; + + private static Set defaultAllowedClasses = null; + private static Set defaultForbiddenClasses = null; + + static { + String systemAllowedClasses = System.getProperty("org.apache.unomi.scripting.allow", + "org.apache.unomi.api.Event,org.apache.unomi.api.Profile,org.apache.unomi.api.Session,org.apache.unomi.api.Item,org.apache.unomi.api.CustomItem,ognl.*,java.lang.Object,java.util.Map,java.lang.Integer,org.mvel2.*"); + if (systemAllowedClasses != null) { + if ("all".equals(systemAllowedClasses.trim())) { + defaultAllowedClasses = null; + } else { + if (systemAllowedClasses.trim().length() > 0) { + String[] systemAllowedClassesParts = systemAllowedClasses.split(","); + defaultAllowedClasses = new HashSet<>(); + defaultAllowedClasses.addAll(Arrays.asList(systemAllowedClassesParts)); + } else { + defaultAllowedClasses = null; + } + } + } + + String systemForbiddenClasses = System.getProperty("org.apache.unomi.scripting.forbid", null); + if (systemForbiddenClasses != null) { + if (systemForbiddenClasses.trim().length() > 0) { + String[] systemForbiddenClassesParts = systemForbiddenClasses.split(","); + defaultForbiddenClasses = new HashSet<>(); + defaultForbiddenClasses.addAll(Arrays.asList(systemForbiddenClassesParts)); + } else { + defaultForbiddenClasses = null; + } + } + + } + + ClassLoader delegate; + + /** + * Sets up the securing filtering class loader, using the default allowed and forbidden classes. By default the + * @param delegate the class loader we delegate to if the filtering was not applied. + */ + public SecureFilteringClassLoader(ClassLoader delegate) { + this(defaultAllowedClasses, defaultForbiddenClasses, delegate); + } + + /** + * Sets up the secure filtering class loader + * @param allowedClasses the list of allowed FQN class names, or if this filtering is to be deactivated, pass null. + * if you want to allow no class, pass an empty hashset + * @param forbiddenClasses the list of forbidden FQN class names, or if this filtering is to be deactivated, pass null or an empty set + * + * @param delegate the class loader we delegate to if the filtering was not applied. + */ + public SecureFilteringClassLoader(Set allowedClasses, Set forbiddenClasses, ClassLoader delegate) { + super(delegate); + this.allowedClasses = allowedClasses; + this.forbiddenClasses = forbiddenClasses; + this.delegate = delegate; + } + + @Override + public Class loadClass(String name) throws ClassNotFoundException { + if (forbiddenClasses != null && classNameMatches(forbiddenClasses, name)) { + throw new ClassNotFoundException("Access to class " + name + " not allowed"); + } + if (allowedClasses != null && !classNameMatches(allowedClasses, name)) { + throw new ClassNotFoundException("Access to class " + name + " not allowed"); + } + return delegate.loadClass(name); + } + + private boolean classNameMatches(Set classesToTest, String className) { + for (String classToTest : classesToTest) { + if (classToTest.endsWith("*")) { + if (className.startsWith(classToTest.substring(0, classToTest.length() - 1))) return true; + } else { + if (className.equals(classToTest)) return true; + } + } + return false; + } + +} diff --git a/docker/README.md b/docker/README.md index c8f41fbcb..c1f5bb491 100644 --- a/docker/README.md +++ b/docker/README.md @@ -51,12 +51,12 @@ For ElasticSearch: For Unomi: - docker pull apache/unomi:1.5.0-SNAPSHOT - docker run --name unomi --net unomi -p 8181:8181 -p 9443:9443 -p 8102:8102 -e UNOMI_ELASTICSEARCH_ADDRESSES=elasticsearch:9200 apache/unomi:1.5.0-SNAPSHOT + docker pull apache/unomi:1.5.1-SNAPSHOT + docker run --name unomi --net unomi -p 8181:8181 -p 9443:9443 -p 8102:8102 -e UNOMI_ELASTICSEARCH_ADDRESSES=elasticsearch:9200 apache/unomi:1.5.1-SNAPSHOT ## Using a host OS ElasticSearch installation (only supported on macOS & Windows) - docker run --name unomi -p 8181:8181 -p 9443:9443 -p 8102:8102 -e UNOMI_ELASTICSEARCH_ADDRESSES=host.docker.internal:9200 apache/unomi:1.5.0-SNAPSHOT + docker run --name unomi -p 8181:8181 -p 9443:9443 -p 8102:8102 -e UNOMI_ELASTICSEARCH_ADDRESSES=host.docker.internal:9200 apache/unomi:1.5.1-SNAPSHOT Note: Linux doesn't support the host.docker.internal DNS lookup method yet, it should be available in an upcoming version of Docker. See https://github.com/docker/for-linux/issues/264 diff --git a/docker/pom.xml b/docker/pom.xml index d92be43a6..3fcb87b74 100644 --- a/docker/pom.xml +++ b/docker/pom.xml @@ -16,14 +16,13 @@ ~ limitations under the License. --> - + 4.0.0 org.apache.unomi unomi-root - 1.5.0-SNAPSHOT + 1.5.1 unomi-docker diff --git a/extensions/geonames/pom.xml b/extensions/geonames/pom.xml index d8a4dc333..fc56da837 100644 --- a/extensions/geonames/pom.xml +++ b/extensions/geonames/pom.xml @@ -27,7 +27,7 @@ org.apache.unomi unomi-extensions - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT cxs-geonames diff --git a/extensions/geonames/rest/pom.xml b/extensions/geonames/rest/pom.xml index ab2eae36b..0de6111b6 100644 --- a/extensions/geonames/rest/pom.xml +++ b/extensions/geonames/rest/pom.xml @@ -20,7 +20,7 @@ org.apache.unomi cxs-geonames - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 diff --git a/extensions/geonames/services/pom.xml b/extensions/geonames/services/pom.xml index 2deef86b0..37f57f828 100644 --- a/extensions/geonames/services/pom.xml +++ b/extensions/geonames/services/pom.xml @@ -20,7 +20,7 @@ org.apache.unomi cxs-geonames - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 diff --git a/extensions/groovy-actions/karaf-kar/pom.xml b/extensions/groovy-actions/karaf-kar/pom.xml index 6abe66614..b1f259e51 100644 --- a/extensions/groovy-actions/karaf-kar/pom.xml +++ b/extensions/groovy-actions/karaf-kar/pom.xml @@ -20,7 +20,7 @@ org.apache.unomi unomi-groovy-actions-root - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 diff --git a/extensions/groovy-actions/pom.xml b/extensions/groovy-actions/pom.xml index b87a60eb8..e5c5c6e2a 100644 --- a/extensions/groovy-actions/pom.xml +++ b/extensions/groovy-actions/pom.xml @@ -21,7 +21,7 @@ org.apache.unomi unomi-extensions - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-groovy-actions-root diff --git a/extensions/groovy-actions/services/pom.xml b/extensions/groovy-actions/services/pom.xml index 7720c1142..3c476e732 100644 --- a/extensions/groovy-actions/services/pom.xml +++ b/extensions/groovy-actions/services/pom.xml @@ -20,7 +20,7 @@ org.apache.unomi unomi-groovy-actions-root - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 diff --git a/extensions/lists-extension/actions/pom.xml b/extensions/lists-extension/actions/pom.xml index 009c14b17..2f5ac47fe 100644 --- a/extensions/lists-extension/actions/pom.xml +++ b/extensions/lists-extension/actions/pom.xml @@ -20,7 +20,7 @@ cxs-lists-extension org.apache.unomi - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 diff --git a/extensions/lists-extension/pom.xml b/extensions/lists-extension/pom.xml index 71e1335db..d5fd07fdd 100644 --- a/extensions/lists-extension/pom.xml +++ b/extensions/lists-extension/pom.xml @@ -28,7 +28,7 @@ org.apache.unomi unomi-extensions - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT cxs-lists-extension diff --git a/extensions/lists-extension/rest/pom.xml b/extensions/lists-extension/rest/pom.xml index 6356bdff5..327f81e9f 100644 --- a/extensions/lists-extension/rest/pom.xml +++ b/extensions/lists-extension/rest/pom.xml @@ -20,7 +20,7 @@ cxs-lists-extension org.apache.unomi - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 diff --git a/extensions/lists-extension/services/pom.xml b/extensions/lists-extension/services/pom.xml index a1cae0b30..99305998f 100644 --- a/extensions/lists-extension/services/pom.xml +++ b/extensions/lists-extension/services/pom.xml @@ -20,7 +20,7 @@ cxs-lists-extension org.apache.unomi - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 diff --git a/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java b/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java index dc3bbc8f2..37ca72e04 100644 --- a/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java +++ b/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java @@ -94,7 +94,7 @@ public void delete(String listId) { if(index != -1){ ((List) profileSystemProperties.get("lists")).remove(index); profileSystemProperties.put("lastUpdated", new Date()); - persistenceService.update(p.getItemId(), null, Profile.class, "systemProperties", profileSystemProperties); + persistenceService.update(p, null, Profile.class, "systemProperties", profileSystemProperties); } } } diff --git a/extensions/pom.xml b/extensions/pom.xml index 64dadc8e0..9a91c4942 100644 --- a/extensions/pom.xml +++ b/extensions/pom.xml @@ -22,7 +22,7 @@ org.apache.unomi unomi-root - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-extensions diff --git a/extensions/privacy-extension/pom.xml b/extensions/privacy-extension/pom.xml index f8f916786..2b2a0356e 100644 --- a/extensions/privacy-extension/pom.xml +++ b/extensions/privacy-extension/pom.xml @@ -27,7 +27,7 @@ org.apache.unomi unomi-extensions - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT cxs-privacy-extension diff --git a/extensions/privacy-extension/rest/pom.xml b/extensions/privacy-extension/rest/pom.xml index ec4eb87c3..a038a1881 100644 --- a/extensions/privacy-extension/rest/pom.xml +++ b/extensions/privacy-extension/rest/pom.xml @@ -20,7 +20,7 @@ cxs-privacy-extension org.apache.unomi - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 @@ -83,22 +83,22 @@ - - com.sebastian-daschner - jaxrs-analyzer-maven-plugin - 0.17 - - - - analyze-jaxrs - - - swagger - unomi.apache.org - - - - + + + + + + + + + + + + + + + + diff --git a/extensions/privacy-extension/services/pom.xml b/extensions/privacy-extension/services/pom.xml index 19f6e7ba0..eb41ab439 100644 --- a/extensions/privacy-extension/services/pom.xml +++ b/extensions/privacy-extension/services/pom.xml @@ -20,14 +20,14 @@ cxs-privacy-extension org.apache.unomi - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 cxs-privacy-extension-services Apache Unomi :: Extensions :: Privacy :: Services Privacy management extension service implementation for the Apache Unomi Context Server - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT bundle diff --git a/extensions/privacy-extension/services/src/main/java/org/apache/unomi/privacy/internal/PrivacyServiceImpl.java b/extensions/privacy-extension/services/src/main/java/org/apache/unomi/privacy/internal/PrivacyServiceImpl.java index 1675e1100..3d3d68d7f 100644 --- a/extensions/privacy-extension/services/src/main/java/org/apache/unomi/privacy/internal/PrivacyServiceImpl.java +++ b/extensions/privacy-extension/services/src/main/java/org/apache/unomi/privacy/internal/PrivacyServiceImpl.java @@ -137,7 +137,7 @@ public Boolean anonymizeBrowsingData(String profileId) { persistenceService.save(session); List events = eventService.searchEvents(session.getItemId(), new String[0], null, 0, -1, null).getList(); for (Event event : events) { - persistenceService.update(event.getItemId(), event.getTimeStamp(), Event.class, "profileId", newProfile.getItemId()); + persistenceService.update(event, event.getTimeStamp(), Event.class, "profileId", newProfile.getItemId()); } } diff --git a/extensions/router/pom.xml b/extensions/router/pom.xml index eff81ba74..83a2d6fcd 100644 --- a/extensions/router/pom.xml +++ b/extensions/router/pom.xml @@ -20,7 +20,7 @@ org.apache.unomi unomi-extensions - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-router diff --git a/extensions/router/router-api/pom.xml b/extensions/router/router-api/pom.xml index 7c2dd6e60..bb1314a6e 100644 --- a/extensions/router/router-api/pom.xml +++ b/extensions/router/router-api/pom.xml @@ -19,7 +19,7 @@ unomi-router org.apache.unomi - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 diff --git a/extensions/router/router-core/pom.xml b/extensions/router/router-core/pom.xml index b2373bd5d..ae047e9bc 100644 --- a/extensions/router/router-core/pom.xml +++ b/extensions/router/router-core/pom.xml @@ -19,7 +19,7 @@ unomi-router org.apache.unomi - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 diff --git a/extensions/router/router-karaf-feature/pom.xml b/extensions/router/router-karaf-feature/pom.xml index 2f84c89b4..03f598632 100644 --- a/extensions/router/router-karaf-feature/pom.xml +++ b/extensions/router/router-karaf-feature/pom.xml @@ -19,7 +19,7 @@ unomi-router org.apache.unomi - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 diff --git a/extensions/router/router-rest/pom.xml b/extensions/router/router-rest/pom.xml index b1efeb8f4..91b447fff 100644 --- a/extensions/router/router-rest/pom.xml +++ b/extensions/router/router-rest/pom.xml @@ -19,7 +19,7 @@ unomi-router org.apache.unomi - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 diff --git a/extensions/router/router-service/pom.xml b/extensions/router/router-service/pom.xml index 639cf68cb..2fde1e3e4 100644 --- a/extensions/router/router-service/pom.xml +++ b/extensions/router/router-service/pom.xml @@ -19,7 +19,7 @@ unomi-router org.apache.unomi - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 diff --git a/extensions/salesforce-connector/actions/pom.xml b/extensions/salesforce-connector/actions/pom.xml index 89124cb6f..4c364245c 100644 --- a/extensions/salesforce-connector/actions/pom.xml +++ b/extensions/salesforce-connector/actions/pom.xml @@ -20,7 +20,7 @@ org.apache.unomi unomi-salesforce-connector - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 diff --git a/extensions/salesforce-connector/karaf-kar/pom.xml b/extensions/salesforce-connector/karaf-kar/pom.xml index cb5cce22f..7f1f96a67 100644 --- a/extensions/salesforce-connector/karaf-kar/pom.xml +++ b/extensions/salesforce-connector/karaf-kar/pom.xml @@ -20,7 +20,7 @@ org.apache.unomi unomi-salesforce-connector - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 diff --git a/extensions/salesforce-connector/pom.xml b/extensions/salesforce-connector/pom.xml index 5a578a908..b173fb6de 100644 --- a/extensions/salesforce-connector/pom.xml +++ b/extensions/salesforce-connector/pom.xml @@ -21,7 +21,7 @@ org.apache.unomi unomi-extensions - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-salesforce-connector diff --git a/extensions/salesforce-connector/rest/pom.xml b/extensions/salesforce-connector/rest/pom.xml index 3e023fe99..0a0db7e01 100644 --- a/extensions/salesforce-connector/rest/pom.xml +++ b/extensions/salesforce-connector/rest/pom.xml @@ -20,7 +20,7 @@ org.apache.unomi unomi-salesforce-connector - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 diff --git a/extensions/salesforce-connector/services/pom.xml b/extensions/salesforce-connector/services/pom.xml index af0c281a1..1c92c8545 100644 --- a/extensions/salesforce-connector/services/pom.xml +++ b/extensions/salesforce-connector/services/pom.xml @@ -20,7 +20,7 @@ org.apache.unomi unomi-salesforce-connector - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 diff --git a/extensions/unomi-mailchimp/actions/pom.xml b/extensions/unomi-mailchimp/actions/pom.xml index a0704c127..1b218df09 100644 --- a/extensions/unomi-mailchimp/actions/pom.xml +++ b/extensions/unomi-mailchimp/actions/pom.xml @@ -21,7 +21,7 @@ org.apache.unomi unomi-mailchimp-connector - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-mailchimp-connector-actions diff --git a/extensions/unomi-mailchimp/karaf-kar/pom.xml b/extensions/unomi-mailchimp/karaf-kar/pom.xml index d29097bf9..5d31393f1 100644 --- a/extensions/unomi-mailchimp/karaf-kar/pom.xml +++ b/extensions/unomi-mailchimp/karaf-kar/pom.xml @@ -22,7 +22,7 @@ org.apache.unomi unomi-mailchimp-connector - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-mailchimp-connector-karaf-kar diff --git a/extensions/unomi-mailchimp/pom.xml b/extensions/unomi-mailchimp/pom.xml index bac4cba7c..e2de00c9a 100644 --- a/extensions/unomi-mailchimp/pom.xml +++ b/extensions/unomi-mailchimp/pom.xml @@ -21,7 +21,7 @@ org.apache.unomi unomi-extensions - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-mailchimp-connector diff --git a/extensions/unomi-mailchimp/rest/pom.xml b/extensions/unomi-mailchimp/rest/pom.xml index c59bbd69d..c22ebed41 100644 --- a/extensions/unomi-mailchimp/rest/pom.xml +++ b/extensions/unomi-mailchimp/rest/pom.xml @@ -20,7 +20,7 @@ org.apache.unomi unomi-mailchimp-connector - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 diff --git a/extensions/unomi-mailchimp/services/pom.xml b/extensions/unomi-mailchimp/services/pom.xml index 62ca1bc3f..5a63b069a 100644 --- a/extensions/unomi-mailchimp/services/pom.xml +++ b/extensions/unomi-mailchimp/services/pom.xml @@ -22,7 +22,7 @@ org.apache.unomi unomi-mailchimp-connector - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-mailchimp-connector-services diff --git a/extensions/weather-update/core/pom.xml b/extensions/weather-update/core/pom.xml index beb6bc249..72a6c80bb 100755 --- a/extensions/weather-update/core/pom.xml +++ b/extensions/weather-update/core/pom.xml @@ -21,7 +21,7 @@ org.apache.unomi unomi-weather-update - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-weather-update-core diff --git a/extensions/weather-update/karaf-kar/pom.xml b/extensions/weather-update/karaf-kar/pom.xml index de882c925..7ce4811f3 100644 --- a/extensions/weather-update/karaf-kar/pom.xml +++ b/extensions/weather-update/karaf-kar/pom.xml @@ -20,7 +20,7 @@ org.apache.unomi unomi-weather-update - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 diff --git a/extensions/weather-update/pom.xml b/extensions/weather-update/pom.xml index 45ce01aed..ba6beb2cf 100644 --- a/extensions/weather-update/pom.xml +++ b/extensions/weather-update/pom.xml @@ -21,7 +21,7 @@ org.apache.unomi unomi-extensions - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-weather-update diff --git a/extensions/web-tracker/javascript/pom.xml b/extensions/web-tracker/javascript/pom.xml index 557e5a719..9fb7ca083 100755 --- a/extensions/web-tracker/javascript/pom.xml +++ b/extensions/web-tracker/javascript/pom.xml @@ -21,7 +21,7 @@ org.apache.unomi unomi-web-tracker - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-web-tracker-javascript diff --git a/extensions/web-tracker/karaf-kar/pom.xml b/extensions/web-tracker/karaf-kar/pom.xml index 0fc32b9ee..1f0a68317 100644 --- a/extensions/web-tracker/karaf-kar/pom.xml +++ b/extensions/web-tracker/karaf-kar/pom.xml @@ -20,7 +20,7 @@ org.apache.unomi unomi-web-tracker - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 diff --git a/extensions/web-tracker/pom.xml b/extensions/web-tracker/pom.xml index e70efeb4c..61ce2b82d 100644 --- a/extensions/web-tracker/pom.xml +++ b/extensions/web-tracker/pom.xml @@ -21,7 +21,7 @@ org.apache.unomi unomi-extensions - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-web-tracker diff --git a/extensions/web-tracker/wab/pom.xml b/extensions/web-tracker/wab/pom.xml index cc16780cb..3e1020fd5 100755 --- a/extensions/web-tracker/wab/pom.xml +++ b/extensions/web-tracker/wab/pom.xml @@ -21,7 +21,7 @@ org.apache.unomi unomi-web-tracker - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-web-tracker-wab diff --git a/itests/pom.xml b/itests/pom.xml index a1448caf3..3b9dbfac8 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -21,7 +21,7 @@ org.apache.unomi unomi-root - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-itests Apache Unomi :: Integration Tests @@ -129,87 +129,95 @@ - - - - - org.apache.servicemix.tooling - depends-maven-plugin - - - generate-depends-file - - generate-depends-file - - - - - - com.github.alexcojocaru - elasticsearch-maven-plugin - - 6.16 - - contextElasticSearchITests - 9500 - 9400 - ${elasticsearch.version} - true - 120 - - -Xms2g -Xmx2g - - - - false - - - - - - - start-elasticsearch - pre-integration-test - - runforked - - - - stop-elasticsearch - post-integration-test - - stop - - - - - - org.apache.maven.plugins - maven-failsafe-plugin - 3.0.0-M4 - - - **/*AllITs.java - - - - - integration-test - - integration-test - - - - verify - - verify - - - - - - + + + run-tests + + true + + + + + + org.apache.servicemix.tooling + depends-maven-plugin + + + generate-depends-file + + generate-depends-file + + + + + + com.github.alexcojocaru + elasticsearch-maven-plugin + + 6.16 + + contextElasticSearchITests + 9500 + 9400 + ${elasticsearch.version} + true + 120 + + -Xms2g -Xmx2g + + + + false + + + + + + + start-elasticsearch + pre-integration-test + + runforked + + + + stop-elasticsearch + post-integration-test + + stop + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + 3.0.0-M4 + + + **/*AllITs.java + + + + + integration-test + + integration-test + + + + verify + + verify + + + + + + + + \ No newline at end of file diff --git a/itests/src/test/java/org/apache/unomi/itests/AllITs.java b/itests/src/test/java/org/apache/unomi/itests/AllITs.java index 938fcf816..58786f9da 100644 --- a/itests/src/test/java/org/apache/unomi/itests/AllITs.java +++ b/itests/src/test/java/org/apache/unomi/itests/AllITs.java @@ -41,7 +41,9 @@ PropertiesUpdateActionIT.class, ModifyConsentIT.class, PatchIT.class, - UpdateEventFromContextServletIT.class + ContextServletIT.class, + SecurityIT.class, + EventServiceIT.class }) public class AllITs { } diff --git a/itests/src/test/java/org/apache/unomi/itests/BaseIT.java b/itests/src/test/java/org/apache/unomi/itests/BaseIT.java index d1bdd33c2..6651562b2 100644 --- a/itests/src/test/java/org/apache/unomi/itests/BaseIT.java +++ b/itests/src/test/java/org/apache/unomi/itests/BaseIT.java @@ -69,7 +69,6 @@ public Option[] config() throws InterruptedException { List \ No newline at end of file diff --git a/persistence-elasticsearch/core/pom.xml b/persistence-elasticsearch/core/pom.xml index 5b9d5691b..c2cc4600f 100644 --- a/persistence-elasticsearch/core/pom.xml +++ b/persistence-elasticsearch/core/pom.xml @@ -22,7 +22,7 @@ org.apache.unomi unomi-persistence-elasticsearch - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-persistence-elasticsearch-core @@ -48,6 +48,12 @@ ${project.version} provided + + org.apache.unomi + unomi-common + ${project.version} + provided + org.apache.unomi unomi-persistence-spi @@ -64,6 +70,12 @@ elasticsearch-rest-high-level-client ${elasticsearch.version} + + net.jodah + failsafe + ${fail-safe.version} + provided + diff --git a/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg b/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg index 54522c8ad..b29db0d7c 100644 --- a/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg +++ b/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg @@ -20,11 +20,14 @@ cluster.name=${org.apache.unomi.elasticsearch.cluster.name:-contextElasticSearch # hostA:9200,hostB:9200 # Note: the port number must be repeated for each host. elasticSearchAddresses=${org.apache.unomi.elasticsearch.addresses:-localhost:9200} +fatalIllegalStateErrors=${org.apache.unomi.elasticsearch.fatalIllegalStateErrors:-} index.prefix=${org.apache.unomi.elasticsearch.index.prefix:-context} +mappingOverride=${org.apache.unomi.elasticsearch.mappingOverride:-false} monthlyIndex.numberOfShards=${org.apache.unomi.elasticsearch.monthlyIndex.nbShards:-5} monthlyIndex.numberOfReplicas=${org.apache.unomi.elasticsearch.monthlyIndex.nbReplicas:-0} monthlyIndex.indexMappingTotalFieldsLimit=${org.apache.unomi.elasticsearch.monthlyIndex.indexMappingTotalFieldsLimit:-1000} monthlyIndex.indexMaxDocValueFieldsSearch=${org.apache.unomi.elasticsearch.monthlyIndex.indexMaxDocValueFieldsSearch:-1000} +monthlyIndex.itemsMonthlyIndexedOverride=${org.apache.unomi.elasticsearch.monthlyIndex.itemsMonthlyIndexedOverride:-event,session} numberOfShards=${org.apache.unomi.elasticsearch.defaultIndex.nbShards:-5} numberOfReplicas=${org.apache.unomi.elasticsearch.defaultIndex.nbReplicas:-0} indexMappingTotalFieldsLimit=${org.apache.unomi.elasticsearch.defaultIndex.indexMappingTotalFieldsLimit:-1000} @@ -53,8 +56,36 @@ aggregateQueryBucketSize=${org.apache.unomi.elasticsearch.aggregateQueryBucketSi # Maximum size allowed for an elastic "ids" query maximumIdsQueryCount=${org.apache.unomi.elasticsearch.maximumIdsQueryCount:-5000} +# Disable partitions on aggregation queries for past events. +pastEventsDisablePartitions=${org.apache.unomi.elasticsearch.pastEventsDisablePartitions:-false} + +# max socket timeout in millis +clientSocketTimeout=${org.apache.unomi.elasticsearch.clientSocketTimeout:-} + +# refresh policy per item type in Json. +# Valid values are WAIT_UNTIL/IMMEDIATE/NONE. The default refresh policy is NONE. +# Example: "{"event":"WAIT_UNTIL","rule":"NONE"} +itemTypeToRefreshPolicy=${org.apache.unomi.elasticsearch.itemTypeToRefreshPolicy:-} + +# hidden fields per item type in Json. +# Example: "{"Profile":"properties"} +itemTypeToHiddenFields=${org.apache.unomi.elasticsearch.itemTypeToHiddenFields:-} + +# Retrun error in docs are missing in es aggregation calculation +aggQueryThrowOnMissingDocs=${org.apache.unomi.elasticsearch.aggQueryThrowOnMissingDocs:-false} + +aggQueryMaxResponseSizeHttp=${org.apache.unomi.elasticsearch.aggQueryMaxResponseSizeHttp:-} + # Authentication username=${org.apache.unomi.elasticsearch.username:-} password=${org.apache.unomi.elasticsearch.password:-} sslEnable=${org.apache.unomi.elasticsearch.sslEnable:-false} -sslTrustAllCertificates=${org.apache.unomi.elasticsearch.sslTrustAllCertificates:-false} \ No newline at end of file +sslTrustAllCertificates=${org.apache.unomi.elasticsearch.sslTrustAllCertificates:-false} + +# Errors +throwExceptions=${org.apache.unomi.elasticsearch.throwExceptions:-false} + +alwaysOverwrite=${org.apache.unomi.elasticsearch.alwaysOverwrite:-true} +useBatchingForUpdate=${org.apache.unomi.elasticsearch.useBatchingForUpdate:-true} + +updateSlices=${org.apache.unomi.elasticsearch.updateSlices:-2} diff --git a/persistence-elasticsearch/pom.xml b/persistence-elasticsearch/pom.xml index 2ff1d71d9..f7c489c9f 100644 --- a/persistence-elasticsearch/pom.xml +++ b/persistence-elasticsearch/pom.xml @@ -16,14 +16,13 @@ ~ limitations under the License. --> - + 4.0.0 org.apache.unomi unomi-root - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-persistence-elasticsearch diff --git a/persistence-spi/pom.xml b/persistence-spi/pom.xml index 8b9b0b60b..a189277cd 100644 --- a/persistence-spi/pom.xml +++ b/persistence-spi/pom.xml @@ -22,7 +22,7 @@ org.apache.unomi unomi-root - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-persistence-spi diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java index 7311353cd..742c90ecb 100644 --- a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java +++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java @@ -59,6 +59,34 @@ public interface PersistenceService { */ PartialList getAllItems(Class clazz, int offset, int size, String sortBy); + /** + * Retrieves all known items of the specified class, ordered according to the specified {@code sortBy} String and and paged: only {@code size} of them are retrieved, + * starting with the {@code offset}-th one. + * + * TODO: use a Query object instead of distinct parameters? + * + * @param the type of the {@link Item}s we want to retrieve + * @param clazz the {@link Item} subclass of entities we want to retrieve + * @param offset zero or a positive integer specifying the position of the first item in the total ordered collection of matching items + * @param size a positive integer specifying how many matching items should be retrieved or {@code -1} if all of them should be retrieved + * @param sortBy an optional ({@code null} if no sorting is required) String of comma ({@code ,}) separated property names on which ordering should be performed, ordering + * elements according to the property order in the + * String, considering each in turn and moving on to the next one in case of equality of all preceding ones. Each property name is optionally followed by + * a column ({@code :}) and an order specifier: {@code asc} or {@code desc}. + * @param scrollTimeValidity the time the scrolling query should stay valid. This must contain a time unit value such as the ones supported by ElasticSearch, such as + * * the ones declared here : https://www.elastic.co/guide/en/elasticsearch/reference/current/common-options.html#time-units + * @return a {@link PartialList} of pages items with the given type + */ + PartialList getAllItems(final Class clazz, int offset, int size, String sortBy, String scrollTimeValidity); + + /** + * Return true if the item which is saved in the persistence service is consistent + * + * @param item the item to the check if consistent + * @return {@code true} if the item is consistent, false otherwise + */ + boolean isConsistent(Item item); + /** * Persists the specified Item in the context server. * @@ -78,42 +106,97 @@ public interface PersistenceService { */ boolean save(Item item, boolean useBatching); + /** + * Persists the specified Item in the context server. + * + * @param item the item to persist + * @param useBatching whether to use batching or not for saving the item. If activating there may be a delay between + * the call to this method and the actual saving in the persistence backend + * @param alwaysOverwrite whether to overwrite a document even if we are holding an old item when saving + * + * @return {@code true} if the item was properly persisted, {@code false} otherwise + */ + boolean save(Item item, Boolean useBatching, Boolean alwaysOverwrite); + /** * Updates the item of the specified class and identified by the specified identifier with new property values provided as name - value pairs in the specified Map. * - * @param itemId the identifier of the item we want to update + * @param item the item we want to update * @param dateHint a Date helping in identifying where the item is located * @param clazz the Item subclass of the item to update * @param source a Map with entries specifying as key the property name to update and as value its new value * @return {@code true} if the update was successful, {@code false} otherwise */ - boolean update(String itemId, Date dateHint, Class clazz, Map source); + boolean update(Item item, Date dateHint, Class clazz, Map source); /** * Updates the item of the specified class and identified by the specified identifier with a new property value for the specified property name. Same as * {@code update(itemId, dateHint, clazz, Collections.singletonMap(propertyName, propertyValue))} * - * @param itemId the identifier of the item we want to update + * @param item the item we want to update * @param dateHint a Date helping in identifying where the item is located * @param clazz the Item subclass of the item to update * @param propertyName the name of the property to update * @param propertyValue the new value of the property * @return {@code true} if the update was successful, {@code false} otherwise */ - boolean update(String itemId, Date dateHint, Class clazz, String propertyName, Object propertyValue); + boolean update(Item item, Date dateHint, Class clazz, String propertyName, Object propertyValue); + + /** + * Updates the item of the specified class and identified by the specified identifier with new property values provided as name - value pairs in the specified Map. + * + * @param item the item we want to update + * @param dateHint a Date helping in identifying where the item is located + * @param clazz the Item subclass of the item to update + * @param source a Map with entries specifying as key the property name to update and as value its new value + * @param alwaysOverwrite whether to overwrite a document even if we are holding an old item when saving + * @return {@code true} if the update was successful, {@code false} otherwise + */ + boolean update(Item item, Date dateHint, Class clazz, Map source, final boolean alwaysOverwrite); + + /** + * Updates Map of items of the specified class and identified by the specified identifier with a new property value for the specified property name. Same as + * {@code update(itemId, dateHint, clazz, Collections.singletonMap(propertyName, propertyValue))} + * + * @param items A map the consist of item (key) and properties to update (value) + * @param dateHint a Date helping in identifying where the item is located + * @param clazz the Item subclass of the item to update + * @return List of failed Items Ids, if all succesful then returns an empty list. if the whole operation failed then will return null + */ + List update(Map items, Date dateHint, Class clazz); /** * Updates the item of the specified class and identified by the specified identifier with a new property value for the specified property name. Same as * {@code update(itemId, dateHint, clazz, Collections.singletonMap(propertyName, propertyValue))} * - * @param itemId the identifier of the item we want to update + * @param item the item we want to update * @param dateHint a Date helping in identifying where the item is located * @param clazz the Item subclass of the item to update * @param script inline script * @param scriptParams script params * @return {@code true} if the update was successful, {@code false} otherwise */ - boolean updateWithScript(String itemId, Date dateHint, Class clazz, String script, Map scriptParams); + boolean updateWithScript(Item item, Date dateHint, Class clazz, String script, Map scriptParams); + + /** + * Adds or removes an element from a list by a query. + * Based on provided scripts and script parameters. + * + * @param dateHint a Date helping in identifying where the item is located + * @param clazz the Item subclass of the item to update + * @param updateSystemLastUpdated whether to update 'systemProperties.lastUpdated' or not + * @param scriptParams script params array. (Must contain 'listPropName' entry which specifies the field to update). + * @param conditions conditions array + * @param numberOfRetries how many retries on version-conflict + * @param secondsDelayForRetryUpdate how many seconds to wait between retries + * @param isAddToList whether to add or remove item from list. + * @param batchSize Size of batch to be processed. + * @return Updated profiles quantity, -1 if update failed. + */ + Long updateListWithQuery(final Date dateHint, final Class clazz, + boolean updateSystemLastUpdated, + final Map[] scriptParams, final Condition[] conditions, + int numberOfRetries, long secondsDelayForRetryUpdate, boolean isAddToList, Integer batchSize); /** * Updates the items of the specified class by a query with a new property value for the specified property name @@ -126,7 +209,25 @@ public interface PersistenceService { * @param conditions conditions array * @return {@code true} if the update was successful, {@code false} otherwise */ - boolean updateWithQueryAndScript(Date dateHint, Class clazz, String[] scripts, Map[] scriptParams, Condition[] conditions); + Boolean updateWithQueryAndScript(Date dateHint, Class clazz, String[] scripts, Map[] scriptParams, Condition[] conditions); + + /** + * Updates the items of the specified class by a query with a new property value for the specified property name + * based on provided scripts and script parameters + * + * @param dateHint a Date helping in identifying where the item is located + * @param clazz the Item subclass of the item to update + * @param scripts inline scripts array + * @param scriptParams script params array + * @param conditions conditions array + * @param numberOfRetries how many retries on version-conflict + * @param secondsDelayForRetryUpdate how many seconds to wait between retries + * @param batchSize Size of batch to be processed. + * @return Updated profiles quantity, -1 if update failed. + */ + Long updateWithQueryAndScript(final Date dateHint, final Class clazz, final String[] scripts, + final Map[] scriptParams, final Condition[] conditions, + int numberOfRetries, long secondsDelayForRetryUpdate, Integer batchSize); /** * Retrieves the item identified with the specified identifier and with the specified Item subclass if it exists. @@ -149,6 +250,17 @@ public interface PersistenceService { */ T load(String itemId, Date dateHint, Class clazz); + /** + * + * @param dateHint a Date helping in identifying where the item is located + * @param clazz the {@link Item} subclass of the item we want to retrieve + * @param itemId the identifier of the item we want to retrieve + * @param the type of the Item subclass we want to retrieve + * @return A list of Items identified with the specified identifiers and with the specified Item subclass if exist, empty {@link List} otherwise + */ + List load(Date dateHint, Class clazz, String... itemId); + + /** * Deletes the item identified with the specified identifier and with the specified Item subclass if it exists. * @@ -225,6 +337,14 @@ public interface PersistenceService { */ boolean testMatch(Condition query, Item item); + /** + * validates if a condition throws exception at query build. + * + * @param query the condition we're testing the specified item against + * @param item the item we're checking against the specified condition + * @return {@code true} if the item satisfies the condition, {@code false} otherwise + */ + void validateCondition(Condition query, Item item); /** * Same as {@code query(fieldName, fieldValue, sortBy, clazz, 0, -1).getList()} * @@ -341,6 +461,24 @@ public interface PersistenceService { */ PartialList query(Condition query, String sortBy, Class clazz, int offset, int size); + /** + * Retrieves a list of items satisfying the specified {@link Condition}, ordered according to the specified {@code sortBy} String and and paged: only {@code size} of them + * are retrieved, starting with the {@code offset}-th one. + * + * @param the type of the Item subclass we want to retrieve + * @param query the {@link Condition} the items must satisfy to be retrieved + * @param sortBy an optional ({@code null} if no sorting is required) String of comma ({@code ,}) separated property names on which ordering should be performed, ordering + * elements according to the property order in the + * String, considering each in turn and moving on to the next one in case of equality of all preceding ones. Each property name is optionally followed by + * a column ({@code :}) and an order specifier: {@code asc} or {@code desc}. + * @param clazz the {@link Item} subclass of the items we want to retrieve + * @param offset zero or a positive integer specifying the position of the first item in the total ordered collection of matching items + * @param size a positive integer specifying how many matching items should be retrieved or {@code -1} if all of them should be retrieved + * @param returnHiddenFields - if true, hidden fields that are defined in configuration will be returned as well + * @return a {@link PartialList} of items matching the specified criteria + */ + PartialList query(Condition query, String sortBy, Class clazz, int offset, int size, boolean returnHiddenFields); + /** * Retrieves a list of items satisfying the specified {@link Condition}, ordered according to the specified {@code sortBy} String and and paged: only {@code size} of them * are retrieved, starting with the {@code offset}-th one. If a scroll identifier and time validity are specified, they will be used to perform a scrolling query, meaning @@ -437,6 +575,18 @@ public interface PersistenceService { */ Map aggregateWithOptimizedQuery(Condition filter, BaseAggregate aggregate, String itemType); + /** + * Retrieves the number of items with the specified type as defined by the Item subclass public field {@code ITEM_TYPE} matching the optional specified condition and + * aggregated according to the specified {@link BaseAggregate}. + * + * @param filter the condition the items must match or {@code null} if no filtering is needed + * @param aggregate an aggregate specifying how matching items must be bundled + * @param itemType the String representation of the item type we want to retrieve the count of, as defined by its class' {@code ITEM_TYPE} field + * @param size size of returned buckets in the response + * @return a Map associating aggregation dimension name as key and cardinality for that dimension as value + */ + Map aggregateWithOptimizedQuery(Condition filter, BaseAggregate aggregate, String itemType, int size); + /** * Updates the persistence's engine indices if needed. */ @@ -507,4 +657,4 @@ public interface PersistenceService { */ void purge(final String scope); -} \ No newline at end of file +} diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PropertyHelper.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PropertyHelper.java index 283ebbeb9..63150250d 100644 --- a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PropertyHelper.java +++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PropertyHelper.java @@ -21,14 +21,13 @@ import org.apache.commons.beanutils.NestedNullException; import org.apache.commons.beanutils.PropertyUtils; import org.apache.commons.beanutils.expression.DefaultResolver; +import org.apache.unomi.api.Profile; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; +import java.util.*; +import java.util.stream.Collectors; /** * Helper method for properties @@ -85,7 +84,23 @@ public static boolean setProperty(Object target, String propertyName, Object pro BeanUtils.setProperty(target, propertyName, values); return true; } - } else if (propertyValue != null && !compareValues(propertyValue, BeanUtils.getProperty(target, propertyName))) { + } + if (setPropertyStrategy != null && setPropertyStrategy.equals("addValues")) { + Object newValues = propertyValue; + List newValuesList = convertToList(newValues); + + Object previousValue = PropertyUtils.getProperty(target, propertyName); + List previousValueList = convertToList(previousValue); + + newValuesList.addAll(previousValueList); + Set propertiesSet = new HashSet<>(newValuesList); + List propertiesList = Arrays.asList(propertiesSet.toArray()); + + BeanUtils.setProperty(target, propertyName, propertiesList); + return true; + + } + else if (propertyValue != null && !compareValues(propertyValue, BeanUtils.getProperty(target, propertyName))) { if (setPropertyStrategy == null || setPropertyStrategy.equals("alwaysSet") || (setPropertyStrategy.equals("setIfMissing") && BeanUtils.getProperty(target, propertyName) == null)) { @@ -99,6 +114,16 @@ public static boolean setProperty(Object target, String propertyName, Object pro return false; } + public static List convertToList(Object value) { + List convertedList = new ArrayList<>(); + if (value != null && value instanceof List) { + convertedList.addAll((List) value); + } else if (value != null) { + convertedList.add(value); + } + return convertedList; + } + public static Integer getInteger(Object value) { if (value instanceof Number) { return ((Number) value).intValue(); @@ -112,6 +137,19 @@ public static Integer getInteger(Object value) { return null; } + public static Double getDouble(Object value) { + if (value instanceof Number) { + return ((Number) value).doubleValue(); + } else { + try { + return Double.parseDouble(value.toString()); + } catch (NumberFormatException e) { + // Not a number + } + } + return null; + } + public static Boolean getBooleanValue(Object setPropertyValueBoolean) { if (setPropertyValueBoolean instanceof Boolean) { @@ -158,5 +196,31 @@ public static boolean compareValues(Object propertyValue, Object beanPropertyVal } } + public static Map flatten(Map in) { + return in.entrySet().stream() + .filter(entry -> entry.getValue() != null) + .flatMap(entry -> flatten(entry).entrySet().stream()) + .collect(Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue)); + } + + private static Map flatten(Map.Entry in) { + // for other then Map objects return them + if (!Map.class.isInstance(in.getValue())) { + return Collections.singletonMap(in.getKey(), in.getValue()); + } + // extract the key prefix for nested objects + String prefix = in.getKey(); + Map values = (Map) in.getValue(); + // create a new Map, with prefix added to each key + Map flattenMap = new HashMap<>(); + values.keySet().forEach(key -> { + // use a dot as a joining char + flattenMap.put(prefix + "." + key, values.get(key)); + }); + // use recursion to flatten the structure deeper + return flatten(flattenMap); + } } diff --git a/plugins/baseplugin/pom.xml b/plugins/baseplugin/pom.xml index 0e61331a6..6b50c2071 100644 --- a/plugins/baseplugin/pom.xml +++ b/plugins/baseplugin/pom.xml @@ -22,7 +22,7 @@ org.apache.unomi unomi-plugins - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-plugins-base @@ -36,6 +36,12 @@ javax.servlet-api provided + + javax.xml.bind + jaxb-api + 2.2.11 + provided + commons-collections commons-collections @@ -78,6 +84,12 @@ jackson-databind provided + + org.apache.unomi + unomi-common + ${project.version} + provided + diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java index da92a24f0..a152c91ac 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/MergeProfilesOnPropertyAction.java @@ -47,6 +47,9 @@ public class MergeProfilesOnPropertyAction implements ActionExecutor { private ConfigSharingService configSharingService; public int execute(Action action, Event event) { + if (event.getAttributes().get("alreadyMerged") != null && (boolean) event.getAttributes().get("alreadyMerged") == true) { + return EventService.NO_CHANGE; + } String profileIdCookieName = (String) configSharingService.getProperty("profileIdCookieName"); String profileIdCookieDomain = (String) configSharingService.getProperty("profileIdCookieDomain"); Integer profileIdCookieMaxAgeInSeconds = (Integer) configSharingService.getProperty("profileIdCookieMaxAgeInSeconds"); @@ -70,6 +73,9 @@ public int execute(Action action, Event event) { final Session currentSession = event.getSession(); + boolean forceEventProfileAsMaster = action.getParameterValues().containsKey("forceEventProfileAsMaster") ? + (boolean) action.getParameterValues().get("forceEventProfileAsMaster") : false; + // store the profile id in case the merge change it to a previous one String profileId = profile.getItemId(); @@ -86,20 +92,26 @@ public int execute(Action action, Event event) { c.setParameter("operator", "and"); c.setParameter("subConditions", Arrays.asList(propertyCondition, excludeMergedProfilesCondition)); - final List profiles = persistenceService.query(c, "properties.firstVisit", Profile.class); + final List profiles = persistenceService.query(c, "properties.firstVisit", Profile.class, 0, 1000, true).getList(); // Check if the user switched to another profile if (StringUtils.isNotEmpty(mergeProfilePreviousPropertyValue) && !mergeProfilePreviousPropertyValue.equals(mergeProfilePropertyValue)) { - if (profiles.size() > 0) { - // Take existing profile - profile = profiles.get(0); + if (forceEventProfileAsMaster == false) { + if (profiles.size() > 0) { + // Take existing profile + profile = profiles.get(0); + } else { + // Create a new profile + profile = new Profile(UUID.randomUUID().toString()); + profile.setProperty("firstVisit", event.getTimeStamp()); + profile.getSystemProperties().put(mergeProfilePropertyName, mergeProfilePropertyValue); + } } else { - // Create a new profile - profile = new Profile(UUID.randomUUID().toString()); - profile.setProperty("firstVisit", currentSession.getTimeStamp()); profile.getSystemProperties().put(mergeProfilePropertyName, mergeProfilePropertyValue); } + + logger.info("Different users, switch to " + profile.getItemId()); HttpServletResponse httpServletResponse = (HttpServletResponse) event.getAttributes().get(Event.HTTP_RESPONSE_ATTRIBUTE); @@ -109,9 +121,10 @@ public int execute(Action action, Event event) { event.setProfileId(profile.getItemId()); event.setProfile(profile); - currentSession.setProfile(profile); - - eventService.send(new Event("sessionReassigned", currentSession, profile, event.getScope(), event, currentSession, event.getTimeStamp())); + if (currentSession != null) { + currentSession.setProfile(profile); + eventService.send(new Event("sessionReassigned", currentSession, profile, event.getScope(), event, currentSession, event.getTimeStamp())); + } return EventService.PROFILE_UPDATED + EventService.SESSION_UPDATED; } else { @@ -131,52 +144,74 @@ public int execute(Action action, Event event) { return StringUtils.isEmpty(mergeProfilePreviousPropertyValue) ? EventService.PROFILE_UPDATED : EventService.NO_CHANGE; } - // Use oldest profile for master profile - final Profile masterProfile = profileService.mergeProfiles(profiles.get(0), profiles); + Profile markedMasterProfile; + if (forceEventProfileAsMaster) + markedMasterProfile = event.getProfile(); + else + markedMasterProfile = profiles.get(0);// Use oldest profile for master profile + + final Profile masterProfile = profileService.mergeProfiles(markedMasterProfile, profiles); // Profile has changed - if (!masterProfile.getItemId().equals(profileId)) { + if (forceEventProfileAsMaster || !masterProfile.getItemId().equals(profileId)) { HttpServletResponse httpServletResponse = (HttpServletResponse) event.getAttributes().get(Event.HTTP_RESPONSE_ATTRIBUTE); // we still send back the current profile cookie. It will be changed on the next request to the ContextServlet. // The current profile will be deleted only then because we cannot delete it right now (too soon) - sendProfileCookie(currentSession.getProfile(), httpServletResponse, + sendProfileCookie(profile, httpServletResponse, profileIdCookieName, profileIdCookieDomain, profileIdCookieMaxAgeInSeconds); final String masterProfileId = masterProfile.getItemId(); + // At the end of the merge, we must set the merged profile as profile event to process other Actions event.setProfileId(masterProfileId); event.setProfile(masterProfile); - currentSession.setProfile(masterProfile); - if (privacyService.isRequireAnonymousBrowsing(profile)) { - privacyService.setRequireAnonymousBrowsing(masterProfileId, true, event.getScope()); - } - final Boolean anonymousBrowsing = privacyService.isRequireAnonymousBrowsing(masterProfileId); - if (anonymousBrowsing) { - currentSession.setProfile(privacyService.getAnonymousProfile(masterProfile)); - event.setProfileId(null); - persistenceService.save(event); + + if (currentSession != null){ + currentSession.setProfile(masterProfile); + if (privacyService.isRequireAnonymousBrowsing(profile)) { + privacyService.setRequireAnonymousBrowsing(masterProfileId, true, event.getScope()); + } + + if (anonymousBrowsing) { + currentSession.setProfile(privacyService.getAnonymousProfile(masterProfile)); + event.setProfileId(null); + persistenceService.save(event); + } } event.getActionPostExecutors().add(new ActionPostExecutor() { @Override public boolean execute() { try { + Event currentEvent = event; + // Update current event explicitly, as it might not return from search query if there wasn't a refresh in ES + if (!StringUtils.equals(profileId, masterProfileId)) { + if (currentEvent.isPersistent()) { + persistenceService.update(currentEvent, currentEvent.getTimeStamp(), Event.class, "profileId", anonymousBrowsing ? null : masterProfileId); + } + } + for (Profile profile : profiles) { String profileId = profile.getItemId(); if (!StringUtils.equals(profileId, masterProfileId)) { List sessions = persistenceService.query("profileId", profileId, null, Session.class); - if (masterProfileId.equals(profileId) && !sessions.contains(currentSession)) { - sessions.add(currentSession); + if (currentSession != null){ + if (masterProfileId.equals(profileId) && !sessions.contains(currentSession)) { + sessions.add(currentSession); + } } + for (Session session : sessions) { - persistenceService.update(session.getItemId(), session.getTimeStamp(), Session.class, "profileId", anonymousBrowsing ? null : masterProfileId); + persistenceService.update(session, session.getTimeStamp(), Session.class, "profileId", anonymousBrowsing ? null : masterProfileId); } List events = persistenceService.query("profileId", profileId, null, Event.class); for (Event event : events) { - persistenceService.update(event.getItemId(), event.getTimeStamp(), Event.class, "profileId", anonymousBrowsing ? null : masterProfileId); + if (!event.getItemId().equals(currentEvent.getItemId())) { + persistenceService.update(event, event.getTimeStamp(), Event.class, "profileId", anonymousBrowsing ? null : masterProfileId); + } } // we must mark all the profiles that we merged into the master as merged with the master, and they will // be deleted upon next load @@ -185,7 +220,14 @@ public boolean execute() { sourceMap.put("mergedWith", masterProfileId); profile.setSystemProperty("lastUpdated", new Date()); sourceMap.put("systemProperties", profile.getSystemProperties()); - persistenceService.update(profile.getItemId(), null, Profile.class, sourceMap); + + boolean isExist = persistenceService.load(profile.getItemId(), Profile.class) != null; + + if (isExist == false) //save the original event profile is it has been changed + persistenceService.save(profile); + else + persistenceService.update(profile, null, Profile.class, sourceMap,true); + } } } catch (Exception e) { diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/SetEventOccurenceCountAction.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/SetEventOccurenceCountAction.java index cd1b5e330..19b1afa76 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/SetEventOccurenceCountAction.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/SetEventOccurenceCountAction.java @@ -25,10 +25,16 @@ import org.apache.unomi.api.services.EventService; import org.apache.unomi.persistence.spi.PersistenceService; +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.ZoneId; import java.util.ArrayList; +import java.util.Calendar; import java.util.LinkedHashMap; import java.util.Map; +import javax.xml.bind.DatatypeConverter; + public class SetEventOccurenceCountAction implements ActionExecutor { private DefinitionsService definitionsService; @@ -60,15 +66,32 @@ public int execute(Action action, Event event) { c.setParameter("propertyValue", event.getProfileId()); conditions.add(c); - if (pastEventCondition.getParameter("numberOfDays") != null) { - int i = (Integer) pastEventCondition.getParameter("numberOfDays"); - - Condition timeCondition = new Condition(definitionsService.getConditionType("eventPropertyCondition")); - timeCondition.setParameter("propertyName", "timeStamp"); - timeCondition.setParameter("comparisonOperator", "greaterThan"); - timeCondition.setParameter("propertyValueDateExpr", "now-" + i + "d"); + Integer numberOfDays = (Integer) pastEventCondition.getParameter("numberOfDays"); + String fromDate = (String) pastEventCondition.getParameter("fromDate"); + String toDate = (String) pastEventCondition.getParameter("toDate"); - conditions.add(timeCondition); + if (numberOfDays != null) { + Condition numberOfDaysCondition = new Condition(definitionsService.getConditionType("eventPropertyCondition")); + numberOfDaysCondition.setParameter("propertyName", "timeStamp"); + numberOfDaysCondition.setParameter("comparisonOperator", "greaterThan"); + numberOfDaysCondition.setParameter("propertyValueDateExpr", "now-" + numberOfDays + "d"); + conditions.add(numberOfDaysCondition); + } + if (fromDate != null) { + Condition startDateCondition = new Condition(); + startDateCondition.setConditionType(definitionsService.getConditionType("eventPropertyCondition")); + startDateCondition.setParameter("propertyName", "timeStamp"); + startDateCondition.setParameter("comparisonOperator", "greaterThanOrEqualTo"); + startDateCondition.setParameter("propertyValueDate", fromDate); + conditions.add(startDateCondition); + } + if (toDate != null) { + Condition endDateCondition = new Condition(); + endDateCondition.setConditionType(definitionsService.getConditionType("eventPropertyCondition")); + endDateCondition.setParameter("propertyName", "timeStamp"); + endDateCondition.setParameter("comparisonOperator", "lessThanOrEqualTo"); + endDateCondition.setParameter("propertyValueDate", toDate); + conditions.add(endDateCondition); } andCondition.setParameter("subConditions", conditions); @@ -80,8 +103,51 @@ public int execute(Action action, Event event) { pastEvents = new LinkedHashMap<>(); event.getProfile().getSystemProperties().put("pastEvents", pastEvents); } - pastEvents.put((String) pastEventCondition.getParameter("generatedPropertyKey"), count + 1); + + LocalDateTime fromDateTime = null; + if (fromDate != null) { + Calendar fromDateCalendar = DatatypeConverter.parseDateTime(fromDate); + fromDateTime = LocalDateTime.ofInstant(fromDateCalendar.toInstant(), ZoneId.of("UTC")); + } + LocalDateTime toDateTime = null; + if (toDate != null) { + Calendar toDateCalendar = DatatypeConverter.parseDateTime(toDate); + toDateTime = LocalDateTime.ofInstant(toDateCalendar.toInstant(), ZoneId.of("UTC")); + } + + LocalDateTime eventTime = LocalDateTime.ofInstant(event.getTimeStamp().toInstant(),ZoneId.of("UTC")); + + if (!persistenceService.isConsistent(event)) { + if (inTimeRange(eventTime, numberOfDays, fromDateTime, toDateTime)) { + count++; + } + } + + pastEvents.put((String) pastEventCondition.getParameter("generatedPropertyKey"), count); return EventService.PROFILE_UPDATED; } + + private boolean inTimeRange(LocalDateTime eventTime, Integer numberOfDays, LocalDateTime fromDate, LocalDateTime toDate) { + boolean inTimeRange = true; + + if (numberOfDays != null) { + LocalDateTime now = LocalDateTime.now(ZoneId.of("UTC")); + if (eventTime.isAfter(now)) { + inTimeRange = false; + } + long daysDiff = Duration.between(eventTime, now).toDays(); + if (daysDiff > numberOfDays) { + inTimeRange = false; + } + } + if (fromDate != null && fromDate.isAfter(eventTime)) { + inTimeRange = false; + } + if (toDate != null && toDate.isBefore(eventTime)) { + inTimeRange = false; + } + + return inTimeRange; + } } diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/SetPropertyAction.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/SetPropertyAction.java index 4134cec9b..e8894b24d 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/SetPropertyAction.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/SetPropertyAction.java @@ -72,6 +72,12 @@ public int execute(Action action, Event event) { propertyValue = format.format(event.getTimeStamp()); } + if (propertyValue != null && propertyValue.equals("eventArrivalTime")) { + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + format.setTimeZone(TimeZone.getTimeZone("UTC")); + propertyValue = format.format(new Date()); + } + if (storeInSession) { // in the case of session storage we directly update the session if (PropertyHelper.setProperty(event.getSession(), propertyName, propertyValue, (String) action.getParameterValues().get("setPropertyStrategy"))) { diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/UpdatePropertiesAction.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/UpdatePropertiesAction.java index 04d2e9b1d..b5b8e65d0 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/UpdatePropertiesAction.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/actions/UpdatePropertiesAction.java @@ -38,6 +38,8 @@ public class UpdatePropertiesAction implements ActionExecutor { public static final String PROPS_TO_UPDATE = "update"; public static final String PROPS_TO_DELETE = "delete"; + public static final String PROPS_TO_ADD_TO_SET = "addToSet"; + public static final String TARGET_ID_KEY = "targetId"; public static final String TARGET_TYPE_KEY = "targetType"; @@ -76,6 +78,11 @@ public int execute(Action action, Event event) { isProfileOrPersonaUpdated |= processProperties(target, propsToUpdate, "alwaysSet"); } + Map propsToAddToSet = (HashMap) event.getProperties().get(PROPS_TO_ADD_TO_SET); + if (propsToAddToSet != null) { + isProfileOrPersonaUpdated |= processProperties(target, propsToAddToSet, "addValues"); + } + List propsToDelete = (List) event.getProperties().get(PROPS_TO_DELETE); if (propsToDelete != null) { for (String prop : propsToDelete) { @@ -119,7 +126,7 @@ private boolean processProperties(Profile target, Map propsMap, } } if (propType != null) { - isProfileOrPersonaUpdated |= PropertyHelper.setProperty(target, prop, PropertyHelper.getValueByTypeId(propsMap.get(prop), propType.getValueTypeId()), "alwaysSet"); + isProfileOrPersonaUpdated |= PropertyHelper.setProperty(target, prop, PropertyHelper.getValueByTypeId(propsMap.get(prop), propType.getValueTypeId()), strategy); } else { isProfileOrPersonaUpdated |= PropertyHelper.setProperty(target, prop, propsMap.get(prop), strategy); } diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionESQueryBuilder.java index 001bd3de0..21d27b127 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionESQueryBuilder.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionESQueryBuilder.java @@ -29,14 +29,15 @@ public class GeoLocationByPointSessionConditionESQueryBuilder implements Conditi @Override public QueryBuilder buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { String type = (String) condition.getParameter("type"); + String name = condition.getParameter("name") == null ? "location" : (String) condition.getParameter("name"); if("circle".equals(type)) { - Double circleLatitude = (Double) condition.getParameter("circleLatitude"); - Double circleLongitude = (Double) condition.getParameter("circleLongitude"); + Double circleLatitude = ((Number) condition.getParameter("circleLatitude")).doubleValue(); + Double circleLongitude = ((Number) condition.getParameter("circleLongitude")).doubleValue(); String distance = condition.getParameter("distance").toString(); if(circleLatitude != null && circleLongitude != null && distance != null) { - return QueryBuilders.geoDistanceQuery("location") + return QueryBuilders.geoDistanceQuery(name) .point(circleLatitude, circleLongitude) .distance(distance); } @@ -47,7 +48,7 @@ public QueryBuilder buildQuery(Condition condition, Map context, Double rectLongitudeSW = (Double) condition.getParameter("rectLongitudeSW"); if(rectLatitudeNE != null && rectLongitudeNE != null && rectLatitudeSW != null && rectLongitudeSW != null) { - return QueryBuilders.geoBoundingBoxQuery("location") + return QueryBuilders.geoBoundingBoxQuery(name) .setCorners(rectLatitudeNE, rectLongitudeNE,rectLatitudeSW, rectLongitudeSW); } } diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionEvaluator.java index 8de237b94..2d3667e07 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionEvaluator.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionEvaluator.java @@ -37,8 +37,11 @@ public class GeoLocationByPointSessionConditionEvaluator implements ConditionEva public boolean eval(Condition condition, Item item, Map context, ConditionEvaluatorDispatcher dispatcher) { try { String type = (String) condition.getParameter("type"); - Double latitudeProperty = Double.parseDouble(BeanUtils.getProperty(item, "properties.location.lat")); - Double longitudeProperty = Double.parseDouble(BeanUtils.getProperty(item, "properties.location.lon")); + String name = condition.getParameter("name") == null ? "properties.location" : (String) condition.getParameter("name"); + + Double latitudeProperty = Double.parseDouble(BeanUtils.getProperty(item, name + ".lat")); + Double longitudeProperty = Double.parseDouble(BeanUtils.getProperty(item, name + ".lon")); + if("circle".equals(type)) { Double circleLatitude = (Double) condition.getParameter("circleLatitude"); @@ -61,7 +64,7 @@ public boolean eval(Condition condition, Item item, Map context, } } } catch (Exception e) { - logger.debug("Cannot get properties", e); + logger.error("Cannot get properties", e); } return false; } diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PastEventConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PastEventConditionESQueryBuilder.java index 65d72d15e..23efbaa92 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PastEventConditionESQueryBuilder.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PastEventConditionESQueryBuilder.java @@ -32,6 +32,7 @@ import org.elasticsearch.index.query.RangeQueryBuilder; import java.util.*; +import java.util.stream.Collectors; public class PastEventConditionESQueryBuilder implements ConditionESQueryBuilder { @@ -41,6 +42,7 @@ public class PastEventConditionESQueryBuilder implements ConditionESQueryBuilder private int maximumIdsQueryCount = 5000; private int aggregateQueryBucketSize = 5000; + private boolean pastEventsDisablePartitions = false; public void setDefinitionsService(DefinitionsService definitionsService) { this.definitionsService = definitionsService; @@ -58,6 +60,10 @@ public void setAggregateQueryBucketSize(int aggregateQueryBucketSize) { this.aggregateQueryBucketSize = aggregateQueryBucketSize; } + public void setPastEventsDisablePartitions(boolean pastEventsDisablePartitions) { + this.pastEventsDisablePartitions = pastEventsDisablePartitions; + } + public void setSegmentService(SegmentService segmentService) { this.segmentService = segmentService; } @@ -89,25 +95,34 @@ public QueryBuilder buildQuery(Condition condition, Map context, Set ids = new HashSet<>(); - // Get full cardinality to partition the terms aggreggation - Map m = persistenceService.getSingleValuesMetrics(eventCondition, new String[]{"card"}, "profileId.keyword", Event.ITEM_TYPE); - long card = m.get("_card").longValue(); - - int numParts = (int) (card / aggregateQueryBucketSize) + 2; - for (int i = 0; i < numParts; i++) { - Map eventCountByProfile = persistenceService.aggregateWithOptimizedQuery(eventCondition, new TermsAggregate("profileId", i, numParts), Event.ITEM_TYPE); - if (eventCountByProfile != null) { - eventCountByProfile.remove("_filtered"); - for (Map.Entry entry : eventCountByProfile.entrySet()) { - if (entry.getValue() < minimumEventCount) { - // No more interesting buckets in this partition - break; - } else if (entry.getValue() <= maximumEventCount) { - ids.add(entry.getKey()); - - if (ids.size() > maximumIdsQueryCount) { - // Avoid building too big ids query - throw exception instead - throw new UnsupportedOperationException("Too many profiles"); + if (pastEventsDisablePartitions) { + Map eventCountByProfile = persistenceService.aggregateWithOptimizedQuery(eventCondition, new TermsAggregate("profileId"), Event.ITEM_TYPE, maximumIdsQueryCount); + ids = eventCountByProfile.entrySet().stream() + .filter(x -> !x.getKey().equals("_filtered")) + .filter(x -> x.getValue() >= minimumEventCount && x.getValue() <= maximumEventCount) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); + } else { + // Get full cardinality to partition the terms aggreggation + Map m = persistenceService.getSingleValuesMetrics(eventCondition, new String[]{"card"}, "profileId.keyword", Event.ITEM_TYPE); + long card = m.get("_card").longValue(); + + int numParts = (int) (card / aggregateQueryBucketSize) + 2; + for (int i = 0; i < numParts; i++) { + Map eventCountByProfile = persistenceService.aggregateWithOptimizedQuery(eventCondition, new TermsAggregate("profileId", i, numParts), Event.ITEM_TYPE); + if (eventCountByProfile != null) { + eventCountByProfile.remove("_filtered"); + for (Map.Entry entry : eventCountByProfile.entrySet()) { + if (entry.getValue() < minimumEventCount) { + // No more interesting buckets in this partition + break; + } else if (entry.getValue() <= maximumEventCount) { + ids.add(entry.getKey()); + + if (ids.size() > maximumIdsQueryCount) { + // Avoid building too big ids query - throw exception instead + throw new UnsupportedOperationException("Too many profiles"); + } } } } @@ -120,16 +135,28 @@ public QueryBuilder buildQuery(Condition condition, Map context, public long count(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { Condition eventCondition = getEventCondition(condition, context); + Map aggResult = null; Integer minimumEventCount = condition.getParameter("minimumEventCount") == null ? 1 : (Integer) condition.getParameter("minimumEventCount"); Integer maximumEventCount = condition.getParameter("maximumEventCount") == null ? Integer.MAX_VALUE : (Integer) condition.getParameter("maximumEventCount"); - // Get full cardinality to partition the terms aggreggation - Map m = persistenceService.getSingleValuesMetrics(eventCondition, new String[]{"card"}, "profileId.keyword", Event.ITEM_TYPE); - long card = m.get("_card").longValue(); + // No count filter - simply get the full number of distinct profiles + if (minimumEventCount == 1 && maximumEventCount == Integer.MAX_VALUE) { + aggResult = persistenceService.getSingleValuesMetrics(eventCondition, new String[]{"card"}, "profileId.keyword", Event.ITEM_TYPE); + return aggResult.get("_card").longValue(); + } - if (minimumEventCount != 1 || maximumEventCount != Integer.MAX_VALUE) { - // Event count specified, must check occurences count for each profile + if (pastEventsDisablePartitions) { + Map eventCountByProfile = persistenceService.aggregateWithOptimizedQuery(eventCondition, new TermsAggregate("profileId"), Event.ITEM_TYPE, maximumIdsQueryCount); + return eventCountByProfile.entrySet().stream() + .filter(x -> x.getKey().equals("_filtered")) + .filter(x -> x.getValue() >= minimumEventCount && x.getValue() <= maximumEventCount) + .count(); + } else { + // Get full cardinality to partition the terms aggreggation + aggResult = persistenceService.getSingleValuesMetrics(eventCondition, new String[]{"card"}, "profileId.keyword", Event.ITEM_TYPE); + long card = aggResult.get("_card").longValue(); + // Event count specified, must check occurences count for each profile int result = 0; int numParts = (int) (card / aggregateQueryBucketSize) + 2; for (int i = 0; i < numParts; i++) { @@ -153,9 +180,6 @@ public long count(Condition condition, Map context, ConditionESQ } } return result; - } else { - // Simply get the full number of distinct profiles - return card; } } @@ -178,6 +202,9 @@ private Condition getEventCondition(Condition condition, Map con l.add(ConditionContextHelper.getContextualCondition(eventCondition, context)); Integer numberOfDays = (Integer) condition.getParameter("numberOfDays"); + String fromDate = (String) condition.getParameter("fromDate"); + String toDate = (String) condition.getParameter("toDate"); + if (numberOfDays != null) { Condition numberOfDaysCondition = new Condition(); numberOfDaysCondition.setConditionType(definitionsService.getConditionType("sessionPropertyCondition")); @@ -186,6 +213,23 @@ private Condition getEventCondition(Condition condition, Map con numberOfDaysCondition.setParameter("propertyValueDateExpr", "now-" + numberOfDays + "d"); l.add(numberOfDaysCondition); } + if (fromDate != null) { + Condition startDateCondition = new Condition(); + startDateCondition.setConditionType(definitionsService.getConditionType("sessionPropertyCondition")); + startDateCondition.setParameter("propertyName", "timeStamp"); + startDateCondition.setParameter("comparisonOperator", "greaterThanOrEqualTo"); + startDateCondition.setParameter("propertyValueDate", fromDate); + l.add(startDateCondition); + } + if (toDate != null) { + Condition endDateCondition = new Condition(); + endDateCondition.setConditionType(definitionsService.getConditionType("sessionPropertyCondition")); + endDateCondition.setParameter("propertyName", "timeStamp"); + endDateCondition.setParameter("comparisonOperator", "lessThanOrEqualTo"); + endDateCondition.setParameter("propertyValueDate", toDate); + l.add(endDateCondition); + } + return andCondition; } diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PastEventConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PastEventConditionEvaluator.java index 8d4a6a29d..1d5533d02 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PastEventConditionEvaluator.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PastEventConditionEvaluator.java @@ -86,6 +86,9 @@ public boolean eval(Condition condition, Item item, Map context, l.add(profileCondition); Integer numberOfDays = (Integer) condition.getParameter("numberOfDays"); + String fromDate = (String) condition.getParameter("fromDate"); + String toDate = (String) condition.getParameter("toDate"); + if (numberOfDays != null) { Condition numberOfDaysCondition = new Condition(); numberOfDaysCondition.setConditionType(definitionsService.getConditionType("sessionPropertyCondition")); @@ -94,6 +97,22 @@ public boolean eval(Condition condition, Item item, Map context, numberOfDaysCondition.setParameter("propertyValueDateExpr", "now-" + numberOfDays + "d"); l.add(numberOfDaysCondition); } + if (fromDate != null) { + Condition startDateCondition = new Condition(); + startDateCondition.setConditionType(definitionsService.getConditionType("sessionPropertyCondition")); + startDateCondition.setParameter("propertyName", "timeStamp"); + startDateCondition.setParameter("comparisonOperator", "greaterThanOrEqualTo"); + startDateCondition.setParameter("propertyValueDate", fromDate); + l.add(startDateCondition); + } + if (toDate != null) { + Condition endDateCondition = new Condition(); + endDateCondition.setConditionType(definitionsService.getConditionType("sessionPropertyCondition")); + endDateCondition.setParameter("propertyName", "timeStamp"); + endDateCondition.setParameter("comparisonOperator", "lessThanOrEqualTo"); + endDateCondition.setParameter("propertyValueDate", toDate); + l.add(endDateCondition); + } count = persistenceService.queryCount(andCondition, Event.ITEM_TYPE); } diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java index b06c94ec4..2a9421e3a 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java @@ -53,17 +53,19 @@ public QueryBuilder buildQuery(Condition condition, Map context, String expectedValue = ConditionContextHelper.foldToASCII((String) condition.getParameter("propertyValue")); Object expectedValueInteger = condition.getParameter("propertyValueInteger"); + Object expectedValueDouble = condition.getParameter("propertyValueDouble"); Object expectedValueDate = convertDateToISO(condition.getParameter("propertyValueDate")); Object expectedValueDateExpr = condition.getParameter("propertyValueDateExpr"); List expectedValues = ConditionContextHelper.foldToASCII((List) condition.getParameter("propertyValues")); List expectedValuesInteger = (List) condition.getParameter("propertyValuesInteger"); + List expectedValuesDouble = (List) condition.getParameter("propertyValuesDouble"); List expectedValuesDate = convertDatesToISO((List) condition.getParameter("propertyValuesDate")); List expectedValuesDateExpr = (List) condition.getParameter("propertyValuesDateExpr"); - Object value = ObjectUtils.firstNonNull(expectedValue, expectedValueInteger, expectedValueDate, expectedValueDateExpr); + Object value = ObjectUtils.firstNonNull(expectedValue, expectedValueInteger, expectedValueDouble, expectedValueDate, expectedValueDateExpr); @SuppressWarnings("unchecked") - List values = ObjectUtils.firstNonNull(expectedValues, expectedValuesInteger, expectedValuesDate, expectedValuesDateExpr); + List values = ObjectUtils.firstNonNull(expectedValues, expectedValuesInteger, expectedValuesDouble, expectedValuesDate, expectedValuesDateExpr); switch (comparisonOperator) { case "equals": @@ -147,7 +149,7 @@ public QueryBuilder buildQuery(Condition condition, Map context, checkRequiredValue(value, name, comparisonOperator, false); return QueryBuilders.boolQuery().mustNot(getIsSameDayRange(value, name)); } - return null; + throw new IllegalArgumentException("Unknown comparisonOperator [" + comparisonOperator + "]"); } private void checkRequiredValuesSize(List values, String name, String operator, int expectedSize) { diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java index 2b5d7f1f9..c47a54891 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java @@ -23,6 +23,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.unomi.api.*; import org.apache.unomi.api.conditions.Condition; +import org.apache.unomi.common.SecureFilteringClassLoader; import org.apache.unomi.persistence.elasticsearch.conditions.ConditionContextHelper; import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluator; import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluatorDispatcher; @@ -56,7 +57,7 @@ public void setUsePropertyConditionOptimizations(boolean usePropertyConditionOpt this.usePropertyConditionOptimizations = usePropertyConditionOptimizations; } - private int compare(Object actualValue, String expectedValue, Object expectedValueDate, Object expectedValueInteger, Object expectedValueDateExpr) { + private int compare(Object actualValue, String expectedValue, Object expectedValueDate, Object expectedValueInteger, Object expectedValueDateExpr, Object expectedValueDouble) { if (expectedValue == null && expectedValueDate == null && expectedValueInteger == null && getDate(expectedValueDateExpr) == null) { return actualValue == null ? 0 : 1; } else if (actualValue == null) { @@ -65,6 +66,8 @@ private int compare(Object actualValue, String expectedValue, Object expectedVal if (expectedValueInteger != null) { return PropertyHelper.getInteger(actualValue).compareTo(PropertyHelper.getInteger(expectedValueInteger)); + } else if (expectedValueDouble != null) { + return PropertyHelper.getDouble(actualValue).compareTo(PropertyHelper.getDouble(expectedValueDouble)); } else if (expectedValueDate != null) { return getDate(actualValue).compareTo(getDate(expectedValueDate)); } else if (expectedValueDateExpr != null) { @@ -148,6 +151,7 @@ public boolean eval(Condition condition, Item item, Map context, String expectedValue = ConditionContextHelper.foldToASCII((String) condition.getParameter("propertyValue")); Object expectedValueInteger = condition.getParameter("propertyValueInteger"); + Object expectedValueDouble = condition.getParameter("propertyValueDouble"); Object expectedValueDate = condition.getParameter("propertyValueDate"); Object expectedValueDateExpr = condition.getParameter("propertyValueDateExpr"); @@ -179,11 +183,20 @@ public boolean eval(Condition condition, Item item, Map context, actualValue = ConditionContextHelper.foldToASCII((String) actualValue); } + return isMatch(op, actualValue, expectedValue, expectedValueInteger, expectedValueDouble, expectedValueDate, + expectedValueDateExpr, condition); + } + + protected boolean isMatch(String op, Object actualValue, String expectedValue, Object expectedValueInteger, Object expectedValueDouble, + Object expectedValueDate, Object expectedValueDateExpr, Condition condition) { if (op == null) { return false; } else if (actualValue == null) { - return op.equals("missing"); + return op.equals("missing") || op.equals("notIn") || op.equals("notEquals"); } else if (op.equals("exists")) { + if (actualValue instanceof List) { + return ((List) actualValue).size() > 0; + } return true; } else if (op.equals("equals")) { if (actualValue instanceof Collection) { @@ -191,36 +204,39 @@ public boolean eval(Condition condition, Item item, Map context, if (o instanceof String) { o = ConditionContextHelper.foldToASCII((String) o); } - if (compare(o, expectedValue, expectedValueDate, expectedValueInteger, expectedValueDateExpr) == 0) { + if (compare(o, expectedValue, expectedValueDate, expectedValueInteger, expectedValueDateExpr, expectedValueDouble) == 0) { return true; } } return false; } - return compare(actualValue, expectedValue, expectedValueDate, expectedValueInteger, expectedValueDateExpr) == 0; + return compare(actualValue, expectedValue, expectedValueDate, expectedValueInteger, expectedValueDateExpr, expectedValueDouble) == 0; } else if (op.equals("notEquals")) { - return compare(actualValue, expectedValue, expectedValueDate, expectedValueInteger, expectedValueDateExpr) != 0; + return compare(actualValue, expectedValue, expectedValueDate, expectedValueInteger, expectedValueDateExpr, expectedValueDouble) != 0; } else if (op.equals("greaterThan")) { - return compare(actualValue, expectedValue, expectedValueDate, expectedValueInteger, expectedValueDateExpr) > 0; + return compare(actualValue, expectedValue, expectedValueDate, expectedValueInteger, expectedValueDateExpr, expectedValueDouble) > 0; } else if (op.equals("greaterThanOrEqualTo")) { - return compare(actualValue, expectedValue, expectedValueDate, expectedValueInteger, expectedValueDateExpr) >= 0; + return compare(actualValue, expectedValue, expectedValueDate, expectedValueInteger, expectedValueDateExpr, expectedValueDouble) >= 0; } else if (op.equals("lessThan")) { - return compare(actualValue, expectedValue, expectedValueDate, expectedValueInteger, expectedValueDateExpr) < 0; + return compare(actualValue, expectedValue, expectedValueDate, expectedValueInteger, expectedValueDateExpr, expectedValueDouble) < 0; } else if (op.equals("lessThanOrEqualTo")) { - return compare(actualValue, expectedValue, expectedValueDate, expectedValueInteger, expectedValueDateExpr) <= 0; + return compare(actualValue, expectedValue, expectedValueDate, expectedValueInteger, expectedValueDateExpr, expectedValueDouble) <= 0; } else if (op.equals("between")) { List expectedValuesInteger = (List) condition.getParameter("propertyValuesInteger"); + List expectedValuesDouble = (List) condition.getParameter("propertyValuesDouble"); List expectedValuesDate = (List) condition.getParameter("propertyValuesDate"); List expectedValuesDateExpr = (List) condition.getParameter("propertyValuesDateExpr"); return compare(actualValue, null, (expectedValuesDate != null && expectedValuesDate.size() >= 1) ? getDate(expectedValuesDate.get(0)) : null, (expectedValuesInteger != null && expectedValuesInteger.size() >= 1) ? (Integer) expectedValuesInteger.get(0) : null, - (expectedValuesDateExpr != null && expectedValuesDateExpr.size() >= 1) ? (String) expectedValuesDateExpr.get(0) : null) >= 0 + (expectedValuesDateExpr != null && expectedValuesDateExpr.size() >= 1) ? (String) expectedValuesDateExpr.get(0) : null, + (expectedValuesDouble != null && expectedValuesDouble.size() >= 1) ? (String) expectedValuesDouble.get(0) : null) >= 0 && compare(actualValue, null, (expectedValuesDate != null && expectedValuesDate.size() >= 2) ? getDate(expectedValuesDate.get(1)) : null, (expectedValuesInteger != null && expectedValuesInteger.size() >= 2) ? (Integer) expectedValuesInteger.get(1) : null, - (expectedValuesDateExpr != null && expectedValuesDateExpr.size() >= 2) ? (String) expectedValuesDateExpr.get(1) : null) <= 0; + (expectedValuesDateExpr != null && expectedValuesDateExpr.size() >= 2) ? (String) expectedValuesDateExpr.get(1) : null, + (expectedValuesDouble != null && expectedValuesDouble.size() >= 2) ? (String) expectedValuesDouble.get(1) : null) <= 0; } else if (op.equals("contains")) { return actualValue.toString().contains(expectedValue); } else if (op.equals("notContains")) { @@ -265,6 +281,9 @@ protected Object getHardcodedPropertyValue(Item item, String expression) { if (expression.startsWith("properties.")) { return getNestedPropertyValue(expression.substring("properties.".length()), event.getProperties()); } + if (expression.equals("persistent")) { + return event.isPersistent(); + } if ("target.itemId".equals(expression)) { return event.getTarget().getItemId(); } @@ -325,8 +344,10 @@ protected Object getHardcodedPropertyValue(Item item, String expression) { } protected Object getOGNLPropertyValue(Item item, String expression) throws Exception { - ExpressionAccessor accessor = getPropertyAccessor(item, expression); - return accessor != null ? accessor.get(getOgnlContext(), item) : null; + ClassLoader secureFilteringClassLoader = new SecureFilteringClassLoader(PropertyConditionEvaluator.class.getClassLoader()); + OgnlContext ognlContext = getOgnlContext(secureFilteringClassLoader); + ExpressionAccessor accessor = getPropertyAccessor(item, expression, ognlContext, secureFilteringClassLoader); + return accessor != null ? accessor.get(ognlContext, item) : null; } private Object getNestedPropertyValue(String expressionPart, Map properties) { @@ -337,35 +358,48 @@ private Object getNestedPropertyValue(String expressionPart, Map if (mapValue == null) { return null; } - String nextExpression = expressionPart.substring(nextDotPos+1); - return getNestedPropertyValue(nextExpression, (Map) mapValue); + String nextExpression = expressionPart.substring(nextDotPos + 1); + return getNestedPropertyValue(nextExpression, (Map) mapValue); } else { return properties.get(expressionPart); } } - private OgnlContext getOgnlContext() { - return (OgnlContext) Ognl.createDefaultContext(null, new MemberAccess() { - @Override - public Object setup(Map context, Object target, Member member, String propertyName) { - return null; - } + private class ClassLoaderClassResolver extends DefaultClassResolver { + private ClassLoader classLoader; - @Override - public void restore(Map context, Object target, Member member, String propertyName, Object state) { + public ClassLoaderClassResolver(ClassLoader classLoader) { + this.classLoader = classLoader; + } - } + @Override + protected Class toClassForName(String className) throws ClassNotFoundException { + return Class.forName(className, true, classLoader); + } + } - @Override - public boolean isAccessible(Map context, Object target, Member member, String propertyName) { - int modifiers = member.getModifiers(); - boolean result = Modifier.isPublic(modifiers); - return result; - } - }); + private OgnlContext getOgnlContext(ClassLoader classLoader) { + return (OgnlContext) Ognl.createDefaultContext(null, new MemberAccess() { + @Override + public Object setup(Map context, Object target, Member member, String propertyName) { + return null; + } + + @Override + public void restore(Map context, Object target, Member member, String propertyName, Object state) { + } + + @Override + public boolean isAccessible(Map context, Object target, Member member, String propertyName) { + int modifiers = member.getModifiers(); + boolean result = Modifier.isPublic(modifiers); + return result; + } + }, new ClassLoaderClassResolver(classLoader), + null); } - private ExpressionAccessor getPropertyAccessor(Item item, String expression) throws Exception { + private ExpressionAccessor getPropertyAccessor(Item item, String expression, OgnlContext ognlContext, ClassLoader classLoader) throws Exception { ExpressionAccessor accessor = null; String clazz = item.getClass().getName(); Map expressions = expressionCache.get(clazz); @@ -380,8 +414,8 @@ private ExpressionAccessor getPropertyAccessor(Item item, String expression) thr Thread current = Thread.currentThread(); ClassLoader contextCL = current.getContextClassLoader(); try { - current.setContextClassLoader(PropertyConditionEvaluator.class.getClassLoader()); - Node node = Ognl.compileExpression(getOgnlContext() , item, expression); + current.setContextClassLoader(classLoader); + Node node = Ognl.compileExpression(ognlContext, item, expression); accessor = node.getAccessor(); } finally { current.setContextClassLoader(contextCL); diff --git a/plugins/baseplugin/src/main/resources/META-INF/cxs/actions/mergeProfilesOnPropertyAction.json b/plugins/baseplugin/src/main/resources/META-INF/cxs/actions/mergeProfilesOnPropertyAction.json index 6082f9f3a..07bf38cde 100644 --- a/plugins/baseplugin/src/main/resources/META-INF/cxs/actions/mergeProfilesOnPropertyAction.json +++ b/plugins/baseplugin/src/main/resources/META-INF/cxs/actions/mergeProfilesOnPropertyAction.json @@ -15,6 +15,11 @@ "id": "mergeProfilePropertyName", "type": "string", "multivalued": false + }, + { + "id": "forceEventProfileAsMaster", + "type": "boolean", + "multivalued": false } ] } \ No newline at end of file diff --git a/plugins/baseplugin/src/main/resources/META-INF/cxs/conditions/geoLocationByPointSessionCondition.json b/plugins/baseplugin/src/main/resources/META-INF/cxs/conditions/geoLocationByPointSessionCondition.json index 809d69095..f33be796c 100644 --- a/plugins/baseplugin/src/main/resources/META-INF/cxs/conditions/geoLocationByPointSessionCondition.json +++ b/plugins/baseplugin/src/main/resources/META-INF/cxs/conditions/geoLocationByPointSessionCondition.json @@ -54,6 +54,11 @@ "id": "distance", "type": "string", "multivalued": false + }, + { + "id": "name", + "type": "string", + "multivalued": false } ] } \ No newline at end of file diff --git a/plugins/baseplugin/src/main/resources/META-INF/cxs/conditions/pastEventCondition.json b/plugins/baseplugin/src/main/resources/META-INF/cxs/conditions/pastEventCondition.json index 71a87f3ba..6740a731d 100644 --- a/plugins/baseplugin/src/main/resources/META-INF/cxs/conditions/pastEventCondition.json +++ b/plugins/baseplugin/src/main/resources/META-INF/cxs/conditions/pastEventCondition.json @@ -20,6 +20,16 @@ "type": "integer", "multivalued": false }, + { + "id": "fromDate", + "type": "date", + "multivalued": false + }, + { + "id": "toDate", + "type": "date", + "multivalued": false + }, { "id": "minimumEventCount", "type": "integer", diff --git a/plugins/baseplugin/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/plugins/baseplugin/src/main/resources/OSGI-INF/blueprint/blueprint.xml index 5a102146f..567706987 100644 --- a/plugins/baseplugin/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ b/plugins/baseplugin/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -34,6 +34,7 @@ + @@ -103,6 +104,7 @@ + diff --git a/plugins/baseplugin/src/test/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluatorTest.java b/plugins/baseplugin/src/test/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluatorTest.java index fefab83f1..dff90fb8f 100644 --- a/plugins/baseplugin/src/test/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluatorTest.java +++ b/plugins/baseplugin/src/test/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluatorTest.java @@ -16,12 +16,14 @@ */ package org.apache.unomi.plugins.baseplugin.conditions; +import ognl.MethodFailedException; import org.apache.unomi.api.CustomItem; import org.apache.unomi.api.Event; import org.apache.unomi.api.Profile; import org.apache.unomi.api.Session; import org.junit.Test; +import java.io.File; import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; @@ -31,6 +33,7 @@ import static junit.framework.TestCase.assertEquals; import static junit.framework.TestCase.assertNull; import static org.apache.unomi.plugins.baseplugin.conditions.PropertyConditionEvaluator.NOT_OPTIMIZED_MARKER; +import static org.junit.Assert.assertFalse; public class PropertyConditionEvaluatorTest { @@ -52,14 +55,14 @@ public void testHardcodedEvaluator() { assertEquals("Target itemId value is not correct", MOCK_ITEM_ID, propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.itemId")); assertEquals("Target scope is not correct", DIGITALL_SCOPE, propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.scope")); assertEquals("Target page path value is not correct", PAGE_PATH_VALUE, propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.properties.pageInfo.pagePath")); - assertEquals("Target page url value is not correct", PAGE_URL_VALUE, propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.properties.pageInfo.pageURL")); + assertEquals("Target page url value is not correct", PAGE_URL_VALUE, propertyConditionEvaluator.getHardcodedPropertyValue(mockEvent, "target.properties.pageInfo.pageURL")); assertEquals("Session size should be 10", 10, propertyConditionEvaluator.getHardcodedPropertyValue(mockSession, "size")); assertNull("Unexisting property should be null", propertyConditionEvaluator.getHardcodedPropertyValue(mockSession, "systemProperties.goals._csk6r4cgeStartReached")); assertNull("Unexisting property should be null", propertyConditionEvaluator.getHardcodedPropertyValue(mockProfile, "properties.email")); // here let's make sure our reporting of non optimized expressions works. - assertEquals("Should have received the non-optimized marker string", NOT_OPTIMIZED_MARKER, propertyConditionEvaluator.getHardcodedPropertyValue(mockSession, "lastEventDate") ); + assertEquals("Should have received the non-optimized marker string", NOT_OPTIMIZED_MARKER, propertyConditionEvaluator.getHardcodedPropertyValue(mockSession, "lastEventDate")); } @@ -67,11 +70,11 @@ public void testHardcodedEvaluator() { public void testOGNLEvaluator() throws Exception { Event mockEvent = generateMockEvent(); assertEquals("Target itemId value is not correct", MOCK_ITEM_ID, propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.itemId")); - assertEquals("Target scope is not correct", DIGITALL_SCOPE,propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.scope")); + assertEquals("Target scope is not correct", DIGITALL_SCOPE, propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.scope")); assertEquals("Target page path value is not correct", PAGE_PATH_VALUE, propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.properties.pageInfo.pagePath")); assertEquals("Target page url value is not correct", PAGE_URL_VALUE, propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "target.properties.pageInfo.pageURL")); assertEquals("Session size should be 10", 10, propertyConditionEvaluator.getOGNLPropertyValue(mockSession, "size")); - assertEquals("Should have received the proper last even date", SESSION_LAST_EVENT_DATE, propertyConditionEvaluator.getOGNLPropertyValue(mockSession, "lastEventDate") ); + assertEquals("Should have received the proper last even date", SESSION_LAST_EVENT_DATE, propertyConditionEvaluator.getOGNLPropertyValue(mockSession, "lastEventDate")); assertNull("Unexisting property should be null", propertyConditionEvaluator.getHardcodedPropertyValue(mockSession, "systemProperties.goals._csk6r4cgeStartReached")); assertNull("Unexisting property should be null", propertyConditionEvaluator.getHardcodedPropertyValue(mockProfile, "properties.email")); @@ -97,7 +100,7 @@ public void testCompareOGNLvsHardcodedPerformance() throws InterruptedException public void testPropertyEvaluator() throws Exception { Event mockEvent = generateMockEvent(); assertEquals("Target itemId value is not correct", MOCK_ITEM_ID, propertyConditionEvaluator.getPropertyValue(mockEvent, "target.itemId")); - assertEquals("Target scope is not correct", DIGITALL_SCOPE,propertyConditionEvaluator.getPropertyValue(mockEvent, "target.scope")); + assertEquals("Target scope is not correct", DIGITALL_SCOPE, propertyConditionEvaluator.getPropertyValue(mockEvent, "target.scope")); assertEquals("Target page path value is not correct", PAGE_PATH_VALUE, propertyConditionEvaluator.getPropertyValue(mockEvent, "target.properties.pageInfo.pagePath")); assertNull("Unexisting property should be null", propertyConditionEvaluator.getPropertyValue(mockSession, "systemProperties.goals._csk6r4cgeStartReached")); @@ -107,6 +110,35 @@ public void testPropertyEvaluator() throws Exception { assertEquals("Session last event date is not right", SESSION_LAST_EVENT_DATE, propertyConditionEvaluator.getPropertyValue(mockSession, "lastEventDate")); } + @Test + public void testOGNLSecurity() throws Exception { + Event mockEvent = generateMockEvent(); + File vulnFile = new File("target/vuln-file.txt"); + if (vulnFile.exists()) { + vulnFile.delete(); + } + try { + propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "@java.lang.Runtime@getRuntime().exec('touch " + vulnFile.getCanonicalPath() + "')"); + } catch (RuntimeException | MethodFailedException re) { + // we ignore these exceptions as they are expected. + } + assertFalse("Vulnerability successfully executed ! File created at " + vulnFile.getCanonicalPath(), vulnFile.exists()); + try { + propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "(#cmd='touch " + vulnFile.getCanonicalPath() + "').(#cmds={'bash','-c',#cmd}).(#p=new java.lang.ProcessBuilder(#cmds)).(#p.redirectErrorStream(true)).(#process=#p.start())"); + } catch (RuntimeException | MethodFailedException re) { + // we ignore these exceptions as they are expected. + } + vulnFile = new File("target/vuln-file.txt"); + assertFalse("Vulnerability successfully executed ! File created at " + vulnFile.getCanonicalPath(), vulnFile.exists()); + try { + propertyConditionEvaluator.getOGNLPropertyValue(mockEvent, "(#cmd='touch " + vulnFile.getCanonicalPath() + "').(#iswin=(@java.lang.System@getProperty('os.name').toLowerCase().contains('win'))).(#cmds=(#iswin?{'cmd.exe','/c',#cmd}:{'bash','-c',#cmd})).(#p=new java.lang.ProcessBuilder(#cmds)).(#p.redirectErrorStream(true)).(#process=#p.start())"); + } catch (RuntimeException | MethodFailedException re) { + // we ignore these exceptions as they are expected. + } + vulnFile = new File("target/vuln-file.txt"); + assertFalse("Vulnerability successfully executed ! File created at " + vulnFile.getCanonicalPath(), vulnFile.exists()); + } + private void runHardcodedTest(int workerCount, ExecutorService executorService) throws InterruptedException { List> todo = new ArrayList>(workerCount); long startTime = System.currentTimeMillis(); @@ -135,7 +167,7 @@ private static Event generateMockEvent() { targetItem.setItemId(MOCK_ITEM_ID); targetItem.setScope(DIGITALL_SCOPE); mockEvent.setTarget(targetItem); - Map pageInfoMap = new HashMap<>(); + Map pageInfoMap = new HashMap<>(); pageInfoMap.put("pagePath", PAGE_PATH_VALUE); pageInfoMap.put("pageURL", PAGE_URL_VALUE); targetItem.getProperties().put("pageInfo", pageInfoMap); diff --git a/plugins/hover-event/pom.xml b/plugins/hover-event/pom.xml index 8307cebf5..e935881c3 100644 --- a/plugins/hover-event/pom.xml +++ b/plugins/hover-event/pom.xml @@ -22,7 +22,7 @@ org.apache.unomi unomi-plugins - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-plugins-hover-event diff --git a/plugins/kafka-injector/pom.xml b/plugins/kafka-injector/pom.xml index 5eeed8bef..58f33549d 100644 --- a/plugins/kafka-injector/pom.xml +++ b/plugins/kafka-injector/pom.xml @@ -20,7 +20,7 @@ org.apache.unomi unomi-plugins - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT kafka-extension diff --git a/plugins/mail/pom.xml b/plugins/mail/pom.xml index 1d18d77f6..01763ca39 100644 --- a/plugins/mail/pom.xml +++ b/plugins/mail/pom.xml @@ -22,7 +22,7 @@ org.apache.unomi unomi-plugins - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-plugins-mail diff --git a/plugins/mail/src/main/java/org/apache/unomi/plugins/mail/actions/SendMailAction.java b/plugins/mail/src/main/java/org/apache/unomi/plugins/mail/actions/SendMailAction.java index 8e031752f..acdb1eb7f 100644 --- a/plugins/mail/src/main/java/org/apache/unomi/plugins/mail/actions/SendMailAction.java +++ b/plugins/mail/src/main/java/org/apache/unomi/plugins/mail/actions/SendMailAction.java @@ -116,7 +116,7 @@ public int execute(Action action, Event event) { event.getProfile().setSystemProperty("notificationAck", profileNotif); event.getProfile().setSystemProperty("lastUpdated", new Date()); - persistenceService.update(event.getProfile().getItemId(), null, Profile.class, "systemProperties", event.getProfile().getSystemProperties()); + persistenceService.update(event.getProfile(), null, Profile.class, "systemProperties", event.getProfile().getSystemProperties()); ST stringTemplate = new ST(template, '$', '$'); stringTemplate.add("profile", event.getProfile()); diff --git a/plugins/optimization-test/pom.xml b/plugins/optimization-test/pom.xml index 02031a056..4c9c45be0 100644 --- a/plugins/optimization-test/pom.xml +++ b/plugins/optimization-test/pom.xml @@ -22,7 +22,7 @@ org.apache.unomi unomi-plugins - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-plugins-optimization-test diff --git a/plugins/past-event/pom.xml b/plugins/past-event/pom.xml index 5f4d131d7..1975a79b5 100644 --- a/plugins/past-event/pom.xml +++ b/plugins/past-event/pom.xml @@ -22,7 +22,7 @@ org.apache.unomi unomi-plugins - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-plugins-past-event diff --git a/plugins/pom.xml b/plugins/pom.xml index 2cf4becac..060b13226 100644 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -22,7 +22,7 @@ org.apache.unomi unomi-root - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-plugins diff --git a/plugins/request/pom.xml b/plugins/request/pom.xml index 4cc8f8347..a0c2d80e2 100644 --- a/plugins/request/pom.xml +++ b/plugins/request/pom.xml @@ -22,7 +22,7 @@ org.apache.unomi unomi-plugins - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-plugins-request diff --git a/plugins/tracked-event/pom.xml b/plugins/tracked-event/pom.xml index 5a9636158..94c18902d 100644 --- a/plugins/tracked-event/pom.xml +++ b/plugins/tracked-event/pom.xml @@ -22,7 +22,7 @@ org.apache.unomi unomi-plugins - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-plugins-tracked-event diff --git a/pom.xml b/pom.xml index 222826b6f..4329273cc 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ event tracking server. https://unomi.apache.org - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT pom @@ -69,7 +69,7 @@ 2.9.10.1 2.9.10 4.2.8 - 4.1.3 + 4.2.0 4.13.1 7.4.2 3.0.3 @@ -84,6 +84,8 @@ 2.1.2 4.3.0 + 2.4.0 + 2.11.0_1 2.9.0 @@ -118,7 +120,7 @@ scm:git:https://gitbox.apache.org/repos/asf/unomi.git scm:git:https://gitbox.apache.org/repos/asf/unomi.git - HEAD + unomi-root-1.5.1 https://gitbox.apache.org/repos/asf/unomi.git @@ -971,7 +973,7 @@ ognl ognl - 3.2.11 + 3.2.14 org.javassist @@ -1033,10 +1035,10 @@ plugins extensions tools + manual kar samples package - manual @@ -1046,6 +1048,11 @@ 1.7.1 provided + + net.jodah + failsafe + ${fail-safe.version} + diff --git a/rest/pom.xml b/rest/pom.xml index ca02f35e4..85192dcf6 100644 --- a/rest/pom.xml +++ b/rest/pom.xml @@ -22,7 +22,7 @@ org.apache.unomi unomi-root - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-rest @@ -92,6 +92,10 @@ commons-lang 2.6 + + org.apache.commons + commons-lang3 + + + + + + + + + + + + + + + + @@ -77,6 +99,10 @@ + + + + @@ -89,6 +115,10 @@ + + + + @@ -101,6 +131,10 @@ + + + + @@ -113,6 +147,10 @@ + + + + @@ -125,6 +163,10 @@ + + + + @@ -137,6 +179,10 @@ + + + + @@ -149,6 +195,10 @@ + + + + @@ -161,6 +211,10 @@ + + + + @@ -173,6 +227,10 @@ + + + + @@ -185,6 +243,10 @@ + + + + @@ -197,6 +259,10 @@ + + + + diff --git a/rest/src/main/resources/org.apache.unomi.rest.cfg b/rest/src/main/resources/org.apache.unomi.rest.cfg new file mode 100644 index 000000000..e61a81189 --- /dev/null +++ b/rest/src/main/resources/org.apache.unomi.rest.cfg @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +returnFullErrorMessage=${org.apache.unomi.rest.returnFullErrorMessage:-false} \ No newline at end of file diff --git a/samples/groovy-actions/pom.xml b/samples/groovy-actions/pom.xml index ffb0cc2a6..ab5a624de 100644 --- a/samples/groovy-actions/pom.xml +++ b/samples/groovy-actions/pom.xml @@ -21,7 +21,7 @@ samples org.apache.unomi - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT test-groovy-action-sample diff --git a/samples/login-integration/pom.xml b/samples/login-integration/pom.xml index 20a63d98b..6813fc47e 100644 --- a/samples/login-integration/pom.xml +++ b/samples/login-integration/pom.xml @@ -21,7 +21,7 @@ samples org.apache.unomi - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT login-integration-sample diff --git a/samples/pom.xml b/samples/pom.xml index 8d9c685d2..4501491b7 100644 --- a/samples/pom.xml +++ b/samples/pom.xml @@ -19,7 +19,7 @@ unomi-root org.apache.unomi - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT 4.0.0 @@ -30,6 +30,7 @@ tweet-button-plugin login-integration groovy-actions + trainingplugin \ No newline at end of file diff --git a/samples/trainingplugin/pom.xml b/samples/trainingplugin/pom.xml index 5991114c7..c171322cb 100644 --- a/samples/trainingplugin/pom.xml +++ b/samples/trainingplugin/pom.xml @@ -15,13 +15,11 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + samples org.apache.unomi - 1.3.0-incubating-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT training-plugin Apache Unomi :: Samples :: Training plugin @@ -33,7 +31,7 @@ org.apache.unomi unomi-api - 1.3.0-incubating-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT provided diff --git a/samples/trainingplugin/src/main/java/org/apache/unomi/training/TrainedNotificationAction.java b/samples/trainingplugin/src/main/java/org/apache/unomi/training/TrainedNotificationAction.java index d9493b0d0..74c2242e2 100644 --- a/samples/trainingplugin/src/main/java/org/apache/unomi/training/TrainedNotificationAction.java +++ b/samples/trainingplugin/src/main/java/org/apache/unomi/training/TrainedNotificationAction.java @@ -48,9 +48,10 @@ public int execute(Action action, Event event) { if (trained == null) { // create trained flag property type - PropertyType propertyType = new PropertyType(new Metadata(event.getScope(), TRAINED_NB_PROPERTY, TRAINED_NB_PROPERTY, "Am I trained")); + Metadata propertyTypeMetadata = new Metadata(event.getScope(), TRAINED_NB_PROPERTY, TRAINED_NB_PROPERTY, "Am I trained"); + propertyTypeMetadata.setSystemTags(Collections.singleton("training")); + PropertyType propertyType = new PropertyType(); propertyType.setValueTypeId("boolean"); - propertyType.setTags(Collections.singleton("training")); propertyType.setTarget(TARGET); service.setPropertyType(propertyType); } diff --git a/samples/tweet-button-plugin/pom.xml b/samples/tweet-button-plugin/pom.xml index 5f7babe20..d3ee43ea7 100644 --- a/samples/tweet-button-plugin/pom.xml +++ b/samples/tweet-button-plugin/pom.xml @@ -21,7 +21,7 @@ samples org.apache.unomi - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT tweet-button-plugin diff --git a/services/pom.xml b/services/pom.xml index 7ca0b9a5d..3d778a730 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -22,7 +22,7 @@ org.apache.unomi unomi-root - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-services @@ -133,6 +133,13 @@ provided + + org.apache.unomi + unomi-common + ${project.version} + provided + + com.github.seancfoley ipaddress diff --git a/services/src/main/java/org/apache/unomi/services/actions/ActionExecutorDispatcher.java b/services/src/main/java/org/apache/unomi/services/actions/ActionExecutorDispatcher.java index 91a912541..e148a94e6 100644 --- a/services/src/main/java/org/apache/unomi/services/actions/ActionExecutorDispatcher.java +++ b/services/src/main/java/org/apache/unomi/services/actions/ActionExecutorDispatcher.java @@ -24,6 +24,7 @@ import org.apache.unomi.api.actions.ActionDispatcher; import org.apache.unomi.api.actions.ActionExecutor; import org.apache.unomi.api.services.EventService; +import org.apache.unomi.common.SecureFilteringClassLoader; import org.apache.unomi.metrics.MetricAdapter; import org.apache.unomi.metrics.MetricsService; import org.mvel2.MVEL; @@ -98,23 +99,9 @@ public Object extract(String valueAsString, Event event) throws IllegalAccessExc valueExtractors.put("script", new ValueExtractor() { @Override public Object extract(String valueAsString, Event event) throws IllegalAccessException, NoSuchMethodException, InvocationTargetException { - final ClassLoader tccl = Thread.currentThread().getContextClassLoader(); - try { - Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); - if (!mvelExpressions.containsKey(valueAsString)) { - ParserConfiguration parserConfiguration = new ParserConfiguration(); - parserConfiguration.setClassLoader(getClass().getClassLoader()); - mvelExpressions.put(valueAsString, MVEL.compileExpression(valueAsString, new ParserContext(parserConfiguration))); - } - Map ctx = new HashMap<>(); - ctx.put("event", event); - ctx.put("session", event.getSession()); - ctx.put("profile", event.getProfile()); - return MVEL.executeExpression(mvelExpressions.get(valueAsString), ctx); - } finally { - Thread.currentThread().setContextClassLoader(tccl); - } + return executeScript(valueAsString, event); } + }); } @@ -239,4 +226,24 @@ public void unbindDispatcher(ServiceReference actionDispatcher } } + protected Object executeScript(String valueAsString, Event event) { + final ClassLoader tccl = Thread.currentThread().getContextClassLoader(); + try { + ClassLoader secureFilteringClassLoader = new SecureFilteringClassLoader(getClass().getClassLoader()); + Thread.currentThread().setContextClassLoader(secureFilteringClassLoader); + if (!mvelExpressions.containsKey(valueAsString)) { + ParserConfiguration parserConfiguration = new ParserConfiguration(); + parserConfiguration.setClassLoader(secureFilteringClassLoader); + mvelExpressions.put(valueAsString, MVEL.compileExpression(valueAsString, new ParserContext(parserConfiguration))); + } + Map ctx = new HashMap<>(); + ctx.put("event", event); + ctx.put("session", event.getSession()); + ctx.put("profile", event.getProfile()); + return MVEL.executeExpression(mvelExpressions.get(valueAsString), ctx); + } finally { + Thread.currentThread().setContextClassLoader(tccl); + } + } + } diff --git a/services/src/main/java/org/apache/unomi/services/impl/ParserHelper.java b/services/src/main/java/org/apache/unomi/services/impl/ParserHelper.java index a861f6ccc..9f5afa97b 100644 --- a/services/src/main/java/org/apache/unomi/services/impl/ParserHelper.java +++ b/services/src/main/java/org/apache/unomi/services/impl/ParserHelper.java @@ -22,8 +22,12 @@ import org.apache.unomi.api.actions.Action; import org.apache.unomi.api.actions.ActionType; import org.apache.unomi.api.conditions.Condition; +import org.apache.unomi.api.conditions.ConditionHookFactory; import org.apache.unomi.api.conditions.ConditionType; import org.apache.unomi.api.services.DefinitionsService; +import org.apache.unomi.api.conditions.ConditionHook; +import org.osgi.framework.BundleContext; +import org.osgi.framework.ServiceReference; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,12 +42,19 @@ public class ParserHelper { private static final Set unresolvedActionTypes = new HashSet<>(); private static final Set unresolvedConditionTypes = new HashSet<>(); + private BundleContext bundleContext; - public static boolean resolveConditionType(final DefinitionsService definitionsService, Condition rootCondition) { + public void setBundleContext(BundleContext bundleContext) { + this.bundleContext = bundleContext; + } + + public boolean resolveConditionType(DefinitionsService definitionsService, Condition rootCondition) { if (rootCondition == null) { return false; } final List result = new ArrayList(); + List conditionHooks = getConditionHooks(); + visitConditions(rootCondition, new ConditionVisitor() { @Override public void visit(Condition condition) { @@ -60,12 +71,13 @@ public void visit(Condition condition) { } } } + hookCondition(conditionHooks, condition); } }); return result.isEmpty(); } - public static List getConditionTypeIds(Condition rootCondition) { + public List getConditionTypeIds(Condition rootCondition) { final List result = new ArrayList(); visitConditions(rootCondition, new ConditionVisitor() { @Override @@ -76,7 +88,7 @@ public void visit(Condition condition) { return result; } - private static void visitConditions(Condition rootCondition, ConditionVisitor visitor) { + private void visitConditions(Condition rootCondition, ConditionVisitor visitor) { visitor.visit(rootCondition); // recursive call for sub-conditions as parameters for (Object parameterValue : rootCondition.getParameterValues().values()) { @@ -96,15 +108,15 @@ private static void visitConditions(Condition rootCondition, ConditionVisitor vi } } - public static boolean resolveActionTypes(DefinitionsService definitionsService, List actions) { + public boolean resolveActionTypes(DefinitionsService definitionsService, List actions) { boolean result = true; for (Action action : actions) { - result &= ParserHelper.resolveActionType(definitionsService, action); + result &= resolveActionType(definitionsService, action); } return result; } - public static boolean resolveActionType(DefinitionsService definitionsService, Action action) { + public boolean resolveActionType(DefinitionsService definitionsService, Action action) { if (action.getActionType() == null) { ActionType actionType = definitionsService.getActionType(action.getActionTypeId()); if (actionType != null) { @@ -121,7 +133,7 @@ public static boolean resolveActionType(DefinitionsService definitionsService, A return true; } - public static void resolveValueType(DefinitionsService definitionsService, PropertyType propertyType) { + public void resolveValueType(DefinitionsService definitionsService, PropertyType propertyType) { if (propertyType.getValueType() == null) { ValueType valueType = definitionsService.getValueType(propertyType.getValueTypeId()); if (valueType != null) { @@ -130,6 +142,29 @@ public static void resolveValueType(DefinitionsService definitionsService, Prope } } + private void hookCondition(List conditionHooks, Condition condition) { + for (ConditionHook conditionHook: conditionHooks) { + conditionHook.executeHook(condition); + } + } + + private List getConditionHooks() { + List conditionHooks = new LinkedList<>(); + try { + ServiceReference[] serviceReferences = (ServiceReference[]) bundleContext + .getAllServiceReferences(ConditionHookFactory.class.getName(), null); + if (serviceReferences != null) { + for (ServiceReference serviceReference : serviceReferences) { + ConditionHookFactory conditionHookFactory = bundleContext.getService(serviceReference); + conditionHooks.add(conditionHookFactory.createConditionHook()); + } + } + } catch(Exception e) { + throw new RuntimeException(e); + } + return conditionHooks; + } + interface ConditionVisitor { void visit(Condition condition); } diff --git a/services/src/main/java/org/apache/unomi/services/impl/definitions/DefinitionsServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/definitions/DefinitionsServiceImpl.java index 10cb59829..deec0d977 100644 --- a/services/src/main/java/org/apache/unomi/services/impl/definitions/DefinitionsServiceImpl.java +++ b/services/src/main/java/org/apache/unomi/services/impl/definitions/DefinitionsServiceImpl.java @@ -20,6 +20,7 @@ import org.apache.unomi.api.PluginType; import org.apache.unomi.api.PropertyMergeStrategyType; import org.apache.unomi.api.ValueType; +import org.apache.unomi.api.actions.Action; import org.apache.unomi.api.actions.ActionType; import org.apache.unomi.api.conditions.Condition; import org.apache.unomi.api.conditions.ConditionType; @@ -58,10 +59,17 @@ public class DefinitionsServiceImpl implements DefinitionsService, SynchronousBu private long definitionsRefreshInterval = 10000; private BundleContext bundleContext; + + private ParserHelper parserHelper; + public DefinitionsServiceImpl() { } + public void setParserHelper(ParserHelper parserHelper) { + this.parserHelper = parserHelper; + } + public void setBundleContext(BundleContext bundleContext) { this.bundleContext = bundleContext; } @@ -277,7 +285,7 @@ public Collection getAllConditionTypes() { Collection all = persistenceService.getAllItems(ConditionType.class); for (ConditionType type : all) { if (type != null && type.getParentCondition() != null) { - ParserHelper.resolveConditionType(this, type.getParentCondition()); + parserHelper.resolveConditionType(this, type.getParentCondition()); } } return all; @@ -296,7 +304,7 @@ private Set getConditionTypesBy(String fieldName, String fieldVal List directConditionTypes = persistenceService.query(fieldName, fieldValue,null, ConditionType.class); for (ConditionType type : directConditionTypes) { if (type.getParentCondition() != null) { - ParserHelper.resolveConditionType(this, type.getParentCondition()); + parserHelper.resolveConditionType(this, type.getParentCondition()); } } conditionTypes.addAll(directConditionTypes); @@ -316,7 +324,7 @@ public ConditionType getConditionType(String id) { } } if (type != null && type.getParentCondition() != null) { - ParserHelper.resolveConditionType(this, type.getParentCondition()); + parserHelper.resolveConditionType(this, type.getParentCondition()); } return type; } @@ -516,7 +524,17 @@ public Condition extractConditionBySystemTag(Condition rootCondition, String sys @Override public boolean resolveConditionType(Condition rootCondition) { - return ParserHelper.resolveConditionType(this, rootCondition); + return parserHelper.resolveConditionType(this, rootCondition); + } + + @Override + public boolean resolveActionType(Action action) { + return parserHelper.resolveActionType(this, action); + } + + @Override + public boolean resolveActionTypes(List actions) { + return parserHelper.resolveActionTypes(this, actions); } @Override diff --git a/services/src/main/java/org/apache/unomi/services/impl/events/EventServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/events/EventServiceImpl.java index a6e6bbcd9..d9b647522 100644 --- a/services/src/main/java/org/apache/unomi/services/impl/events/EventServiceImpl.java +++ b/services/src/main/java/org/apache/unomi/services/impl/events/EventServiceImpl.java @@ -32,7 +32,6 @@ import org.apache.unomi.api.services.EventService; import org.apache.unomi.persistence.spi.PersistenceService; import org.apache.unomi.persistence.spi.aggregate.TermsAggregate; -import org.apache.unomi.services.impl.ParserHelper; import org.osgi.framework.BundleContext; import org.osgi.framework.ServiceReference; import org.slf4j.Logger; @@ -152,7 +151,7 @@ private int send(Event event, int depth) { boolean saveSucceeded = true; if (event.isPersistent()) { - saveSucceeded = persistenceService.save(event); + saveSucceeded = persistenceService.save(event, null, true); } int changes; @@ -220,7 +219,7 @@ public Set getEventTypeIds() { @Override public PartialList searchEvents(Condition condition, int offset, int size) { - ParserHelper.resolveConditionType(definitionsService, condition); + definitionsService.resolveConditionType(condition); return persistenceService.query(condition, "timeStamp", Event.class, offset, size); } @@ -282,7 +281,13 @@ public Event getEvent(String id) { public boolean hasEventAlreadyBeenRaised(Event event) { Event pastEvent = this.persistenceService.load(event.getItemId(), Event.class); - return pastEvent != null && pastEvent.getVersion() >= 1 && pastEvent.getSessionId().equals(event.getSessionId()); + if (pastEvent != null && pastEvent.getVersion() >= 1) { + if ((pastEvent.getSessionId() != null && pastEvent.getSessionId().equals(event.getSessionId())) || + (pastEvent.getProfileId() != null && pastEvent.getProfileId().equals(event.getProfileId()))) { + return true; + } + } + return false; } public boolean hasEventAlreadyBeenRaised(Event event, boolean session) { diff --git a/services/src/main/java/org/apache/unomi/services/impl/goals/GoalsServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/goals/GoalsServiceImpl.java index cf2788edc..105601971 100644 --- a/services/src/main/java/org/apache/unomi/services/impl/goals/GoalsServiceImpl.java +++ b/services/src/main/java/org/apache/unomi/services/impl/goals/GoalsServiceImpl.java @@ -37,7 +37,6 @@ import org.apache.unomi.persistence.spi.CustomObjectMapper; import org.apache.unomi.persistence.spi.PersistenceService; import org.apache.unomi.persistence.spi.aggregate.*; -import org.apache.unomi.services.impl.ParserHelper; import org.osgi.framework.Bundle; import org.osgi.framework.BundleContext; import org.osgi.framework.BundleEvent; @@ -219,8 +218,8 @@ public Set getGoalMetadatas(Query query) { public Goal getGoal(String goalId) { Goal goal = persistenceService.load(goalId, Goal.class); if (goal != null) { - ParserHelper.resolveConditionType(definitionsService, goal.getStartEvent()); - ParserHelper.resolveConditionType(definitionsService, goal.getTargetEvent()); + definitionsService.resolveConditionType(goal.getStartEvent()); + definitionsService.resolveConditionType(goal.getTargetEvent()); } return goal; } @@ -234,8 +233,8 @@ public void removeGoal(String goalId) { @Override public void setGoal(Goal goal) { - ParserHelper.resolveConditionType(definitionsService, goal.getStartEvent()); - ParserHelper.resolveConditionType(definitionsService, goal.getTargetEvent()); + definitionsService.resolveConditionType(goal.getStartEvent()); + definitionsService.resolveConditionType(goal.getTargetEvent()); if (goal.getMetadata().isEnabled()) { if (goal.getStartEvent() != null) { @@ -407,7 +406,7 @@ private CampaignDetail getCampaignDetail(Campaign campaign) { public Campaign getCampaign(String id) { Campaign campaign = persistenceService.load(id, Campaign.class); if (campaign != null) { - ParserHelper.resolveConditionType(definitionsService, campaign.getEntryCondition()); + definitionsService.resolveConditionType(campaign.getEntryCondition()); } return campaign; } @@ -421,7 +420,7 @@ public void removeCampaign(String id) { } public void setCampaign(Campaign campaign) { - ParserHelper.resolveConditionType(definitionsService, campaign.getEntryCondition()); + definitionsService.resolveConditionType(campaign.getEntryCondition()); if(rulesService.getRule(campaign.getMetadata().getId() + "EntryEvent") != null) { rulesService.removeRule(campaign.getMetadata().getId() + "EntryEvent"); @@ -466,7 +465,7 @@ public GoalReport getGoalReport(String goalId, AggregateQuery query) { } if (query != null && query.getCondition() != null) { - ParserHelper.resolveConditionType(definitionsService, query.getCondition()); + definitionsService.resolveConditionType(query.getCondition()); list.add(query.getCondition()); } diff --git a/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java index c0269edab..e459e3222 100644 --- a/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java +++ b/services/src/main/java/org/apache/unomi/services/impl/profiles/ProfileServiceImpl.java @@ -32,7 +32,6 @@ import org.apache.unomi.persistence.spi.CustomObjectMapper; import org.apache.unomi.persistence.spi.PersistenceService; import org.apache.unomi.persistence.spi.PropertyHelper; -import org.apache.unomi.services.impl.ParserHelper; import org.osgi.framework.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -109,6 +108,7 @@ public PropertyTypes with(PropertyType newProperty) { /** * Creates a new instance of this class containing given property types. * If property types with the same ID existed before, they will be replaced by the new ones. + * * @param newProperties list of property types to change * @return new instance */ @@ -130,13 +130,14 @@ public PropertyTypes with(List newProperties) { /** * Creates a new instance of this class containing all property types except the one with given ID. + * * @param propertyId ID of the property to delete * @return new instance */ public PropertyTypes without(String propertyId) { List newPropertyTypes = allPropertyTypes.stream() - .filter(property -> property.getItemId().equals(propertyId)) - .collect(Collectors.toList()); + .filter(property -> property.getItemId().equals(propertyId)) + .collect(Collectors.toList()); return new PropertyTypes(newPropertyTypes); } @@ -175,6 +176,8 @@ private void updateListMap(Map> listMap, PropertyType private boolean forceRefreshOnSave = false; + private String defaultMergeStrategy = "defaultMergeStrategy"; + public ProfileServiceImpl() { logger.info("Initializing profile service..."); } @@ -207,6 +210,10 @@ public void setPropertiesRefreshInterval(long propertiesRefreshInterval) { this.propertiesRefreshInterval = propertiesRefreshInterval; } + public void setDefaultMergeStrategy(String defaultMergeStrategy) { + this.defaultMergeStrategy = defaultMergeStrategy; + } + public void postConstruct() { logger.debug("postConstruct {" + bundleContext.getBundle() + "}"); @@ -374,17 +381,20 @@ public PartialList searchSessions(Query query) { } private PartialList doSearch(Query query, Class clazz) { + if (query.getScrollIdentifier() != null) { + return persistenceService.continueScrollQuery(clazz, query.getScrollIdentifier(), query.getScrollTimeValidity()); + } if (query.getCondition() != null && definitionsService.resolveConditionType(query.getCondition())) { if (StringUtils.isNotBlank(query.getText())) { return persistenceService.queryFullText(query.getText(), query.getCondition(), query.getSortby(), clazz, query.getOffset(), query.getLimit()); } else { - return persistenceService.query(query.getCondition(), query.getSortby(), clazz, query.getOffset(), query.getLimit()); + return persistenceService.query(query.getCondition(), query.getSortby(), clazz, query.getOffset(), query.getLimit(), query.getScrollTimeValidity()); } } else { if (StringUtils.isNotBlank(query.getText())) { return persistenceService.queryFullText(query.getText(), query.getSortby(), clazz, query.getOffset(), query.getLimit()); } else { - return persistenceService.getAllItems(clazz, query.getOffset(), query.getLimit(), query.getSortby()); + return persistenceService.getAllItems(clazz, query.getOffset(), query.getLimit(), query.getSortby(), query.getScrollTimeValidity()); } } } @@ -598,7 +608,8 @@ public Profile mergeProfiles(Profile masterProfile, List profilesToMerg Set allProfileProperties = new LinkedHashSet<>(); for (Profile profile : profilesToMerge) { - allProfileProperties.addAll(profile.getProperties().keySet()); + final Set flatNestedPropertiesKeys = PropertyHelper.flatten(profile.getProperties()).keySet(); + allProfileProperties.addAll(flatNestedPropertiesKeys); } Collection profilePropertyTypes = getTargetPropertyTypes("profiles"); @@ -616,7 +627,7 @@ public Profile mergeProfiles(Profile masterProfile, List profilesToMerg for (String profileProperty : allProfileProperties) { PropertyType propertyType = profilePropertyTypeById.get(profileProperty); - String propertyMergeStrategyId = "defaultMergeStrategy"; + String propertyMergeStrategyId = defaultMergeStrategy; if (propertyType != null) { if (propertyType.getMergeStrategy() != null && propertyMergeStrategyId.length() > 0) { propertyMergeStrategyId = propertyType.getMergeStrategy(); @@ -625,13 +636,13 @@ public Profile mergeProfiles(Profile masterProfile, List profilesToMerg PropertyMergeStrategyType propertyMergeStrategyType = definitionsService.getPropertyMergeStrategyType(propertyMergeStrategyId); if (propertyMergeStrategyType == null) { // we couldn't find the strategy - if (propertyMergeStrategyId.equals("defaultMergeStrategy")) { + if (propertyMergeStrategyId.equals(defaultMergeStrategy)) { logger.warn("Couldn't resolve default strategy, ignoring property merge for property " + profileProperty); continue; } else { // todo: improper algorithm… it is possible that the defaultMergeStrategy couldn't be resolved here logger.warn("Couldn't resolve strategy " + propertyMergeStrategyId + " for property " + profileProperty + ", using default strategy instead"); - propertyMergeStrategyId = "defaultMergeStrategy"; + propertyMergeStrategyId = defaultMergeStrategy; propertyMergeStrategyType = definitionsService.getPropertyMergeStrategyType(propertyMergeStrategyId); } } @@ -667,12 +678,12 @@ public Profile mergeProfiles(Profile masterProfile, List profilesToMerg // we now have to merge the profile's consents for (Profile profile : profilesToMerge) { if (profile.getConsents() != null && profile.getConsents().size() > 0) { - for(String consentId : profile.getConsents().keySet()) { - if(masterProfile.getConsents().containsKey(consentId)) { - if(masterProfile.getConsents().get(consentId).getRevokeDate().before(new Date())) { + for (String consentId : profile.getConsents().keySet()) { + if (masterProfile.getConsents().containsKey(consentId)) { + if (masterProfile.getConsents().get(consentId).getRevokeDate().before(new Date())) { masterProfile.getConsents().remove(consentId); masterProfileChanged = true; - } else if(masterProfile.getConsents().get(consentId).getStatusDate().before(profile.getConsents().get(consentId).getStatusDate())) { + } else if (masterProfile.getConsents().get(consentId).getStatusDate().before(profile.getConsents().get(consentId).getStatusDate())) { masterProfile.getConsents().replace(consentId, profile.getConsents().get(consentId)); masterProfileChanged = true; } @@ -756,12 +767,12 @@ public void removeProfileSessions(String profileId) { profileCondition.setParameter("comparisonOperator", "equals"); profileCondition.setParameter("propertyValue", profileId); - persistenceService.removeByQuery(profileCondition,Session.class); + persistenceService.removeByQuery(profileCondition, Session.class); } @Override public boolean matchCondition(Condition condition, Profile profile, Session session) { - ParserHelper.resolveConditionType(definitionsService, condition); + definitionsService.resolveConditionType(condition); if (condition.getConditionTypeId().equals("booleanCondition")) { List subConditions = (List) condition.getParameter("subConditions"); @@ -786,7 +797,7 @@ public boolean matchCondition(Condition condition, Profile profile, Session sess } public void batchProfilesUpdate(BatchUpdate update) { - ParserHelper.resolveConditionType(definitionsService, update.getCondition()); + definitionsService.resolveConditionType(update.getCondition()); List profiles = persistenceService.query(update.getCondition(), null, Profile.class); for (Profile profile : profiles) { @@ -1037,8 +1048,8 @@ private boolean merge(Map target, Map object) { changed = true; } } else if (newEntry.getValue().getClass().isEnum()) { - target.put(newEntry.getKey(), newEntry.getValue()); - changed = true; + target.put(newEntry.getKey(), newEntry.getValue()); + changed = true; } else { if (target.get(newEntry.getKey()) != null) { changed |= merge(target.get(newEntry.getKey()), newEntry.getValue()); diff --git a/services/src/main/java/org/apache/unomi/services/impl/queries/QueryServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/queries/QueryServiceImpl.java index ca278eb9a..714476f58 100644 --- a/services/src/main/java/org/apache/unomi/services/impl/queries/QueryServiceImpl.java +++ b/services/src/main/java/org/apache/unomi/services/impl/queries/QueryServiceImpl.java @@ -23,7 +23,6 @@ import org.apache.unomi.api.services.QueryService; import org.apache.unomi.persistence.spi.PersistenceService; import org.apache.unomi.persistence.spi.aggregate.*; -import org.apache.unomi.services.impl.ParserHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -74,7 +73,7 @@ public Map getAggregateWithOptimizedQuery(String itemType, String @Override public Map getMetric(String type, String property, String slashConcatenatedMetrics, Condition condition) { if (condition.getConditionType() == null) { - ParserHelper.resolveConditionType(definitionsService, condition); + definitionsService.resolveConditionType(condition); } return persistenceService.getSingleValuesMetrics(condition, slashConcatenatedMetrics.split("/"), property, type); } @@ -82,7 +81,7 @@ public Map getMetric(String type, String property, String slashC @Override public long getQueryCount(String itemType, Condition condition) { if (condition.getConditionType() == null) { - ParserHelper.resolveConditionType(definitionsService, condition); + definitionsService.resolveConditionType(condition); } return persistenceService.queryCount(condition, itemType); } @@ -91,7 +90,7 @@ private Map getAggregate(String itemType, String property, Aggrega if (query != null) { // resolve condition if (query.getCondition() != null) { - ParserHelper.resolveConditionType(definitionsService, query.getCondition()); + definitionsService.resolveConditionType(query.getCondition()); } // resolve aggregate diff --git a/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java index d77173c50..8038c9c1e 100644 --- a/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java +++ b/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java @@ -30,7 +30,6 @@ import org.apache.unomi.persistence.spi.CustomObjectMapper; import org.apache.unomi.persistence.spi.PersistenceService; import org.apache.unomi.services.actions.ActionExecutorDispatcher; -import org.apache.unomi.services.impl.ParserHelper; import org.osgi.framework.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -248,8 +247,8 @@ private void updateRuleStatistics(RuleStatistics ruleStatistics, long ruleCondit private List getAllRules() { List allItems = persistenceService.getAllItems(Rule.class, 0, -1, "priority").getList(); for (Rule rule : allItems) { - ParserHelper.resolveConditionType(definitionsService, rule.getCondition()); - ParserHelper.resolveActionTypes(definitionsService, rule.getActions()); + definitionsService.resolveConditionType(rule.getCondition()); + definitionsService.resolveActionTypes(rule.getActions()); } return allItems; } @@ -340,10 +339,10 @@ public Rule getRule(String ruleId) { Rule rule = persistenceService.load(ruleId, Rule.class); if (rule != null) { if (rule.getCondition() != null) { - ParserHelper.resolveConditionType(definitionsService, rule.getCondition()); + definitionsService.resolveConditionType(rule.getCondition()); } if (rule.getActions() != null) { - ParserHelper.resolveActionTypes(definitionsService, rule.getActions()); + definitionsService.resolveActionTypes(rule.getActions()); } } return rule; @@ -356,7 +355,7 @@ public void setRule(Rule rule) { Condition condition = rule.getCondition(); if (condition != null) { if (rule.getMetadata().isEnabled() && !rule.getMetadata().isMissingPlugins()) { - ParserHelper.resolveConditionType(definitionsService, condition); + definitionsService.resolveConditionType(condition); definitionsService.extractConditionBySystemTag(condition, "eventCondition"); } } @@ -373,7 +372,7 @@ public Set getTrackedConditions(Item source){ if(trackedCondition != null){ Condition sourceEventPropertyCondition = definitionsService.extractConditionBySystemTag(r.getCondition(), "sourceEventCondition"); if(source != null && sourceEventPropertyCondition != null) { - ParserHelper.resolveConditionType(definitionsService, sourceEventPropertyCondition); + definitionsService.resolveConditionType(sourceEventPropertyCondition); if(persistenceService.testMatch(sourceEventPropertyCondition, source)){ trackedConditions.add(trackedCondition); } @@ -475,7 +474,7 @@ private void syncRuleStatistics() { } allRuleStatistics.put(ruleStatistics.getItemId(), ruleStatistics); if (mustPersist) { - persistenceService.save(ruleStatistics); + persistenceService.save(ruleStatistics, null, true); } } // now let's iterate over the rules coming from the persistence service, as we may have new ones. diff --git a/services/src/main/java/org/apache/unomi/services/impl/scheduler/SchedulerServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/scheduler/SchedulerServiceImpl.java index 7e7f1cfca..2311beee9 100644 --- a/services/src/main/java/org/apache/unomi/services/impl/scheduler/SchedulerServiceImpl.java +++ b/services/src/main/java/org/apache/unomi/services/impl/scheduler/SchedulerServiceImpl.java @@ -21,6 +21,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.Duration; +import java.time.ZonedDateTime; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -45,4 +47,14 @@ public void preDestroy() { public ScheduledExecutorService getScheduleExecutorService() { return scheduler; } + + public static long getTimeDiffInSeconds(int hourInUtc, ZonedDateTime now) { + ZonedDateTime nextRun = now.withHour(hourInUtc).withMinute(0).withSecond(0); + if(now.compareTo(nextRun) > 0) + nextRun = nextRun.plusDays(1); + + Duration duration = Duration.between(now, nextRun); + long initialDelay = duration.getSeconds(); + return initialDelay; + } } diff --git a/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java index 9c362eddc..97ef82136 100644 --- a/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java +++ b/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java @@ -35,7 +35,7 @@ import org.apache.unomi.persistence.spi.CustomObjectMapper; import org.apache.unomi.persistence.spi.aggregate.TermsAggregate; import org.apache.unomi.services.impl.AbstractServiceImpl; -import org.apache.unomi.services.impl.ParserHelper; +import org.apache.unomi.services.impl.scheduler.SchedulerServiceImpl; import org.osgi.framework.Bundle; import org.osgi.framework.BundleContext; import org.osgi.framework.BundleEvent; @@ -46,8 +46,11 @@ import java.io.IOException; import java.net.URL; import java.security.MessageDigest; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.util.*; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentService, SynchronousBundleListener { @@ -65,6 +68,13 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe private int segmentUpdateBatchSize = 1000; private long segmentRefreshInterval = 1000; private int aggregateQueryBucketSize = 5000; + private int maxRetriesForUpdateProfileSegment = 0; + private long secondsDelayForRetryUpdateProfileSegment = 1; + private boolean segmentProfileUpdateByQuery = false; + private boolean sendProfileUpdateEventForSegmentUpdate = true; + private int maximumIdsQueryCount = 5000; + private boolean pastEventsDisablePartitions = false; + private int dailyDateExprEvaluationHourUtc = 5; public SegmentServiceImpl() { logger.info("Initializing segment service..."); @@ -94,10 +104,38 @@ public void setAggregateQueryBucketSize(int aggregateQueryBucketSize) { this.aggregateQueryBucketSize = aggregateQueryBucketSize; } + public void setMaximumIdsQueryCount(int maximumIdsQueryCount) { + this.maximumIdsQueryCount = maximumIdsQueryCount; + } + + public void setPastEventsDisablePartitions(boolean pastEventsDisablePartitions) { + this.pastEventsDisablePartitions = pastEventsDisablePartitions; + } + public void setSegmentRefreshInterval(long segmentRefreshInterval) { this.segmentRefreshInterval = segmentRefreshInterval; } + public void setMaxRetriesForUpdateProfileSegment(int maxRetriesForUpdateProfileSegment) { + this.maxRetriesForUpdateProfileSegment = maxRetriesForUpdateProfileSegment; + } + + public void setSecondsDelayForRetryUpdateProfileSegment(long secondsDelayForRetryUpdateProfileSegment) { + this.secondsDelayForRetryUpdateProfileSegment = secondsDelayForRetryUpdateProfileSegment; + } + + public void setSegmentProfileUpdateByQuery(boolean segmentProfileUpdateByQuery) { + this.segmentProfileUpdateByQuery = segmentProfileUpdateByQuery; + } + + public void setSendProfileUpdateEventForSegmentUpdate(boolean sendProfileUpdateEventForSegmentUpdate){ + this.sendProfileUpdateEventForSegmentUpdate = sendProfileUpdateEventForSegmentUpdate; + } + + public void setDailyDateExprEvaluationHourUtc(int dailyDateExprEvaluationHourUtc) { + this.dailyDateExprEvaluationHourUtc = dailyDateExprEvaluationHourUtc; + } + public void postConstruct() { logger.debug("postConstruct {" + bundleContext.getBundle() + "}"); loadPredefinedSegments(bundleContext); @@ -208,7 +246,7 @@ public PartialList getSegmentMetadatas(Query query) { private List getAllSegmentDefinitions() { List allItems = persistenceService.getAllItems(Segment.class); for (Segment segment : allItems) { - ParserHelper.resolveConditionType(definitionsService, segment.getCondition()); + definitionsService.resolveConditionType(segment.getCondition()); } return allItems; } @@ -216,20 +254,25 @@ private List getAllSegmentDefinitions() { public Segment getSegmentDefinition(String segmentId) { Segment definition = persistenceService.load(segmentId, Segment.class); if (definition != null) { - ParserHelper.resolveConditionType(definitionsService, definition.getCondition()); + definitionsService.resolveConditionType(definition.getCondition()); } return definition; } public void setSegmentDefinition(Segment segment) { - ParserHelper.resolveConditionType(definitionsService, segment.getCondition()); - if (segment.getMetadata().isEnabled() && !segment.getMetadata().isMissingPlugins()) { - updateAutoGeneratedRules(segment.getMetadata(), segment.getCondition()); + try { + definitionsService.resolveConditionType(segment.getCondition()); + persistenceService.validateCondition(segment.getCondition(), new Profile()); + if (segment.getMetadata().isEnabled() && !segment.getMetadata().isMissingPlugins()) { + updateAutoGeneratedRules(segment.getMetadata(), segment.getCondition()); + } + // make sure we update the name and description metadata that might not match, so first we remove the entry from the map + persistenceService.save(segment, null, true); + updateExistingProfilesForSegment(segment); + } catch (Exception e) { + logger.error("failed at setSegmentDefinition, condition={}", segment.getCondition().toString(), e); + throw e; } - // make sure we update the name and description metadata that might not match, so first we remove the entry from the map - persistenceService.save(segment); - - updateExistingProfilesForSegment(segment); } private boolean checkSegmentDeletionImpact(Condition condition, String segmentToDeleteId) { @@ -337,27 +380,9 @@ public DependentMetadata getSegmentDependentMetadata(String segmentId) { public DependentMetadata removeSegmentDefinition(String segmentId, boolean validate) { Set impactedSegments = getSegmentDependentSegments(segmentId); Set impactedScorings = getSegmentDependentScorings(segmentId); - if (!validate || (impactedSegments.isEmpty() && impactedScorings.isEmpty())) { - // update profiles - Condition segmentCondition = new Condition(); - segmentCondition.setConditionType(definitionsService.getConditionType("profilePropertyCondition")); - segmentCondition.setParameter("propertyName", "segments"); - segmentCondition.setParameter("comparisonOperator", "equals"); - segmentCondition.setParameter("propertyValue", segmentId); - List previousProfiles = persistenceService.query(segmentCondition, null, Profile.class); - long updatedProfileCount = 0; - long profileRemovalStartTime = System.currentTimeMillis(); - for (Profile profileToRemove : previousProfiles) { - profileToRemove.getSegments().remove(segmentId); - Map sourceMap = new HashMap<>(); - sourceMap.put("segments", profileToRemove.getSegments()); - profileToRemove.setSystemProperty("lastUpdated", new Date()); - sourceMap.put("systemProperties", profileToRemove.getSystemProperties()); - persistenceService.update(profileToRemove.getItemId(), null, Profile.class, sourceMap); - updatedProfileCount++; - } - logger.info("Removed segment from {} profiles in {} ms", updatedProfileCount, System.currentTimeMillis() - profileRemovalStartTime); + if (!validate || (impactedSegments.isEmpty() && impactedScorings.isEmpty())) { + removeSegmentFromProfiles(segmentId); // update impacted segments for (Segment segment : impactedSegments) { @@ -393,6 +418,42 @@ public DependentMetadata removeSegmentDefinition(String segmentId, boolean valid clearAutoGeneratedRules(previousRules, segmentId); } + return getDependentMetadata(impactedSegments, impactedScorings); + } + + private void removeSegmentFromProfiles(String segmentId) { + // update profiles + Condition segmentCondition = new Condition(); + segmentCondition.setConditionType(definitionsService.getConditionType("profilePropertyCondition")); + segmentCondition.setParameter("propertyName", "segments"); + segmentCondition.setParameter("comparisonOperator", "equals"); + segmentCondition.setParameter("propertyValue", segmentId); + + long queryTime = System.currentTimeMillis(); + if (segmentProfileUpdateByQuery) { + long updatedProfileCount = updateProfilesSegmentByQuery(segmentCondition, segmentId, false); + if (updatedProfileCount > -1) { + logger.info("Removed segment from {} profiles in {} ms", updatedProfileCount, System.currentTimeMillis() - queryTime); + } else { + logger.warn("Removing segment from profiles failed in {} ms", System.currentTimeMillis() - queryTime); + } + } else { + List previousProfiles = persistenceService.query(segmentCondition, null, Profile.class); + logger.info("removeSegmentDefinition, updateProfilesSegment {} profiles queryTime of segment {} in {}ms", previousProfiles.size(), segmentId, System.currentTimeMillis() - queryTime); + long updatedProfileCount = 0; + long profileRemovalStartTime = System.currentTimeMillis(); + for (Profile profileToRemove : previousProfiles) { + Map sourceMap = buildPropertiesMapForUpdateSegment(profileToRemove, segmentId, false); + persistenceService.update(profileToRemove, null, Profile.class, sourceMap); + } + + updatedProfileCount += previousProfiles.size(); + logger.info("Removed segment from {} profiles in {} ms", updatedProfileCount, System.currentTimeMillis() - profileRemovalStartTime); + // no re-query with scroll? + } + } + + private DependentMetadata getDependentMetadata(Set impactedSegments, Set impactedScorings) { List segments = new LinkedList<>(); List scorings = new LinkedList<>(); for (Segment definition : impactedSegments) { @@ -404,7 +465,6 @@ public DependentMetadata removeSegmentDefinition(String segmentId, boolean valid return new DependentMetadata(segments, scorings); } - public PartialList getMatchingIndividuals(String segmentID, int offset, int size, String sortBy) { Segment segment = getSegmentDefinition(segmentID); if (segment == null) { @@ -494,7 +554,7 @@ private List getAllScoringDefinitions() { List allItems = persistenceService.getAllItems(Scoring.class); for (Scoring scoring : allItems) { for (ScoringElement element : scoring.getElements()) { - ParserHelper.resolveConditionType(definitionsService, element.getCondition()); + definitionsService.resolveConditionType(element.getCondition()); } } return allItems; @@ -504,7 +564,7 @@ public Scoring getScoringDefinition(String scoringId) { Scoring definition = persistenceService.load(scoringId, Scoring.class); if (definition != null) { for (ScoringElement element : definition.getElements()) { - ParserHelper.resolveConditionType(definitionsService, element.getCondition()); + definitionsService.resolveConditionType(element.getCondition()); } } return definition; @@ -512,7 +572,7 @@ public Scoring getScoringDefinition(String scoringId) { public void setScoringDefinition(Scoring scoring) { for (ScoringElement element : scoring.getElements()) { - ParserHelper.resolveConditionType(definitionsService, element.getCondition()); + definitionsService.resolveConditionType(element.getCondition()); } for (ScoringElement element : scoring.getElements()) { if (scoring.getMetadata().isEnabled() && !scoring.getMetadata().isMissingPlugins()) { @@ -523,17 +583,17 @@ public void setScoringDefinition(Scoring scoring) { persistenceService.save(scoring); persistenceService.createMapping(Profile.ITEM_TYPE, String.format( - "{\n" + - " \"properties\": {\n" + - " \"scores\": {\n" + - " \"properties\": {\n" + - " \"%s\": {\n" + - " \"type\":\"long\"\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - "}", scoring.getItemId())); + "{\n" + + " \"properties\": {\n" + + " \"scores\": {\n" + + " \"properties\": {\n" + + " \"%s\": {\n" + + " \"type\":\"long\"\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", scoring.getItemId())); updateExistingProfilesForScoring(scoring); } @@ -682,15 +742,7 @@ public DependentMetadata removeScoringDefinition(String scoringId, boolean valid clearAutoGeneratedRules(previousRules, scoringId); } - List segments = new LinkedList<>(); - List scorings = new LinkedList<>(); - for (Segment definition : impactedSegments) { - segments.add(definition.getMetadata()); - } - for (Scoring definition : impactedScorings) { - scorings.add(definition.getMetadata()); - } - return new DependentMetadata(segments, scorings); + return getDependentMetadata(impactedSegments, impactedScorings); } public void updateAutoGeneratedRules(Metadata metadata, Condition condition) { @@ -713,7 +765,7 @@ private void clearAutoGeneratedRules(List rules, String idWithScope) { // todo remove profile properties ? persistenceService.remove(previousRule.getItemId(), Rule.class); } else { - persistenceService.update(previousRule.getItemId(), null, Rule.class, "linkedItems", previousRule.getLinkedItems()); + persistenceService.update(previousRule, null, Rule.class, "linkedItems", previousRule.getLinkedItems()); } } } @@ -770,6 +822,9 @@ private void updateExistingProfilesForPastEventCondition(Condition eventConditio l.add(eventCondition); Integer numberOfDays = (Integer) parentCondition.getParameter("numberOfDays"); + String fromDate = (String) parentCondition.getParameter("fromDate"); + String toDate = (String) parentCondition.getParameter("toDate"); + if (numberOfDays != null) { Condition numberOfDaysCondition = new Condition(); numberOfDaysCondition.setConditionType(definitionsService.getConditionType("sessionPropertyCondition")); @@ -778,28 +833,35 @@ private void updateExistingProfilesForPastEventCondition(Condition eventConditio numberOfDaysCondition.setParameter("propertyValue", "now-" + numberOfDays + "d"); l.add(numberOfDaysCondition); } + if (fromDate != null) { + Condition startDateCondition = new Condition(); + startDateCondition.setConditionType(definitionsService.getConditionType("sessionPropertyCondition")); + startDateCondition.setParameter("propertyName", "timeStamp"); + startDateCondition.setParameter("comparisonOperator", "greaterThanOrEqualTo"); + startDateCondition.setParameter("propertyValueDate", fromDate); + l.add(startDateCondition); + } + if (toDate != null) { + Condition endDateCondition = new Condition(); + endDateCondition.setConditionType(definitionsService.getConditionType("sessionPropertyCondition")); + endDateCondition.setParameter("propertyName", "timeStamp"); + endDateCondition.setParameter("comparisonOperator", "lessThanOrEqualTo"); + endDateCondition.setParameter("propertyValueDate", toDate); + l.add(endDateCondition); + } + String propertyKey = (String) parentCondition.getParameter("generatedPropertyKey"); - Map m = persistenceService.getSingleValuesMetrics(andCondition, new String[]{"card"}, "profileId.keyword", Event.ITEM_TYPE); - long card = m.get("_card").longValue(); - - int numParts = (int) (card / aggregateQueryBucketSize) + 2; - for (int i = 0; i < numParts; i++) { - Map eventCountByProfile = persistenceService.aggregateWithOptimizedQuery(andCondition, new TermsAggregate("profileId", i, numParts), Event.ITEM_TYPE); - for (Map.Entry entry : eventCountByProfile.entrySet()) { - String profileId = entry.getKey(); - if (!profileId.startsWith("_")) { - Map pastEventCounts = new HashMap<>(); - pastEventCounts.put(propertyKey, entry.getValue()); - Map systemProperties = new HashMap<>(); - systemProperties.put("pastEvents", pastEventCounts); - try { - systemProperties.put("lastUpdated", new Date()); - persistenceService.update(profileId, null, Profile.class, "systemProperties", systemProperties); - } catch (Exception e) { - logger.error("Error updating profile {} past event system properties", profileId, e); - } - } + if(pastEventsDisablePartitions) { + Map eventCountByProfile = persistenceService.aggregateWithOptimizedQuery(eventCondition, new TermsAggregate("profileId"), Event.ITEM_TYPE, maximumIdsQueryCount); + updateProfilesWithPastEventProperty(eventCountByProfile, propertyKey); + } else { + Map m = persistenceService.getSingleValuesMetrics(andCondition, new String[]{"card"}, "profileId.keyword", Event.ITEM_TYPE); + long card = m.get("_card").longValue(); + int numParts = (int) (card / aggregateQueryBucketSize) + 2; + for (int i = 0; i < numParts; i++) { + Map eventCountByProfile = persistenceService.aggregateWithOptimizedQuery(andCondition, new TermsAggregate("profileId", i, numParts), Event.ITEM_TYPE); + updateProfilesWithPastEventProperty(eventCountByProfile, propertyKey); } } @@ -811,6 +873,16 @@ public String getGeneratedPropertyKey(Condition condition, Condition parentCondi Map m = new HashMap<>(); m.put("condition", condition); m.put("numberOfDays", parentCondition.getParameter("numberOfDays")); + // Put fromDate and toDate only if exist - for backward compatibility + Object fromDate = parentCondition.getParameter("fromDate"); + if (fromDate != null) { + m.put("fromDate", parentCondition.getParameter("fromDate")); + } + Object toDate = parentCondition.getParameter("toDate"); + if (toDate != null) { + m.put("fromDate", parentCondition.getParameter("toDate")); + } + String key = CustomObjectMapper.getObjectMapper().writeValueAsString(m); return "eventTriggered" + getMD5(key); } catch (JsonProcessingException e) { @@ -819,6 +891,27 @@ public String getGeneratedPropertyKey(Condition condition, Condition parentCondi } } + private void updateProfilesWithPastEventProperty(Map eventCountByProfile, String propertyKey) { + for (Map.Entry entry : eventCountByProfile.entrySet()) { + String profileId = entry.getKey(); + if (!profileId.startsWith("_")) { + Map pastEventCounts = new HashMap<>(); + pastEventCounts.put(propertyKey, entry.getValue()); + Map systemProperties = new HashMap<>(); + systemProperties.put("pastEvents", pastEventCounts); + try { + systemProperties.put("lastUpdated", new Date()); + Profile profile = new Profile(); + profile.setItemId(profileId); + persistenceService.update(profile, null, Profile.class, "systemProperties", systemProperties); + } catch (Exception e) { + logger.error("Error updating profile {} past event system properties", profileId, e); + } + } + } + + } + private String getMD5(String md5) { try { MessageDigest md = MessageDigest.getInstance("MD5"); @@ -835,14 +928,14 @@ private String getMD5(String md5) { private void updateExistingProfilesForSegment(Segment segment) { long updateProfilesForSegmentStartTime = System.currentTimeMillis(); - Condition segmentCondition = new Condition(); - long updatedProfileCount = 0; + final String segmentId = segment.getItemId(); + Condition segmentCondition = new Condition(); segmentCondition.setConditionType(definitionsService.getConditionType("profilePropertyCondition")); segmentCondition.setParameter("propertyName", "segments"); segmentCondition.setParameter("comparisonOperator", "equals"); - segmentCondition.setParameter("propertyValue", segment.getItemId()); + segmentCondition.setParameter("propertyValue", segmentId); if (segment.getMetadata().isEnabled()) { @@ -867,76 +960,81 @@ private void updateExistingProfilesForSegment(Segment segment) { profilesToRemoveSubConditions.add(notNewSegmentCondition); profilesToRemoveCondition.setParameter("subConditions", profilesToRemoveSubConditions); - PartialList profilesToRemove = persistenceService.query(profilesToRemoveCondition, null, Profile.class, 0, segmentUpdateBatchSize, "10m"); - PartialList profilesToAdd = persistenceService.query(profilesToAddCondition, null, Profile.class, 0, segmentUpdateBatchSize, "10m"); - - while (profilesToAdd.getList().size() > 0) { - long profilesToAddStartTime = System.currentTimeMillis(); - for (Profile profileToAdd : profilesToAdd.getList()) { - profileToAdd.getSegments().add(segment.getItemId()); - Map sourceMap = new HashMap<>(); - sourceMap.put("segments", profileToAdd.getSegments()); - profileToAdd.setSystemProperty("lastUpdated", new Date()); - sourceMap.put("systemProperties", profileToAdd.getSystemProperties()); - persistenceService.update(profileToAdd.getItemId(), null, Profile.class, sourceMap); - Event profileUpdated = new Event("profileUpdated", null, profileToAdd, null, null, profileToAdd, new Date()); - profileUpdated.setPersistent(false); - eventService.send(profileUpdated); - updatedProfileCount++; - } - logger.info("{} profiles added to segment in {}ms", profilesToAdd.size(), System.currentTimeMillis() - profilesToAddStartTime); - profilesToAdd = persistenceService.continueScrollQuery(Profile.class, profilesToAdd.getScrollIdentifier(), profilesToAdd.getScrollTimeValidity()); - if (profilesToAdd == null || profilesToAdd.getList().size() == 0) { - break; - } - } - while (profilesToRemove.getList().size() > 0) { - long profilesToRemoveStartTime = System.currentTimeMillis(); - for (Profile profileToRemove : profilesToRemove.getList()) { - profileToRemove.getSegments().remove(segment.getItemId()); - Map sourceMap = new HashMap<>(); - sourceMap.put("segments", profileToRemove.getSegments()); - profileToRemove.setSystemProperty("lastUpdated", new Date()); - sourceMap.put("systemProperties", profileToRemove.getSystemProperties()); - persistenceService.update(profileToRemove.getItemId(), null, Profile.class, sourceMap); - Event profileUpdated = new Event("profileUpdated", null, profileToRemove, null, null, profileToRemove, new Date()); - profileUpdated.setPersistent(false); - eventService.send(profileUpdated); - updatedProfileCount++; - } - logger.info("{} profiles removed from segment in {}ms", profilesToRemove.size(), System.currentTimeMillis() - profilesToRemoveStartTime ); - profilesToRemove = persistenceService.continueScrollQuery(Profile.class, profilesToRemove.getScrollIdentifier(), profilesToRemove.getScrollTimeValidity()); - if (profilesToRemove == null || profilesToRemove.getList().size() == 0) { - break; - } - } - + updatedProfileCount += updateProfilesSegment(profilesToAddCondition, segmentId, true); + updatedProfileCount += updateProfilesSegment(profilesToRemoveCondition, segmentId, false); } else { - PartialList profilesToRemove = persistenceService.query(segmentCondition, null, Profile.class, 0, 200, "10m"); - while (profilesToRemove.getList().size() > 0) { - long profilesToRemoveStartTime = System.currentTimeMillis(); - for (Profile profileToRemove : profilesToRemove.getList()) { - profileToRemove.getSegments().remove(segment.getItemId()); - Map sourceMap = new HashMap<>(); - sourceMap.put("segments", profileToRemove.getSegments()); - profileToRemove.setSystemProperty("lastUpdated", new Date()); - sourceMap.put("systemProperties", profileToRemove.getSystemProperties()); - persistenceService.update(profileToRemove.getItemId(), null, Profile.class, sourceMap); - Event profileUpdated = new Event("profileUpdated", null, profileToRemove, null, null, profileToRemove, new Date()); - profileUpdated.setPersistent(false); - eventService.send(profileUpdated); - updatedProfileCount++; - } - logger.info("{} profiles removed from segment in {}ms", profilesToRemove.size(), System.currentTimeMillis() - profilesToRemoveStartTime); - profilesToRemove = persistenceService.continueScrollQuery(Profile.class, profilesToRemove.getScrollIdentifier(), profilesToRemove.getScrollTimeValidity()); - if (profilesToRemove == null || profilesToRemove.getList().size() == 0) { - break; - } - } + updatedProfileCount += updateProfilesSegment(segmentCondition, segmentId, false); } logger.info("{} profiles updated in {}ms", updatedProfileCount, System.currentTimeMillis() - updateProfilesForSegmentStartTime); } + private long updateProfilesSegmentByQuery(Condition profilesToUpdateCondition, String segmentId, boolean isAdd) { + Map[] scriptParams = new HashMap[1]; + scriptParams[0] = new HashMap<>(); + scriptParams[0].put("listPropName", "segments"); + scriptParams[0].put("item", segmentId); + Condition[] conditions = new Condition[] { profilesToUpdateCondition }; + + return persistenceService.updateListWithQuery(null, Profile.class, true, + scriptParams, conditions, maxRetriesForUpdateProfileSegment, secondsDelayForRetryUpdateProfileSegment, + isAdd, segmentUpdateBatchSize); + } + + private long updateProfilesSegment(Condition profilesToUpdateCondition, String segmentId, boolean isAdd){ + if (segmentProfileUpdateByQuery) { + logger.info("Updating segment {} by query", segmentId); + return updateProfilesSegmentByQuery(profilesToUpdateCondition, segmentId, isAdd); + } + + long updatedProfileCount= 0; + long queryTime = System.currentTimeMillis(); + PartialList profiles = persistenceService.query(profilesToUpdateCondition, null, Profile.class, 0, segmentUpdateBatchSize, "10m"); + logger.info("updateProfilesSegment {} batch profiles queryTime of segment {} in {}ms", profiles.size(), segmentId, System.currentTimeMillis() - queryTime); + while (profiles != null && profiles.getList().size() > 0) { + long startTime = System.currentTimeMillis(); + for (Profile profileToUpdate : profiles.getList()) { + Map sourceMap = buildPropertiesMapForUpdateSegment(profileToUpdate, segmentId, isAdd); + persistenceService.update(profileToUpdate, null, Profile.class, sourceMap); + } + if (sendProfileUpdateEventForSegmentUpdate) + sendProfileUpdatedEvent(profiles.getList()); + + updatedProfileCount += profiles.size(); + logger.info("updateProfilesSegment {} profiles {} to segment {} in {}ms", profiles.size(), isAdd ? "added" : "removed", segmentId, System.currentTimeMillis() - startTime); + + long scrollTime = System.currentTimeMillis(); + profiles = persistenceService.continueScrollQuery(Profile.class, profiles.getScrollIdentifier(), profiles.getScrollTimeValidity()); + logger.info("updateProfilesSegment {} profiles scrollTime of segment {} in {}ms", profiles.size(),segmentId, System.currentTimeMillis() - scrollTime); + } + + return updatedProfileCount; + } + + private void sendProfileUpdatedEvent(List profiles) { + for (Profile profileToAdd : profiles) { + sendProfileUpdatedEvent(profileToAdd); + } + } + + private void sendProfileUpdatedEvent(Profile profile) { + Event profileUpdated = new Event("profileUpdated", null, profile, null, null, profile, new Date()); + profileUpdated.setPersistent(false); + eventService.send(profileUpdated); + } + + private Map buildPropertiesMapForUpdateSegment(Profile profile, String segmentId, boolean isAdd) { + if (isAdd) + profile.getSegments().add(segmentId); + else + profile.getSegments().remove(segmentId); + + Map sourceMap = new HashMap<>(); + sourceMap.put("segments", profile.getSegments()); + profile.setSystemProperty("lastUpdated", new Date()); + sourceMap.put("systemProperties", profile.getSystemProperties()); + return sourceMap; + } + private void updateExistingProfilesForScoring(Scoring scoring) { long startTime = System.currentTimeMillis(); Condition scoringCondition = new Condition(); @@ -1006,10 +1104,13 @@ public void bundleChanged(BundleEvent event) { } private void initializeTimer() { + TimerTask task = new TimerTask() { @Override public void run() { try { + logger.info("running scheduled task to recalculate segments with pastEventCondition conditions"); + long pastEventsTaskStartTime = System.currentTimeMillis(); for (Metadata metadata : rulesService.getRuleMetadatas()) { Rule rule = rulesService.getRule(metadata.getId()); for (Action action : rule.getActions()) { @@ -1021,6 +1122,7 @@ public void run() { } } } + logger.info("finished recalculate segments with pastEventCondition conditions in {}ms. ", System.currentTimeMillis() - pastEventsTaskStartTime); } catch (Throwable t) { logger.error("Error while updating profiles for past event conditions", t); } @@ -1040,6 +1142,26 @@ public void run() { } }; schedulerService.getScheduleExecutorService().scheduleAtFixedRate(task, 0, segmentRefreshInterval, TimeUnit.MILLISECONDS); + + task = new TimerTask() { + @Override + public void run() { + try { + long dateExprTaskStartTime = System.currentTimeMillis(); + List dateExprSegments = allSegments.stream().filter(segment -> + segment.getCondition().toString().contains("propertyValueDateExpr")).collect(Collectors.toList()); + logger.info("running scheduled task to recalculate segments with DateExpr condition, found {} segments", dateExprSegments.size()); + dateExprSegments.forEach(segment -> updateExistingProfilesForSegment(segment)); + logger.info("finished recalculate segments with DateExpr conditions in {}ms. ", System.currentTimeMillis() - dateExprTaskStartTime); + } catch (Throwable t) { + logger.error("Error while updating profiles for DateExpr conditions", t); + } + } + }; + + long initialDelay = SchedulerServiceImpl.getTimeDiffInSeconds(dailyDateExprEvaluationHourUtc, ZonedDateTime.now(ZoneOffset.UTC)); + logger.info("daily DateExpr segments will run at fixed rate, initialDelay={}, taskExecutionPeriod={}, ", initialDelay, TimeUnit.DAYS.toSeconds(1)); + schedulerService.getScheduleExecutorService().scheduleAtFixedRate(task, initialDelay, TimeUnit.DAYS.toSeconds(1), TimeUnit.SECONDS); } public void setTaskExecutionPeriod(long taskExecutionPeriod) { diff --git a/services/src/main/java/org/apache/unomi/services/mergers/DefaultPropertyMergeStrategyExecutor.java b/services/src/main/java/org/apache/unomi/services/mergers/DefaultPropertyMergeStrategyExecutor.java index e77f7e398..257875c4f 100644 --- a/services/src/main/java/org/apache/unomi/services/mergers/DefaultPropertyMergeStrategyExecutor.java +++ b/services/src/main/java/org/apache/unomi/services/mergers/DefaultPropertyMergeStrategyExecutor.java @@ -20,6 +20,7 @@ import org.apache.unomi.api.Profile; import org.apache.unomi.api.PropertyMergeStrategyExecutor; import org.apache.unomi.api.PropertyType; +import org.apache.unomi.persistence.spi.PropertyHelper; import java.util.List; @@ -27,9 +28,9 @@ public class DefaultPropertyMergeStrategyExecutor implements PropertyMergeStrate public boolean mergeProperty(String propertyName, PropertyType propertyType, List profilesToMerge, Profile targetProfile) { boolean modified = false; for (Profile profileToMerge : profilesToMerge) { - if (profileToMerge.getProperty(propertyName) != null && - profileToMerge.getProperty(propertyName).toString().length() > 0) { - targetProfile.setProperty(propertyName, profileToMerge.getProperty(propertyName)); + if (profileToMerge.getNestedProperty(propertyName) != null && + profileToMerge.getNestedProperty(propertyName).toString().length() > 0) { + PropertyHelper.setProperty(targetProfile, "properties." + propertyName, profileToMerge.getNestedProperty(propertyName), "alwaysSet"); modified = true; } } diff --git a/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml index 72cf5c0a7..26d296ded 100644 --- a/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ b/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -31,10 +31,16 @@ + + + + + + @@ -81,6 +87,7 @@ + @@ -170,6 +177,14 @@ + + + + + + + + @@ -202,6 +217,7 @@ + @@ -252,6 +268,9 @@ + + + + + + ctx = new HashMap<>(); + Event mockEvent = generateMockEvent(); + ctx.put("event", mockEvent); + ctx.put("session", mockEvent.getSession()); + ctx.put("profile", mockEvent.getProfile()); + File vulnFile = new File("target/vuln-file.txt"); + if (vulnFile.exists()) { + vulnFile.delete(); + } + Object result = null; + try { + result = executeMVEL("java.io.PrintWriter writer = new java.io.PrintWriter(new java.io.BufferedWriter(new java.io.FileWriter(\"" + vulnFile.getCanonicalPath() + "\", true)));\nwriter.println(\"test\");\nwriter.close();", ctx); + } catch (CompileException ce) { + // this is expected since access to these classes should not be allowed + } + System.out.println("result=" + result); + try { + result = executeMVEL("import java.util.*;\nimport java.io.*;\nPrintWriter writer = new PrintWriter(new BufferedWriter(new FileWriter(\"" + vulnFile.getCanonicalPath() + "\", true)));\nwriter.println(\"test\");\nwriter.close();", ctx); + } catch (CompileException ce) { + // this is expected since access to these classes should not be allowed + } + System.out.println("result=" + result); + try { + result = executeMVEL("import java.util.*;\nimport java.io.*;\nnew Scanner(new File(\"" + vulnFile.getCanonicalPath() + "\")).useDelimiter(\"\\\\Z\").next();", ctx); + } catch (CompileException ce) { + // this is expected since access to these classes should not be allowed + } + System.out.println("result=" + result); + assertFalse("Vulnerability successfully executed ! File created at " + vulnFile.getCanonicalPath(), vulnFile.exists()); + } + + private Object executeMVEL(String expression, Object ctx) { + final ClassLoader tccl = Thread.currentThread().getContextClassLoader(); + try { + ParserConfiguration parserConfiguration = new ParserConfiguration(); + ClassLoader secureFilteringClassLoader = new SecureFilteringClassLoader(getClass().getClassLoader()); + Thread.currentThread().setContextClassLoader(secureFilteringClassLoader); + parserConfiguration.setClassLoader(secureFilteringClassLoader); + ParserContext parserContext = new ParserContext(parserConfiguration); + Serializable compiledExpression = MVEL.compileExpression(expression, parserContext); + try { + return MVEL.executeExpression(compiledExpression, ctx, new HashMap()); + } catch (CompileException ce) { + // this is expected + } + return null; + } finally { + Thread.currentThread().setContextClassLoader(tccl); + } + } + + private static Event generateMockEvent() { + Event mockEvent = new Event(); + CustomItem targetItem = new CustomItem(); + targetItem.setItemId(MOCK_ITEM_ID); + targetItem.setScope(DIGITALL_SCOPE); + mockEvent.setTarget(targetItem); + Map pageInfoMap = new HashMap<>(); + pageInfoMap.put("pagePath", PAGE_PATH_VALUE); + pageInfoMap.put("pageURL", PAGE_URL_VALUE); + targetItem.getProperties().put("pageInfo", pageInfoMap); + return mockEvent; + } + +} diff --git a/services/src/test/java/org/apache/unomi/services/impl/scheduler/SchedulerServiceImplTest.java b/services/src/test/java/org/apache/unomi/services/impl/scheduler/SchedulerServiceImplTest.java new file mode 100644 index 000000000..b095b5516 --- /dev/null +++ b/services/src/test/java/org/apache/unomi/services/impl/scheduler/SchedulerServiceImplTest.java @@ -0,0 +1,24 @@ +package org.apache.unomi.services.impl.scheduler; + +import org.junit.Test; + +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; + +import static org.junit.Assert.*; + +public class SchedulerServiceImplTest { + + @Test + public void getTimeDiffInSeconds_whenGiveHourOfDay_shouldReturnDifferenceInSeconds(){ + //Arrange + SchedulerServiceImpl service = new SchedulerServiceImpl(); + int hourToRunInUtc = 11; + ZonedDateTime timeNowInUtc = ZonedDateTime.of(LocalDateTime.parse("2020-01-13T10:00:00"), ZoneOffset.UTC); + //Act + long seconds = service.getTimeDiffInSeconds(hourToRunInUtc, timeNowInUtc); + //Assert + assertEquals(3600, seconds); + } +} \ No newline at end of file diff --git a/tools/pom.xml b/tools/pom.xml index 3d6d23cb5..cbde71e17 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -21,7 +21,7 @@ org.apache.unomi unomi-root - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-tools diff --git a/tools/shell-commands/pom.xml b/tools/shell-commands/pom.xml index 02b3829b1..a596d03e5 100644 --- a/tools/shell-commands/pom.xml +++ b/tools/shell-commands/pom.xml @@ -21,7 +21,7 @@ unomi-tools org.apache.unomi - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT shell-commands diff --git a/tools/shell-dev-commands/pom.xml b/tools/shell-dev-commands/pom.xml index d4f1f5242..efe65c912 100644 --- a/tools/shell-dev-commands/pom.xml +++ b/tools/shell-dev-commands/pom.xml @@ -21,7 +21,7 @@ unomi-tools org.apache.unomi - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT shell-dev-commands diff --git a/wab/pom.xml b/wab/pom.xml index 605419727..75fcfb456 100644 --- a/wab/pom.xml +++ b/wab/pom.xml @@ -22,7 +22,7 @@ org.apache.unomi unomi-root - 1.5.0-SNAPSHOT + 1.5.2-YOTPO-SNAPSHOT unomi-wab diff --git a/wab/src/main/java/org/apache/unomi/web/ClientServlet.java b/wab/src/main/java/org/apache/unomi/web/ClientServlet.java index f385a53bb..0c283ddc6 100644 --- a/wab/src/main/java/org/apache/unomi/web/ClientServlet.java +++ b/wab/src/main/java/org/apache/unomi/web/ClientServlet.java @@ -67,24 +67,29 @@ public void destroy() { @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - String[] pathInfo = req.getPathInfo().substring(1).split("\\."); - if (pathInfo != null && pathInfo.length > 0) { - String operation = pathInfo[0]; - String param = pathInfo[1]; - switch (operation) { - case "myprofile": - if (allowedProfileDownloadFormats.contains(param)) { - donwloadCurrentProfile(req, resp, param); - } else { - resp.setStatus(HttpServletResponse.SC_BAD_REQUEST); - } - break; - default: - resp.setStatus(HttpServletResponse.SC_NOT_FOUND); + try { + String[] pathInfo = req.getPathInfo().substring(1).split("\\."); + if (pathInfo != null && pathInfo.length > 0) { + String operation = pathInfo[0]; + String param = pathInfo[1]; + switch (operation) { + case "myprofile": + if (allowedProfileDownloadFormats.contains(param)) { + donwloadCurrentProfile(req, resp, param); + } else { + resp.setStatus(HttpServletResponse.SC_BAD_REQUEST); + } + break; + default: + resp.setStatus(HttpServletResponse.SC_NOT_FOUND); + } + } else { + resp.setStatus(HttpServletResponse.SC_NOT_FOUND); } - } else { - resp.setStatus(HttpServletResponse.SC_NOT_FOUND); + } catch (Throwable t) { // Here in order to return generic message instead of the whole stack trace in case of not caught exception + logger.error("ClientServlet failed to execute get", t); + resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Internal server error"); } } diff --git a/wab/src/main/java/org/apache/unomi/web/ContextServlet.java b/wab/src/main/java/org/apache/unomi/web/ContextServlet.java index 693c23e68..dfc7338b3 100644 --- a/wab/src/main/java/org/apache/unomi/web/ContextServlet.java +++ b/wab/src/main/java/org/apache/unomi/web/ContextServlet.java @@ -70,233 +70,254 @@ public void init(ServletConfig config) throws ServletException { @Override public void service(HttpServletRequest request, HttpServletResponse response) throws IOException { - final Date timestamp = new Date(); - if (request.getParameter("timestamp") != null) { - timestamp.setTime(Long.parseLong(request.getParameter("timestamp"))); - } + try { + final Date timestamp = new Date(); + if (request.getParameter("timestamp") != null) { + timestamp.setTime(Long.parseLong(request.getParameter("timestamp"))); + } - // set up CORS headers as soon as possible so that errors are not misconstrued on the client for CORS errors - HttpUtils.setupCORSHeaders(request, response); + // set up CORS headers as soon as possible so that errors are not misconstrued on the client for CORS errors + HttpUtils.setupCORSHeaders(request, response); - // Handle OPTIONS request - String httpMethod = request.getMethod(); - if ("options".equals(httpMethod.toLowerCase())) { - response.flushBuffer(); - if (logger.isDebugEnabled()) { - logger.debug("OPTIONS request received. No context will be returned."); + // Handle OPTIONS request + String httpMethod = request.getMethod(); + if ("options".equals(httpMethod.toLowerCase())) { + response.flushBuffer(); + if (logger.isDebugEnabled()) { + logger.debug("OPTIONS request received. No context will be returned."); + } + return; } - return; - } - // Handle persona - Profile profile = null; - Session session = null; - String personaId = request.getParameter("personaId"); - if (personaId != null) { - PersonaWithSessions personaWithSessions = profileService.loadPersonaWithSessions(personaId); - if (personaWithSessions == null) { - logger.error("Couldn't find persona with id=" + personaId); - profile = null; - } else { - profile = personaWithSessions.getPersona(); - session = personaWithSessions.getLastSession(); + // Handle persona + Profile profile = null; + Session session = null; + String personaId = request.getParameter("personaId"); + if (personaId != null) { + PersonaWithSessions personaWithSessions = profileService.loadPersonaWithSessions(personaId); + if (personaWithSessions == null) { + logger.error("Couldn't find persona with id=" + personaId); + profile = null; + } else { + profile = personaWithSessions.getPersona(); + session = personaWithSessions.getLastSession(); + } } - } - // Extract payload - ContextRequest contextRequest = null; - String scope = null; - String sessionId = null; - String stringPayload = HttpUtils.getPayload(request); - if (stringPayload != null) { - ObjectMapper mapper = CustomObjectMapper.getObjectMapper(); - JsonFactory factory = mapper.getFactory(); - try { - contextRequest = mapper.readValue(factory.createParser(stringPayload), ContextRequest.class); - } catch (Exception e) { - ((HttpServletResponse)response).sendError(HttpServletResponse.SC_BAD_REQUEST, "Check logs for more details"); - logger.error("Cannot read payload " + stringPayload, e); - return; - } - if (contextRequest.getSource() != null) { - scope = contextRequest.getSource().getScope(); + // Extract payload + ContextRequest contextRequest = null; + String scope = null; + String sessionId = null; + String profileId = null; + String stringPayload = HttpUtils.getPayload(request); + if (stringPayload != null) { + ObjectMapper mapper = CustomObjectMapper.getObjectMapper(); + JsonFactory factory = mapper.getFactory(); + try { + contextRequest = mapper.readValue(factory.createParser(stringPayload), ContextRequest.class); + } catch (Exception e) { + ((HttpServletResponse) response).sendError(HttpServletResponse.SC_BAD_REQUEST, "Check logs for more details"); + logger.error("Cannot read payload " + stringPayload, e); + return; + } + if (contextRequest.getSource() != null) { + scope = contextRequest.getSource().getScope(); + } + sessionId = contextRequest.getSessionId(); + profileId = contextRequest.getProfileId(); } - sessionId = contextRequest.getSessionId(); - } - if (sessionId == null) { - sessionId = request.getParameter("sessionId"); - } + if (sessionId == null) { + sessionId = request.getParameter("sessionId"); + } - // Get profile id from the cookie - String cookieProfileId = ServletCommon.getProfileIdCookieValue(request, profileIdCookieName); + if (profileId == null) { + // Get profile id from the cookie + profileId = ServletCommon.getProfileIdCookieValue(request, profileIdCookieName); + } - if (cookieProfileId == null && sessionId == null && personaId == null) { - ((HttpServletResponse)response).sendError(HttpServletResponse.SC_BAD_REQUEST, "Check logs for more details"); - logger.error("Couldn't find cookieProfileId, sessionId or personaId in incoming request! Stopped processing request. See debug level for more information"); - if (logger.isDebugEnabled()) { - logger.debug("Request dump: {}", HttpUtils.dumpRequestInfo(request)); + if (profileId == null && sessionId == null && personaId == null) { + ((HttpServletResponse) response).sendError(HttpServletResponse.SC_BAD_REQUEST, "Check logs for more details"); + logger.error("Couldn't find profileId, sessionId or personaId in incoming request! Stopped processing request. See debug level for more information"); + if (logger.isDebugEnabled()) { + logger.debug("Request dump: {}", HttpUtils.dumpRequestInfo(request)); + } + return; } - return; - } - int changes = EventService.NO_CHANGE; - if (profile == null) { - // Not a persona, resolve profile now - boolean profileCreated = false; - - boolean invalidateProfile = request.getParameter("invalidateProfile") != null ? - new Boolean(request.getParameter("invalidateProfile")) : false; - if (cookieProfileId == null || invalidateProfile) { - // no profileId cookie was found or the profile has to be invalidated, we generate a new one and create the profile in the profile service - profile = createNewProfile(null, response, timestamp); - profileCreated = true; - } else { - profile = profileService.load(cookieProfileId); - if (profile == null) { - // this can happen if we have an old cookie but have reset the server, - // or if we merged the profiles and somehow this cookie didn't get updated. + int changes = EventService.NO_CHANGE; + Map eventsAttributes = new HashMap(); + + if (profile == null) { + // Not a persona, resolve profile now + boolean profileCreated = false; + + boolean invalidateProfile = request.getParameter("invalidateProfile") != null ? + new Boolean(request.getParameter("invalidateProfile")) : false; + if (profileId == null || invalidateProfile) { + // no profileId cookie was found or the profile has to be invalidated, we generate a new one and create the profile in the profile service profile = createNewProfile(null, response, timestamp); profileCreated = true; } else { - Changes changesObject = checkMergedProfile(response, profile, session); - changes |= changesObject.getChangeType(); - profile = changesObject.getProfile(); + profile = profileService.load(profileId); + if (profile == null) { + // this can happen if we have an old cookie but have reset the server, + // or if we merged the profiles and somehow this cookie didn't get updated. + profile = createNewProfile(profileId, response, timestamp); + profileCreated = true; + } else { + Changes changesObject = checkMergedProfile(response, profile, session); + changes |= changesObject.getChangeType(); + if (!profile.getItemId().equals(changesObject.getProfile().getItemId())) { + eventsAttributes.put("alreadyMerged", true); + } + profile = changesObject.getProfile(); + } } - } - Profile sessionProfile; - boolean invalidateSession = request.getParameter("invalidateSession") != null ? - new Boolean(request.getParameter("invalidateSession")) : false; - if (StringUtils.isNotBlank(sessionId) && !invalidateSession) { - session = profileService.loadSession(sessionId, timestamp); - if (session != null) { - sessionProfile = session.getProfile(); - - boolean anonymousSessionProfile = sessionProfile.isAnonymousProfile(); - if (!profile.isAnonymousProfile() && !anonymousSessionProfile && !profile.getItemId().equals(sessionProfile.getItemId())) { - // Session user has been switched, profile id in cookie is not up to date - // We must reload the profile with the session ID as some properties could be missing from the session profile - // #personalIdentifier - profile = profileService.load(sessionProfile.getItemId()); - HttpUtils.sendProfileCookie(profile, response, profileIdCookieName, profileIdCookieDomain, profileIdCookieMaxAgeInSeconds); - } + Profile sessionProfile; + boolean invalidateSession = request.getParameter("invalidateSession") != null ? + new Boolean(request.getParameter("invalidateSession")) : false; + if (StringUtils.isNotBlank(sessionId) && !invalidateSession) { + session = profileService.loadSession(sessionId, timestamp); + if (session != null) { + sessionProfile = session.getProfile(); + + boolean anonymousSessionProfile = sessionProfile.isAnonymousProfile(); + if (!profile.isAnonymousProfile() && !anonymousSessionProfile && !profile.getItemId().equals(sessionProfile.getItemId())) { + // Session user has been switched, profile id in cookie is not up to date + // We must reload the profile with the session ID as some properties could be missing from the session profile + // #personalIdentifier + profile = profileService.load(sessionProfile.getItemId()); + HttpUtils.sendProfileCookie(profile, response, profileIdCookieName, profileIdCookieDomain, profileIdCookieMaxAgeInSeconds); + } - // Handle anonymous situation - Boolean requireAnonymousBrowsing = privacyService.isRequireAnonymousBrowsing(profile); - if (requireAnonymousBrowsing && anonymousSessionProfile) { - // User wants to browse anonymously, anonymous profile is already set. - } else if (requireAnonymousBrowsing && !anonymousSessionProfile) { - // User wants to browse anonymously, update the sessionProfile to anonymous profile - sessionProfile = privacyService.getAnonymousProfile(profile); - session.setProfile(sessionProfile); - changes |= EventService.SESSION_UPDATED; - } else if (!requireAnonymousBrowsing && anonymousSessionProfile) { - // User does not want to browse anonymously anymore, update the sessionProfile to real profile - sessionProfile = profile; - session.setProfile(sessionProfile); - changes |= EventService.SESSION_UPDATED; - } else if (!requireAnonymousBrowsing && !anonymousSessionProfile) { - // User does not want to browse anonymously, use the real profile. Check that session contains the current profile. - sessionProfile = profile; - if (!session.getProfileId().equals(sessionProfile.getItemId())) { + // Handle anonymous situation + Boolean requireAnonymousBrowsing = privacyService.isRequireAnonymousBrowsing(profile); + if (requireAnonymousBrowsing && anonymousSessionProfile) { + // User wants to browse anonymously, anonymous profile is already set. + } else if (requireAnonymousBrowsing && !anonymousSessionProfile) { + // User wants to browse anonymously, update the sessionProfile to anonymous profile + sessionProfile = privacyService.getAnonymousProfile(profile); + session.setProfile(sessionProfile); + changes |= EventService.SESSION_UPDATED; + } else if (!requireAnonymousBrowsing && anonymousSessionProfile) { + // User does not want to browse anonymously anymore, update the sessionProfile to real profile + sessionProfile = profile; + session.setProfile(sessionProfile); changes |= EventService.SESSION_UPDATED; + } else if (!requireAnonymousBrowsing && !anonymousSessionProfile) { + // User does not want to browse anonymously, use the real profile. Check that session contains the current profile. + sessionProfile = profile; + if (!session.getProfileId().equals(sessionProfile.getItemId())) { + changes |= EventService.SESSION_UPDATED; + } + session.setProfile(sessionProfile); } - session.setProfile(sessionProfile); } } - } - if (session == null || invalidateSession) { - sessionProfile = privacyService.isRequireAnonymousBrowsing(profile) ? privacyService.getAnonymousProfile(profile) : profile; - - if (StringUtils.isNotBlank(sessionId)) { - // Only save session and send event if a session id was provided, otherwise keep transient session - session = new Session(sessionId, sessionProfile, timestamp, scope); - changes |= EventService.SESSION_UPDATED; - Event event = new Event("sessionCreated", session, profile, scope, null, session, timestamp); - if (sessionProfile.isAnonymousProfile()) { - // Do not keep track of profile in event - event.setProfileId(null); - } - event.getAttributes().put(Event.HTTP_REQUEST_ATTRIBUTE, request); - event.getAttributes().put(Event.HTTP_RESPONSE_ATTRIBUTE, response); - if (logger.isDebugEnabled()) { - logger.debug("Received event {} for profile={} session={} target={} timestamp={}", - event.getEventType(), profile.getItemId(), session.getItemId(), event.getTarget(), timestamp); + if (session == null || invalidateSession) { + sessionProfile = privacyService.isRequireAnonymousBrowsing(profile) ? privacyService.getAnonymousProfile(profile) : profile; + + if (StringUtils.isNotBlank(sessionId)) { + // Only save session and send event if a session id was provided, otherwise keep transient session + session = new Session(sessionId, sessionProfile, timestamp, scope); + changes |= EventService.SESSION_UPDATED; + Event event = new Event("sessionCreated", session, profile, scope, null, session, timestamp); + if (sessionProfile.isAnonymousProfile()) { + // Do not keep track of profile in event + event.setProfileId(null); + } + event.getAttributes().put(Event.HTTP_REQUEST_ATTRIBUTE, request); + event.getAttributes().put(Event.HTTP_RESPONSE_ATTRIBUTE, response); + if (logger.isDebugEnabled()) { + logger.debug("Received event {} for profile={} session={} target={} timestamp={}", + event.getEventType(), profile.getItemId(), session.getItemId(), event.getTarget(), timestamp); + } + changes |= eventService.send(event); } - changes |= eventService.send(event); } - } - if (profileCreated) { - changes |= EventService.PROFILE_UPDATED; + if (profileCreated) { + changes |= EventService.PROFILE_UPDATED; - Event profileUpdated = new Event("profileUpdated", session, profile, scope, null, profile, timestamp); - profileUpdated.setPersistent(false); - profileUpdated.getAttributes().put(Event.HTTP_REQUEST_ATTRIBUTE, request); - profileUpdated.getAttributes().put(Event.HTTP_RESPONSE_ATTRIBUTE, response); + Event profileUpdated = new Event("profileUpdated", session, profile, scope, null, profile, timestamp); + profileUpdated.setPersistent(false); + profileUpdated.getAttributes().put(Event.HTTP_REQUEST_ATTRIBUTE, request); + profileUpdated.getAttributes().put(Event.HTTP_RESPONSE_ATTRIBUTE, response); - if (logger.isDebugEnabled()) { - logger.debug("Received event {} for profile={} {} target={} timestamp={}", profileUpdated.getEventType(), profile.getItemId(), - " session=" + (session != null ? session.getItemId() : null), profileUpdated.getTarget(), timestamp); + if (logger.isDebugEnabled()) { + logger.debug("Received event {} for profile={} {} target={} timestamp={}", profileUpdated.getEventType(), profile.getItemId(), + " session=" + (session != null ? session.getItemId() : null), profileUpdated.getTarget(), timestamp); + } + changes |= eventService.send(profileUpdated); } - changes |= eventService.send(profileUpdated); } - } - ContextResponse contextResponse = new ContextResponse(); - contextResponse.setProfileId(profile.getItemId()); - if (session != null) { - contextResponse.setSessionId(session.getItemId()); - } else if (sessionId != null) { - contextResponse.setSessionId(sessionId); - } + ContextResponse contextResponse = new ContextResponse(); + contextResponse.setProfileId(profile.getItemId()); + if (session != null) { + contextResponse.setSessionId(session.getItemId()); + } else if (sessionId != null) { + contextResponse.setSessionId(sessionId); + } - if (contextRequest != null) { - Changes changesObject = handleRequest(contextRequest, session, profile, contextResponse, request, response, timestamp); - changes |= changesObject.getChangeType(); - profile = changesObject.getProfile(); - } + if (contextRequest != null) { + Changes changesObject = handleRequest(contextRequest, session, profile, contextResponse, request, response, timestamp, eventsAttributes); + changes |= changesObject.getChangeType(); + profile = changesObject.getProfile(); + } - if ((changes & EventService.PROFILE_UPDATED) == EventService.PROFILE_UPDATED) { - profileService.save(profile); - contextResponse.setProfileId(profile.getItemId()); - } - if ((changes & EventService.SESSION_UPDATED) == EventService.SESSION_UPDATED && session != null) { - profileService.saveSession(session); - contextResponse.setSessionId(session.getItemId()); - } + if ((changes & EventService.PROFILE_UPDATED) == EventService.PROFILE_UPDATED) { + profileService.save(profile); + contextResponse.setProfileId(profile.getItemId()); + } + if ((changes & EventService.SESSION_UPDATED) == EventService.SESSION_UPDATED && session != null) { + profileService.saveSession(session); + contextResponse.setSessionId(session.getItemId()); + } - if ((changes & EventService.ERROR) == EventService.ERROR) { - response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - } + if ((changes & EventService.ERROR) == EventService.ERROR) { + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } - String extension = request.getRequestURI().substring(request.getRequestURI().lastIndexOf(".") + 1); - boolean noScript = "json".equals(extension); - String contextAsJSONString = CustomObjectMapper.getObjectMapper().writeValueAsString(contextResponse); - Writer responseWriter; - response.setCharacterEncoding("UTF-8"); - if (noScript) { - responseWriter = response.getWriter(); - response.setContentType("application/json"); - IOUtils.write(contextAsJSONString, responseWriter); - } else { - responseWriter = response.getWriter(); - responseWriter.append("window.digitalData = window.digitalData || {};\n") - .append("var cxs = ") - .append(contextAsJSONString) - .append(";\n"); - } + String extension = request.getRequestURI().substring(request.getRequestURI().lastIndexOf(".") + 1); + boolean noScript = "json".equals(extension); + String contextAsJSONString = CustomObjectMapper.getObjectMapper().writeValueAsString(contextResponse); + Writer responseWriter; + response.setCharacterEncoding("UTF-8"); + if (noScript) { + responseWriter = response.getWriter(); + response.setContentType("application/json"); + IOUtils.write(contextAsJSONString, responseWriter); + } else { + responseWriter = response.getWriter(); + responseWriter.append("window.digitalData = window.digitalData || {};\n") + .append("var cxs = ") + .append(contextAsJSONString) + .append(";\n"); + } - responseWriter.flush(); + responseWriter.flush(); + } catch (Throwable t) { // Here in order to return generic message instead of the whole stack trace in case of not caught exception + logger.error("ContextServlet failed to execute request", t); + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Internal server error"); + } } private Changes checkMergedProfile(ServletResponse response, Profile profile, Session session) { int changes = EventService.NO_CHANGE; - if (profile.getMergedWith() != null && !privacyService.isRequireAnonymousBrowsing(profile) && !profile.isAnonymousProfile()) { + + Set mergedWithProfileIds = new HashSet<>(); + + while (profile.getMergedWith() != null && !privacyService.isRequireAnonymousBrowsing(profile) && !profile.isAnonymousProfile()) { Profile currentProfile = profile; String masterProfileId = profile.getMergedWith(); + if (!mergedWithProfileIds.add(masterProfileId)) { // Return false if the value is already exist + throw new RuntimeException("Profiles are merged in a loop" + mergedWithProfileIds.toString()); + } + Profile masterProfile = profileService.load(masterProfileId); if (masterProfile != null) { if (logger.isDebugEnabled()) { @@ -320,9 +341,9 @@ private Changes checkMergedProfile(ServletResponse response, Profile profile, Se } private Changes handleRequest(ContextRequest contextRequest, Session session, Profile profile, ContextResponse data, - ServletRequest request, ServletResponse response, Date timestamp) { + ServletRequest request, ServletResponse response, Date timestamp, Map eventsAttributes) { Changes changes = ServletCommon.handleEvents(contextRequest.getEvents(), session, profile, request, response, timestamp, - privacyService, eventService); + privacyService, eventService, eventsAttributes); data.setProcessedEvents(changes.getProcessedItems()); profile = changes.getProfile(); diff --git a/wab/src/main/java/org/apache/unomi/web/EventsCollectorServlet.java b/wab/src/main/java/org/apache/unomi/web/EventsCollectorServlet.java index 2c3465962..aaf4bffb4 100644 --- a/wab/src/main/java/org/apache/unomi/web/EventsCollectorServlet.java +++ b/wab/src/main/java/org/apache/unomi/web/EventsCollectorServlet.java @@ -36,6 +36,7 @@ import java.io.IOException; import java.io.PrintWriter; import java.util.Date; +import java.util.HashMap; import java.util.UUID; public class EventsCollectorServlet extends HttpServlet { @@ -62,12 +63,22 @@ public void destroy() { @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - doEvent(req, resp); + try { + doEvent(req, resp); + } catch (Throwable t) { // Here in order to return generic message instead of the whole stack trace in case of not caught exception + logger.error("EventsCollectorServlet failed to execute get", t); + resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Internal server error"); + } } @Override protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - doEvent(req, resp); + try { + doEvent(req, resp); + } catch (Throwable t) { // Here in order to return generic message instead of the whole stack trace in case of not caught exception + logger.error("EventsCollectorServlet failed to execute post", t); + resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Internal server error"); + } } @Override @@ -175,7 +186,7 @@ private void doEvent(HttpServletRequest request, HttpServletResponse response) t } Changes changesObject = ServletCommon.handleEvents(eventsCollectorRequest.getEvents(), session, profile, request, response, - timestamp, privacyService, eventService); + timestamp, privacyService, eventService, new HashMap()); int changes = changesObject.getChangeType(); profile = changesObject.getProfile(); diff --git a/wab/src/main/java/org/apache/unomi/web/ServletCommon.java b/wab/src/main/java/org/apache/unomi/web/ServletCommon.java index 597295f66..ac89aef9d 100644 --- a/wab/src/main/java/org/apache/unomi/web/ServletCommon.java +++ b/wab/src/main/java/org/apache/unomi/web/ServletCommon.java @@ -32,6 +32,7 @@ import javax.servlet.http.HttpServletRequest; import java.util.Date; import java.util.List; +import java.util.Map; /** * @author dgaillard @@ -56,7 +57,7 @@ public static String getProfileIdCookieValue(HttpServletRequest httpServletReque public static Changes handleEvents(List events, Session session, Profile profile, ServletRequest request, ServletResponse response, Date timestamp, - PrivacyService privacyService, EventService eventService) { + PrivacyService privacyService, EventService eventService, Map eventsAttributes) { List filteredEventTypes = privacyService.getFilteredEventTypes(profile); String thirdPartyId = eventService.authenticateThirdPartyServer(((HttpServletRequest) request).getHeader("X-Unomi-Peer"), @@ -77,6 +78,7 @@ public static Changes handleEvents(List events, Session session, Profile } if (thirdPartyId != null && event.getItemId() != null) { eventToSend = new Event(event.getItemId(), event.getEventType(), session, profile, event.getScope(), event.getSource(), event.getTarget(), event.getProperties(), timestamp, event.isPersistent()); + eventToSend.setAttributes(eventsAttributes); } if (filteredEventTypes != null && filteredEventTypes.contains(event.getEventType())) { logger.debug("Profile is filtering event type {}", event.getEventType()); diff --git a/wab/src/main/resources/org.apache.unomi.web.cfg b/wab/src/main/resources/org.apache.unomi.web.cfg index b78678251..5d6b0f8d4 100644 --- a/wab/src/main/resources/org.apache.unomi.web.cfg +++ b/wab/src/main/resources/org.apache.unomi.web.cfg @@ -23,4 +23,4 @@ contextserver.profileIdCookieName=${org.apache.unomi.profile.cookie.name:-contex # This setting controls the maximum age of the profile cookie. By default it is set to a year. contextserver.profileIdCookieMaxAgeInSeconds=${org.apache.unomi.profile.cookie.maxAgeInSeconds:-31536000} #Allowed profile download formats, actually only csv (horizontal and vertical), json, text and yaml are allowed. -allowed.profile.download.formats=${org.apache.unomi.profile.download.formats:-csv,yaml,json,text} \ No newline at end of file +allowed.profile.download.formats=${org.apache.unomi.profile.download.formats:-csv,yaml,json,text}