From 86913ccb547f20ab38b1418d9be32907123e3610 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Wed, 5 Oct 2016 00:36:59 +0300 Subject: [PATCH 1/9] IGNITE-4011 Automatic hash code computing for BinaryBuilder constructed objects --- .../ignite/binary/BinaryKeyHashingMode.java | 125 +++++++++++ .../binary/BinaryObjectHashCodeResolver.java | 40 ++++ .../ignite/cache/CacheKeyConfiguration.java | 81 +++++++ .../ignite/internal/binary/BinaryContext.java | 95 ++++++++- .../internal/binary/BinaryObjectExImpl.java | 19 ++ .../internal/binary/BinaryObjectImpl.java | 6 +- .../binary/BinaryObjectOffheapImpl.java | 5 + .../binary/FieldsListHashCodeResolver.java | 76 +++++++ .../builder/BinaryObjectBuilderImpl.java | 41 ++++ ...ridCacheBinaryObjectsAbstractSelfTest.java | 197 +++++++++++++++++- 10 files changed, 664 insertions(+), 21 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/binary/BinaryKeyHashingMode.java create mode 100644 modules/core/src/main/java/org/apache/ignite/binary/BinaryObjectHashCodeResolver.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/binary/FieldsListHashCodeResolver.java diff --git a/modules/core/src/main/java/org/apache/ignite/binary/BinaryKeyHashingMode.java b/modules/core/src/main/java/org/apache/ignite/binary/BinaryKeyHashingMode.java new file mode 100644 index 0000000000000..b43f2eee85557 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/binary/BinaryKeyHashingMode.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.binary; + +import java.util.Set; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.cache.CacheKeyConfiguration; +import org.apache.ignite.internal.binary.FieldsListHashCodeResolver; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.U; + +/** + * Mode of generating hash codes for keys created with {@link BinaryObjectBuilder}. + */ +public enum BinaryKeyHashingMode { + /** + * Default (also legacy pre 1.8) mode. Use this mode if you use no SQL DML commands - INSERT, UPDATE, DELETE, MERGE, + * in other words, if you put data to cache NOT via SQL. + * Effect from choosing this mode is identical to omitting mode settings from key configuration at all. + */ + DEFAULT, + + /** + * Generate hash code based upon serialized representation of binary object fields - namely, byte array constructed + * by {@link BinaryObjectBuilder}. Use this mode if you are NOT planning to retrieve data from cache via + * ordinary cache methods like {@link IgniteCache#get(Object)}, {@link IgniteCache#getAll(Set)}, etc. - + * it's an convenient way to manipulate and retrieve binary data in cache only via full-scale SQL features + * with as little additional configuration overhead as choosing this mode. + */ + BYTES_HASH, + + /** + * Generate hash code based upon on list of fields declared in {@link BinaryObjectBuilder} + * (not in {@link BinaryObject} as hash code has to be computed before {@link BinaryObject} is fully built) - + * this mode requires that you set {@link CacheKeyConfiguration#binHashCodeFields} for it to work. + */ + FIELDS_HASH { + /** {@inheritDoc} */ + @Override public BinaryObjectHashCodeResolver createResolver(CacheKeyConfiguration keyConfiguration) { + return new FieldsListHashCodeResolver(keyConfiguration.getBinaryHashCodeFields()); + } + }, + + /** + * Generate hash code arbitrarily based on {@link BinaryObjectBuilder} using specified class implementing + * {@link BinaryObjectHashCodeResolver}- this mode requires that you set + * {@link CacheKeyConfiguration#binHashCodeRslvrClsName} for it to work. + */ + CUSTOM { + /** {@inheritDoc} */ + @Override public BinaryObjectHashCodeResolver createResolver(CacheKeyConfiguration keyConfiguration) { + String clsName = keyConfiguration.getTypeName(); + + String rslvrClsName = keyConfiguration.getBinaryHashCodeResolverClassName(); + + if (F.isEmpty(rslvrClsName)) + throw new IllegalArgumentException("No custom hash code resolver class name specified, please review " + + "your configuration [clsName=" + clsName + ']'); + + Class rslvrCls = U.classForName(rslvrClsName, null); + + if (rslvrCls == null) + throw new IllegalArgumentException("Hash code resolver class not found [clsName=" + clsName + + ", rslvrClsName=" + rslvrClsName + ']'); + + if (!BinaryObjectHashCodeResolver.class.isAssignableFrom(rslvrCls)) + throw new IllegalArgumentException("Hash code resolver class does not implement " + + "BinaryObjectHashCodeResolver interface [clsName=" + clsName + ", rslvrClsName=" + + rslvrClsName + ']'); + + try { + return (BinaryObjectHashCodeResolver) U.newInstance(rslvrCls); + } + catch (IgniteCheckedException e) { + throw new BinaryObjectException("Failed to instantiate hash code resolver [clsName=" + + clsName + ", rslvrClsName=" + rslvrClsName + ']', e); + } + } + }; + + /** + * @return {@code true} if this mode relies on params set in {@link CacheKeyConfiguration}. + */ + public final boolean isConfigurable() { + switch (this) { + case FIELDS_HASH: + case CUSTOM: + return true; + case DEFAULT: + case BYTES_HASH: + return false; + default: + throw new IllegalArgumentException("Unexpected key hashing mode [mode=" + this.name() + ']'); + } + } + + /** + * Instantiate {@link BinaryObjectHashCodeResolver} based on given {@link CacheKeyConfiguration}. + * + * @param keyConfiguration Configuration to create resolver based upon. + * @return Hash code resolver. + */ + public BinaryObjectHashCodeResolver createResolver(CacheKeyConfiguration keyConfiguration) { + assert !isConfigurable() : "Configurable hashing mode does not override createResolver"; + + throw new UnsupportedOperationException("This binary keys hashing mode is not configurable, " + + "only modes " + FIELDS_HASH.name() + " and " + CUSTOM.name() + " are."); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/binary/BinaryObjectHashCodeResolver.java b/modules/core/src/main/java/org/apache/ignite/binary/BinaryObjectHashCodeResolver.java new file mode 100644 index 0000000000000..bb22bee6c366d --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/binary/BinaryObjectHashCodeResolver.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.binary; + +import org.apache.ignite.internal.binary.BinaryObjectExImpl; + +/** + * Method to compute hash codes for new binary objects. + */ +public interface BinaryObjectHashCodeResolver { + /** + * @param builder Binary object builder. + * @return Hash code value. + */ + public int hash(BinaryObjectBuilder builder); + + /** + * Compare binary objects for equality in consistence with how hash code is computed. + * + * @param o1 First object. + * @param o2 Second object. + * @return + */ + public boolean equals(BinaryObjectExImpl o1, BinaryObjectExImpl o2); +} diff --git a/modules/core/src/main/java/org/apache/ignite/cache/CacheKeyConfiguration.java b/modules/core/src/main/java/org/apache/ignite/cache/CacheKeyConfiguration.java index f1178475b7536..63dcba2eb041c 100644 --- a/modules/core/src/main/java/org/apache/ignite/cache/CacheKeyConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/cache/CacheKeyConfiguration.java @@ -19,7 +19,11 @@ import java.io.Serializable; import java.lang.reflect.Field; +import java.util.List; +import org.apache.ignite.binary.BinaryKeyHashingMode; +import org.apache.ignite.binary.BinaryObjectHashCodeResolver; import org.apache.ignite.cache.affinity.AffinityKeyMapped; +import org.apache.ignite.internal.binary.FieldsListHashCodeResolver; import org.apache.ignite.internal.util.typedef.internal.S; /** @@ -35,6 +39,15 @@ public class CacheKeyConfiguration implements Serializable { /** Affinity key field name. */ private String affKeyFieldName; + /** Key hashing mode. */ + private BinaryKeyHashingMode binHashingMode; + + /** Fields to build binary objects' hash code upon. */ + private List binHashCodeFields; + + /** Class name for hash code resolver to automatically compute hash codes for newly built binary objects. */ + private String binHashCodeRslvrClsName; + /** * Creates an empty cache key configuration that should be populated via setters. */ @@ -104,6 +117,74 @@ public void setAffinityKeyFieldName(String affKeyFieldName) { this.affKeyFieldName = affKeyFieldName; } + /** + * Gets hashing mode for binary keys. + * + * @return Binary hashing mode. + * @see BinaryKeyHashingMode + */ + public BinaryKeyHashingMode getBinaryHashingMode() { + return binHashingMode; + } + + /** + * Sets hashing more for binary keys. + * + * @param binHashingMode Binary hashing mode. + * @see BinaryKeyHashingMode + */ + public void setBinaryHashingMode(BinaryKeyHashingMode binHashingMode) { + this.binHashingMode = binHashingMode; + } + + /** + * Gets list of fields for fields list hash code resolver. + * + * @return List of fields for fields list hash code resolver. + * @see #binHashingMode + * @see BinaryKeyHashingMode#FIELDS_HASH + * @see FieldsListHashCodeResolver + */ + public List getBinaryHashCodeFields() { + return binHashCodeFields; + } + + /** + * Sets list of fields for fields list hash code resolver. + * + * @param binHashCodeFields List of fields for fields list hash code resolver. + * @see #binHashingMode + * @see BinaryKeyHashingMode#FIELDS_HASH + * @see FieldsListHashCodeResolver + */ + public void setBinaryHashCodeFields(List binHashCodeFields) { + this.binHashCodeFields = binHashCodeFields; + } + + /** + * Gets binary objects hash code resolver class name. + * + * @return binary objects hash code resolver class name. + * @see #binHashingMode + * @see BinaryKeyHashingMode#CUSTOM + * @see BinaryObjectHashCodeResolver + */ + public String getBinaryHashCodeResolverClassName() { + return binHashCodeRslvrClsName; + } + + /** + * Sets binary objects hash code resolver class name. + * + * @param binHashCodeRslvrClsName resolver class name. + * @see #binHashingMode + * @see BinaryKeyHashingMode#CUSTOM + * @see BinaryObjectHashCodeResolver + */ + public void setBinHashCodeResolverClassName(String binHashCodeRslvrClsName) { + this.binHashCodeRslvrClsName = binHashCodeRslvrClsName; + } + /** {@inheritDoc} */ @Override public String toString() { return S.toString(CacheKeyConfiguration.class, this); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java index c468a4df1eab9..a92c2b2947a66 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java @@ -24,8 +24,11 @@ import org.apache.ignite.binary.BinaryBasicNameMapper; import org.apache.ignite.binary.BinaryIdMapper; import org.apache.ignite.binary.BinaryInvalidTypeException; +import org.apache.ignite.binary.BinaryKeyHashingMode; import org.apache.ignite.binary.BinaryNameMapper; +import org.apache.ignite.binary.BinaryObjectBuilder; import org.apache.ignite.binary.BinaryObjectException; +import org.apache.ignite.binary.BinaryObjectHashCodeResolver; import org.apache.ignite.binary.BinaryReflectiveSerializer; import org.apache.ignite.binary.BinarySerializer; import org.apache.ignite.binary.BinaryType; @@ -36,6 +39,8 @@ import org.apache.ignite.configuration.BinaryConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.igfs.IgfsPath; +import org.apache.ignite.internal.binary.builder.BinaryObjectBuilderImpl; +import org.apache.ignite.internal.processors.cache.CacheObjectImpl; import org.apache.ignite.internal.processors.cache.binary.BinaryMetadataKey; import org.apache.ignite.internal.processors.closure.GridClosureProcessor; import org.apache.ignite.internal.processors.datastructures.CollocatedQueueItemKey; @@ -221,6 +226,12 @@ public class BinaryContext { /** Maps className to mapper */ private final ConcurrentMap cls2Mappers = new ConcurrentHashMap8<>(0); + /** Affinity key field names. */ + private final ConcurrentMap hashingModes = new ConcurrentHashMap8<>(0); + + /** Affinity key field names. */ + private final ConcurrentMap hashCodeRslvrs = new ConcurrentHashMap8<>(0); + /** */ private BinaryMetadataHandler metaHnd; @@ -418,9 +429,18 @@ private void configure( Map affFields = new HashMap<>(); + Map cfgHashingModes = new HashMap<>(); + + Map keyCfgs = new HashMap<>(); + if (!F.isEmpty(igniteCfg.getCacheKeyConfiguration())) { - for (CacheKeyConfiguration keyCfg : igniteCfg.getCacheKeyConfiguration()) + for (CacheKeyConfiguration keyCfg : igniteCfg.getCacheKeyConfiguration()) { affFields.put(keyCfg.getTypeName(), keyCfg.getAffinityKeyFieldName()); + + keyCfgs.put(keyCfg.getTypeName(), keyCfg); + + cfgHashingModes.put(keyCfg.getTypeName(), keyCfg.getBinaryHashingMode()); + } } if (typeCfgs != null) { @@ -446,6 +466,8 @@ private void configure( // Resolve serializer. BinarySerializer serializer = globalSerializer; + BinaryKeyHashingMode hashingMode = cfgHashingModes.get(clsName); + if (typeCfg.getSerializer() != null) serializer = typeCfg.getSerializer(); @@ -453,17 +475,18 @@ private void configure( String pkgName = clsName.substring(0, clsName.length() - 2); for (String clsName0 : classesInPackage(pkgName)) - descs.add(clsName0, mapper, serializer, affFields.get(clsName0), + descs.add(clsName0, mapper, serializer, hashingMode, affFields.get(clsName0), typeCfg.isEnum(), true); } else - descs.add(clsName, mapper, serializer, affFields.get(clsName), + descs.add(clsName, mapper, serializer, hashingMode, affFields.get(clsName), typeCfg.isEnum(), false); } } for (TypeDescriptor desc : descs.descriptors()) - registerUserType(desc.clsName, desc.mapper, desc.serializer, desc.affKeyFieldName, desc.isEnum); + registerUserType(desc.clsName, desc.mapper, desc.serializer, desc.keyHashingMode, desc.affKeyFieldName, + desc.isEnum, keyCfgs.get(desc.clsName)); BinaryInternalMapper globalMapper = resolveMapper(globalNameMapper, globalIdMapper); @@ -1087,16 +1110,20 @@ public BinaryClassDescriptor registerPredefinedType(Class cls, int id, String * @param clsName Class name. * @param mapper ID mapper. * @param serializer Serializer. + * @param keyHashingMode Key hashing mode. * @param affKeyFieldName Affinity key field name. * @param isEnum If enum. + * @param keyConfiguration Key configuration, if applicable. * @throws BinaryObjectException In case of error. */ @SuppressWarnings("ErrorNotRethrown") public void registerUserType(String clsName, BinaryInternalMapper mapper, @Nullable BinarySerializer serializer, + @Nullable BinaryKeyHashingMode keyHashingMode, @Nullable String affKeyFieldName, - boolean isEnum) + boolean isEnum, + @Nullable CacheKeyConfiguration keyConfiguration) throws BinaryObjectException { assert mapper != null; @@ -1115,14 +1142,28 @@ public void registerUserType(String clsName, //Workaround for IGNITE-1358 if (predefinedTypes.get(id) != null) - throw new BinaryObjectException("Duplicate type ID [clsName=" + clsName + ", id=" + id + ']'); + dup(clsName, id); if (typeId2Mapper.put(id, mapper) != null) - throw new BinaryObjectException("Duplicate type ID [clsName=" + clsName + ", id=" + id + ']'); + dup(clsName, id); + + if (keyHashingMode != null) { + if (hashingModes.put(id, keyHashingMode) != null) + dup(clsName, id); + + if (keyHashingMode.isConfigurable()) { + assert keyConfiguration != null; + + BinaryObjectHashCodeResolver rslvr = keyHashingMode.createResolver(keyConfiguration); + + if (hashCodeRslvrs.put(id, rslvr) != null) + dup(clsName, id); + } + } if (affKeyFieldName != null) { if (affKeyFieldNames.put(id, affKeyFieldName) != null) - throw new BinaryObjectException("Duplicate type ID [clsName=" + clsName + ", id=" + id + ']'); + dup(clsName, id); } cls2Mappers.put(clsName, mapper); @@ -1165,6 +1206,16 @@ public void registerUserType(String clsName, metaHnd.addMeta(id, new BinaryMetadata(id, typeName, fieldsMeta, affKeyFieldName, schemas, isEnum).wrap(this)); } + /** + * Throw exception on class duplication. + * + * @param clsName Class name. + * @param id Type id. + */ + private static void dup(String clsName, int id) { + throw new BinaryObjectException("Duplicate type ID [clsName=" + clsName + ", id=" + id + ']'); + } + /** * Check whether reflective serializer can be used for class. * @@ -1209,6 +1260,22 @@ public String affinityKeyFieldName(int typeId) { return affKeyFieldNames.get(typeId); } + /** + * @param typeId Type ID. + * @return Key hashing mode. + */ + public BinaryKeyHashingMode keyHashingMode(int typeId) { + return hashingModes.get(typeId); + } + + /** + * @param typeId Type ID. + * @return Key hashing mode. + */ + public BinaryObjectHashCodeResolver hashCodeResolver(int typeId) { + return hashCodeRslvrs.get(typeId); + } + /** * @param typeId Type ID. * @param meta Meta data. @@ -1318,6 +1385,7 @@ private static class TypeDescriptors { * @param clsName Class name. * @param mapper Mapper. * @param serializer Serializer. + * @param hashingMode Key hashing mode. * @param affKeyFieldName Affinity key field name. * @param isEnum Enum flag. * @param canOverride Whether this descriptor can be override. @@ -1326,6 +1394,7 @@ private static class TypeDescriptors { private void add(String clsName, BinaryInternalMapper mapper, BinarySerializer serializer, + BinaryKeyHashingMode hashingMode, String affKeyFieldName, boolean isEnum, boolean canOverride) @@ -1333,6 +1402,7 @@ private void add(String clsName, TypeDescriptor desc = new TypeDescriptor(clsName, mapper, serializer, + hashingMode, affKeyFieldName, isEnum, canOverride); @@ -1368,6 +1438,9 @@ private static class TypeDescriptor { /** Serializer. */ private BinarySerializer serializer; + /** Key hashing mode. */ + private BinaryKeyHashingMode keyHashingMode; + /** Affinity key field name. */ private String affKeyFieldName; @@ -1383,15 +1456,18 @@ private static class TypeDescriptor { * @param clsName Class name. * @param mapper ID mapper. * @param serializer Serializer. + * @param keyHashingMode Key hashing mode. * @param affKeyFieldName Affinity key field name. * @param isEnum Enum type. * @param canOverride Whether this descriptor can be override. */ private TypeDescriptor(String clsName, BinaryInternalMapper mapper, - BinarySerializer serializer, String affKeyFieldName, boolean isEnum, boolean canOverride) { + BinarySerializer serializer, BinaryKeyHashingMode keyHashingMode, String affKeyFieldName, boolean isEnum, + boolean canOverride) { this.clsName = clsName; this.mapper = mapper; this.serializer = serializer; + this.keyHashingMode = keyHashingMode; this.affKeyFieldName = affKeyFieldName; this.isEnum = isEnum; this.canOverride = canOverride; @@ -1409,6 +1485,7 @@ private void override(TypeDescriptor other) throws BinaryObjectException { if (canOverride) { mapper = other.mapper; serializer = other.serializer; + keyHashingMode = other.keyHashingMode; affKeyFieldName = other.affKeyFieldName; isEnum = other.isEnum; canOverride = other.canOverride; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java index 063bd837ac2cb..886fb609975e3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java @@ -23,9 +23,11 @@ import java.util.Iterator; import java.util.Map; import org.apache.ignite.IgniteException; +import org.apache.ignite.binary.BinaryKeyHashingMode; import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.binary.BinaryObjectBuilder; import org.apache.ignite.binary.BinaryObjectException; +import org.apache.ignite.binary.BinaryObjectHashCodeResolver; import org.apache.ignite.binary.BinaryType; import org.apache.ignite.internal.binary.builder.BinaryObjectBuilderImpl; import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory; @@ -105,6 +107,13 @@ public abstract class BinaryObjectExImpl implements BinaryObjectEx { */ protected abstract BinarySchema createSchema(); + /** + * Get binary context. + * + * @return Binary context. + */ + protected abstract BinaryContext context(); + /** {@inheritDoc} */ @Override public BinaryObjectBuilder toBuilder() throws BinaryObjectException { return BinaryObjectBuilderImpl.wrap(this); @@ -128,6 +137,16 @@ public boolean equals(Object other) { BinaryObjectExImpl otherPo = (BinaryObjectExImpl)other; + BinaryKeyHashingMode hashingMode = context().keyHashingMode(typeId()); + + if (hashingMode == BinaryKeyHashingMode.FIELDS_HASH || hashingMode == BinaryKeyHashingMode.CUSTOM) { + BinaryObjectHashCodeResolver rslvr = context().hashCodeResolver(typeId()); + + assert rslvr != null; + + return rslvr.equals(this, otherPo); + } + if (length() != otherPo.length() || typeId() != otherPo.typeId()) return false; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java index f37d7c2051e36..6e198aa263fc5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java @@ -211,10 +211,8 @@ public void detachAllowed(boolean detachAllowed) { this.detachAllowed = detachAllowed; } - /** - * @return Context. - */ - public BinaryContext context() { + /** {@inheritDoc} */ + @Override public BinaryContext context() { return ctx; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java index 9cbbaa2767092..dbaa3d9790aa2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java @@ -124,6 +124,11 @@ public BinaryObject heapCopy() { return reader(null, false).getOrCreateSchema(); } + /** {@inheritDoc} */ + @Override protected BinaryContext context() { + return ctx; + } + /** {@inheritDoc} */ @Override public int start() { return start; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/FieldsListHashCodeResolver.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/FieldsListHashCodeResolver.java new file mode 100644 index 0000000000000..67608b782e631 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/FieldsListHashCodeResolver.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import java.util.List; +import org.apache.ignite.binary.BinaryObjectBuilder; +import org.apache.ignite.binary.BinaryObjectHashCodeResolver; +import org.apache.ignite.internal.util.typedef.F; + +/** + * Default implementation of fields based hash code resolver. + */ +public final class FieldsListHashCodeResolver implements BinaryObjectHashCodeResolver { + /** + * Fields based on whose values hash code should be computed. + */ + private final List fieldNames; + + /** + * @param fieldNames Fields based on whose values hash code should be computed. + */ + public FieldsListHashCodeResolver(List fieldNames) { + if (F.isEmpty(fieldNames)) + throw new IllegalArgumentException("Empty fields list for FieldsListHashCodeResolver"); + + this.fieldNames = fieldNames; + } + + /** {@inheritDoc} */ + @Override public int hash(BinaryObjectBuilder builder) { + int hash = 0; + + for (String fieldName : fieldNames) { + Object val = builder.getField(fieldName); + + hash = 31 * hash + (val != null ? val.hashCode() : 0); + } + + return hash; + } + + /** {@inheritDoc} */ + @Override public boolean equals(BinaryObjectExImpl o1, BinaryObjectExImpl o2) { + if (o1 == o2) + return true; + + if (o1 == null || o2 == null) + return false; + + for (String fieldName : fieldNames) { + Object v1 = o1.field(fieldName); + + Object v2 = o2.field(fieldName); + + if (!F.eq(v1, v2)) + return false; + } + + return true; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java index 2c761925e68af..9e93818f30587 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java @@ -18,9 +18,11 @@ package org.apache.ignite.internal.binary.builder; import org.apache.ignite.binary.BinaryInvalidTypeException; +import org.apache.ignite.binary.BinaryKeyHashingMode; import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.binary.BinaryObjectBuilder; import org.apache.ignite.binary.BinaryObjectException; +import org.apache.ignite.binary.BinaryObjectHashCodeResolver; import org.apache.ignite.binary.BinaryType; import org.apache.ignite.internal.binary.BinaryMetadata; import org.apache.ignite.internal.binary.BinaryObjectImpl; @@ -36,6 +38,7 @@ import org.apache.ignite.lang.IgniteBiTuple; import org.jetbrains.annotations.Nullable; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -129,6 +132,8 @@ public BinaryObjectBuilderImpl(BinaryObjectImpl obj) { * @param start Start. */ BinaryObjectBuilderImpl(BinaryBuilderReader reader, int start) { + assert reader != null; + this.reader = reader; this.start = start; this.flags = reader.readShortPositioned(start + GridBinaryMarshaller.FLAGS_POS); @@ -192,6 +197,7 @@ public BinaryObjectBuilderImpl(BinaryObjectImpl obj) { * @param writer Writer. * @param serializer Serializer. */ + @SuppressWarnings("ResultOfMethodCallIgnored") void serializeTo(BinaryWriterExImpl writer, BinaryBuilderSerializer serializer) { try { writer.preWrite(registeredType ? null : clsNameToWrite); @@ -332,6 +338,41 @@ else if (readCache == null) { reader.position(start + BinaryUtils.length(reader, start)); } + // Let us not mess with hash codes of objects based on other objects, m'kay? + if (reader == null && !isHashCodeSet) { + BinaryKeyHashingMode hashingMode = ctx.keyHashingMode(typeId()); + + if (hashingMode != null) { + switch (hashingMode) { + case DEFAULT: + // No-op. + break; + + case BYTES_HASH: + // Not considering #start as we're dealing only with #reader free cases. + // At this point, no footer has been written yet, so we are looking at everything + // past the header and down to the end of all data. + byte[] data = Arrays.copyOfRange(writer.array(), hdrLen, writer.out().position()); + + hashCode(Arrays.hashCode(data)); + break; + + case FIELDS_HASH: + case CUSTOM: + BinaryObjectHashCodeResolver rslvr = ctx.hashCodeResolver(typeId()); + + assert rslvr != null; + + hashCode(rslvr.hash(this)); + break; + + default: + throw new BinaryObjectException("Unexpected hashing mode [typeId=" + typeId() + ", typeName=" + + typeName + ']'); + } + } + } + //noinspection NumberEquality writer.postWrite(true, registeredType, hashCode, isHashCodeSet); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java index 7936ea49b7303..21484aeb66aa4 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java @@ -20,11 +20,13 @@ import java.math.BigDecimal; import java.sql.Timestamp; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -36,28 +38,36 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteBinary; import org.apache.ignite.IgniteCache; +import org.apache.ignite.binary.BinaryKeyHashingMode; import org.apache.ignite.binary.BinaryNameMapper; import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.binary.BinaryObjectBuilder; import org.apache.ignite.binary.BinaryObjectException; +import org.apache.ignite.binary.BinaryObjectHashCodeResolver; import org.apache.ignite.binary.BinaryReader; +import org.apache.ignite.binary.BinaryTypeConfiguration; import org.apache.ignite.binary.BinaryWriter; import org.apache.ignite.binary.Binarylizable; import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheKeyConfiguration; import org.apache.ignite.cache.CacheMode; import org.apache.ignite.cache.CachePeekMode; import org.apache.ignite.cache.store.CacheStoreAdapter; +import org.apache.ignite.configuration.BinaryConfiguration; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.NearCacheConfiguration; import org.apache.ignite.internal.IgniteKernal; import org.apache.ignite.internal.binary.BinaryContext; import org.apache.ignite.internal.binary.BinaryMarshaller; +import org.apache.ignite.internal.binary.BinaryObjectExImpl; import org.apache.ignite.internal.binary.BinaryObjectImpl; import org.apache.ignite.internal.binary.BinaryObjectOffheapImpl; import org.apache.ignite.internal.processors.cache.GridCacheAdapter; import org.apache.ignite.internal.processors.cache.GridCacheEntryEx; import org.apache.ignite.internal.processors.cache.IgniteCacheProxy; +import org.apache.ignite.internal.processors.cache.MapCacheStoreStrategy; +import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.P2; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; @@ -103,13 +113,72 @@ public abstract class GridCacheBinaryObjectsAbstractSelfTest extends GridCommonA cfg.setDiscoverySpi(disco); + CacheConfiguration cacheCfg = createCacheConfig(); + + cacheCfg.setCacheStoreFactory(singletonFactory(new TestStore())); + + CacheConfiguration binKeysCacheCfg = createCacheConfig(); + + binKeysCacheCfg.setCacheStoreFactory(singletonFactory(new MapCacheStoreStrategy.MapCacheStore())); + binKeysCacheCfg.setStoreKeepBinary(true); + binKeysCacheCfg.setName("BinKeysCache"); + + cfg.setCacheConfiguration(cacheCfg, binKeysCacheCfg); + cfg.setMarshaller(new BinaryMarshaller()); + + List binTypes = new ArrayList<>(); + + binTypes.add(new BinaryTypeConfiguration() {{ + setTypeName("BytesHashedKey"); + }}); + + binTypes.add(new BinaryTypeConfiguration() {{ + setTypeName("FieldsHashedKey"); + }}); + + binTypes.add(new BinaryTypeConfiguration() {{ + setTypeName("CustomHashedKey"); + }}); + + BinaryConfiguration binCfg = new BinaryConfiguration(); + binCfg.setTypeConfigurations(binTypes); + + cfg.setBinaryConfiguration(binCfg); + + CacheKeyConfiguration arrHashConfiguration = new CacheKeyConfiguration("BytesHashedKey", "fld1") {{ + setBinaryHashingMode(BinaryKeyHashingMode.BYTES_HASH); + }}; + + CacheKeyConfiguration fieldsHashConfiguration = new CacheKeyConfiguration("FieldsHashedKey", "fld1") {{ + setBinaryHashingMode(BinaryKeyHashingMode.FIELDS_HASH); + + setBinaryHashCodeFields(Arrays.asList("fld1", "fld3")); + }}; + + CacheKeyConfiguration customHashConfiguration = new CacheKeyConfiguration("CustomHashedKey", "fld1") {{ + setBinaryHashingMode(BinaryKeyHashingMode.CUSTOM); + + setBinHashCodeResolverClassName(HashCodeResolver.class.getName()); + }}; + + cfg.setCacheKeyConfiguration(arrHashConfiguration, fieldsHashConfiguration, customHashConfiguration); + + GridCacheBinaryObjectsAbstractSelfTest.cfg = cfg; + + return cfg; + } + + /** + * @return Cache configuration with basic settings. + */ + @SuppressWarnings("unchecked") + private CacheConfiguration createCacheConfig() { CacheConfiguration cacheCfg = new CacheConfiguration(); cacheCfg.setCacheMode(cacheMode()); cacheCfg.setAtomicityMode(atomicityMode()); cacheCfg.setNearConfiguration(nearConfiguration()); cacheCfg.setWriteSynchronizationMode(FULL_SYNC); - cacheCfg.setCacheStoreFactory(singletonFactory(new TestStore())); cacheCfg.setReadThrough(true); cacheCfg.setWriteThrough(true); cacheCfg.setLoadPreviousValue(true); @@ -120,13 +189,7 @@ public abstract class GridCacheBinaryObjectsAbstractSelfTest extends GridCommonA cacheCfg.setOffHeapMaxMemory(0); } - cfg.setCacheConfiguration(cacheCfg); - - cfg.setMarshaller(new BinaryMarshaller()); - - this.cfg = cfg; - - return cfg; + return cacheCfg; } /** @@ -917,6 +980,101 @@ public void testPutWithoutHashCode() throws Exception { "please set it explicitly during building by using BinaryBuilder.hashCode(int)"); } + /** + * + */ + @SuppressWarnings("unchecked") + public void testPutWithArrayHashing() { + IgniteCache c = binKeysCache(); + + { + BinaryObjectBuilder bldr = grid(0).binary().builder("BytesHashedKey"); + + bldr.setField("fld1", 5); + bldr.setField("fld2", "abc"); + + BinaryObject binKey = bldr.build(); + + c.put(binKey, "zzz"); + } + + // Now let's build an identical key for get + { + BinaryObjectBuilder bldr = grid(0).binary().builder("BytesHashedKey"); + + bldr.setField("fld1", 5); + bldr.setField("fld2", "abc"); + + BinaryObject binKey = bldr.build(); + + assertEquals("zzz", c.get(binKey)); + } + } + + /** + * + */ + @SuppressWarnings("unchecked") + public void testPutWithFieldsHashing() { + IgniteCache c = binKeysCache(); + + { + BinaryObjectBuilder bldr = grid(0).binary().builder("FieldsHashedKey"); + + bldr.setField("fld1", 5); + bldr.setField("fld2", 1); + bldr.setField("fld3", "abc"); + + BinaryObject binKey = bldr.build(); + + c.put(binKey, "zzz"); + } + + // Now let's build an identical key for get + { + BinaryObjectBuilder bldr = grid(0).binary().builder("FieldsHashedKey"); + + bldr.setField("fld1", 5); + bldr.setField("fld2", 100); // This one does not participate in hashing + bldr.setField("fld3", "abc"); + + BinaryObject binKey = bldr.build(); + + c.put(binKey, "zzz"); + } + } + + /** + * + */ + @SuppressWarnings("unchecked") + public void testPutWithCustomHashing() { + IgniteCache c = binKeysCache(); + + { + BinaryObjectBuilder bldr = grid(0).binary().builder("CustomHashedKey"); + + bldr.setField("fld1", 5); + bldr.setField("fld2", "abc"); + + BinaryObject binKey = bldr.build(); + + c.put(binKey, "zzz"); + } + + // Now let's build an identical key for get + { + BinaryObjectBuilder bldr = grid(0).binary().builder("CustomHashedKey"); + + bldr.setField("fld1", 5); + bldr.setField("fld2", "xxx"); + + BinaryObject binKey = bldr.build(); + + assertEquals("zzz", c.get(binKey)); + } + } + /** * @throws Exception if failed. */ @@ -1027,6 +1185,13 @@ private IgniteCache keepBinaryCache() { return ignite(0).cache(null).withKeepBinary(); } + /** + * @return Cache tuned to utilize classless binary objects as keys. + */ + private IgniteCache binKeysCache() { + return ignite(0).cache("BinKeysCache").withKeepBinary(); + } + /** * @param key Key. * @throws Exception If failed. @@ -1215,4 +1380,20 @@ private static class TestStore extends CacheStoreAdapter { // No-op. } } + + /** + * + */ + private final static class HashCodeResolver implements BinaryObjectHashCodeResolver { + /** {@inheritDoc} */ + @Override public int hash(BinaryObjectBuilder builder) { + return (Integer) builder.getField("fld1") * 31 / 5; + } + + /** {@inheritDoc} */ + @Override public boolean equals(BinaryObjectExImpl o1, BinaryObjectExImpl o2) { + return o1 == o2 || (o1 != null && o2 != null && F.eq(o1.field("fld1"), o2.field("fld1"))); + + } + } } From 9d2a2e4b8f803eb1f26dc70a7e5d826f56b3c141 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Thu, 6 Oct 2016 23:43:40 +0300 Subject: [PATCH 2/9] Comment fixed --- .../java/org/apache/ignite/binary/BinaryKeyHashingMode.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/binary/BinaryKeyHashingMode.java b/modules/core/src/main/java/org/apache/ignite/binary/BinaryKeyHashingMode.java index b43f2eee85557..4de62042dee9d 100644 --- a/modules/core/src/main/java/org/apache/ignite/binary/BinaryKeyHashingMode.java +++ b/modules/core/src/main/java/org/apache/ignite/binary/BinaryKeyHashingMode.java @@ -39,8 +39,9 @@ public enum BinaryKeyHashingMode { /** * Generate hash code based upon serialized representation of binary object fields - namely, byte array constructed * by {@link BinaryObjectBuilder}. Use this mode if you are NOT planning to retrieve data from cache via - * ordinary cache methods like {@link IgniteCache#get(Object)}, {@link IgniteCache#getAll(Set)}, etc. - - * it's an convenient way to manipulate and retrieve binary data in cache only via full-scale SQL features + * ordinary cache methods like {@link IgniteCache#get(Object)}, {@link IgniteCache#getAll(Set)}, etc., or + * if you don't have particular classes for keys neither on client nor on server - it's an convenient way + * to manipulate and retrieve binary data in cache only via full-scale SQL features * with as little additional configuration overhead as choosing this mode. */ BYTES_HASH, From 1bf64ad6652d80674ad7d6600688381fa7be664e Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Tue, 11 Oct 2016 13:59:34 +0300 Subject: [PATCH 3/9] IGNITE-4011 Code review fixes and new full hashing/comparison mode for DML --- .../ignite/binary/BinaryKeyHashingMode.java | 126 ------------------ .../binary/BinaryTypeConfiguration.java | 21 +++ ...eResolver.java => BinaryTypeIdentity.java} | 10 +- .../ignite/cache/CacheKeyConfiguration.java | 81 ----------- .../configuration/BinaryConfiguration.java | 22 +++ .../ignite/internal/binary/BinaryContext.java | 99 +++++--------- .../internal/binary/BinaryObjectExImpl.java | 16 +-- .../ignite/internal/binary/BinarySchema.java | 19 +++ .../ignite/internal/binary/BinaryUtils.java | 2 +- ...eResolver.java => FieldsListIdentity.java} | 59 +++++--- .../ignite/internal/binary/FullIdentity.java | 78 +++++++++++ .../builder/BinaryObjectBuilderImpl.java | 58 +++----- ...ridCacheBinaryObjectsAbstractSelfTest.java | 65 +++++---- 13 files changed, 282 insertions(+), 374 deletions(-) delete mode 100644 modules/core/src/main/java/org/apache/ignite/binary/BinaryKeyHashingMode.java rename modules/core/src/main/java/org/apache/ignite/binary/{BinaryObjectHashCodeResolver.java => BinaryTypeIdentity.java} (80%) rename modules/core/src/main/java/org/apache/ignite/internal/binary/{FieldsListHashCodeResolver.java => FieldsListIdentity.java} (50%) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/binary/FullIdentity.java diff --git a/modules/core/src/main/java/org/apache/ignite/binary/BinaryKeyHashingMode.java b/modules/core/src/main/java/org/apache/ignite/binary/BinaryKeyHashingMode.java deleted file mode 100644 index 4de62042dee9d..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/binary/BinaryKeyHashingMode.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.binary; - -import java.util.Set; -import org.apache.ignite.IgniteCache; -import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.cache.CacheKeyConfiguration; -import org.apache.ignite.internal.binary.FieldsListHashCodeResolver; -import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.internal.U; - -/** - * Mode of generating hash codes for keys created with {@link BinaryObjectBuilder}. - */ -public enum BinaryKeyHashingMode { - /** - * Default (also legacy pre 1.8) mode. Use this mode if you use no SQL DML commands - INSERT, UPDATE, DELETE, MERGE, - * in other words, if you put data to cache NOT via SQL. - * Effect from choosing this mode is identical to omitting mode settings from key configuration at all. - */ - DEFAULT, - - /** - * Generate hash code based upon serialized representation of binary object fields - namely, byte array constructed - * by {@link BinaryObjectBuilder}. Use this mode if you are NOT planning to retrieve data from cache via - * ordinary cache methods like {@link IgniteCache#get(Object)}, {@link IgniteCache#getAll(Set)}, etc., or - * if you don't have particular classes for keys neither on client nor on server - it's an convenient way - * to manipulate and retrieve binary data in cache only via full-scale SQL features - * with as little additional configuration overhead as choosing this mode. - */ - BYTES_HASH, - - /** - * Generate hash code based upon on list of fields declared in {@link BinaryObjectBuilder} - * (not in {@link BinaryObject} as hash code has to be computed before {@link BinaryObject} is fully built) - - * this mode requires that you set {@link CacheKeyConfiguration#binHashCodeFields} for it to work. - */ - FIELDS_HASH { - /** {@inheritDoc} */ - @Override public BinaryObjectHashCodeResolver createResolver(CacheKeyConfiguration keyConfiguration) { - return new FieldsListHashCodeResolver(keyConfiguration.getBinaryHashCodeFields()); - } - }, - - /** - * Generate hash code arbitrarily based on {@link BinaryObjectBuilder} using specified class implementing - * {@link BinaryObjectHashCodeResolver}- this mode requires that you set - * {@link CacheKeyConfiguration#binHashCodeRslvrClsName} for it to work. - */ - CUSTOM { - /** {@inheritDoc} */ - @Override public BinaryObjectHashCodeResolver createResolver(CacheKeyConfiguration keyConfiguration) { - String clsName = keyConfiguration.getTypeName(); - - String rslvrClsName = keyConfiguration.getBinaryHashCodeResolverClassName(); - - if (F.isEmpty(rslvrClsName)) - throw new IllegalArgumentException("No custom hash code resolver class name specified, please review " + - "your configuration [clsName=" + clsName + ']'); - - Class rslvrCls = U.classForName(rslvrClsName, null); - - if (rslvrCls == null) - throw new IllegalArgumentException("Hash code resolver class not found [clsName=" + clsName + - ", rslvrClsName=" + rslvrClsName + ']'); - - if (!BinaryObjectHashCodeResolver.class.isAssignableFrom(rslvrCls)) - throw new IllegalArgumentException("Hash code resolver class does not implement " + - "BinaryObjectHashCodeResolver interface [clsName=" + clsName + ", rslvrClsName=" + - rslvrClsName + ']'); - - try { - return (BinaryObjectHashCodeResolver) U.newInstance(rslvrCls); - } - catch (IgniteCheckedException e) { - throw new BinaryObjectException("Failed to instantiate hash code resolver [clsName=" + - clsName + ", rslvrClsName=" + rslvrClsName + ']', e); - } - } - }; - - /** - * @return {@code true} if this mode relies on params set in {@link CacheKeyConfiguration}. - */ - public final boolean isConfigurable() { - switch (this) { - case FIELDS_HASH: - case CUSTOM: - return true; - case DEFAULT: - case BYTES_HASH: - return false; - default: - throw new IllegalArgumentException("Unexpected key hashing mode [mode=" + this.name() + ']'); - } - } - - /** - * Instantiate {@link BinaryObjectHashCodeResolver} based on given {@link CacheKeyConfiguration}. - * - * @param keyConfiguration Configuration to create resolver based upon. - * @return Hash code resolver. - */ - public BinaryObjectHashCodeResolver createResolver(CacheKeyConfiguration keyConfiguration) { - assert !isConfigurable() : "Configurable hashing mode does not override createResolver"; - - throw new UnsupportedOperationException("This binary keys hashing mode is not configurable, " + - "only modes " + FIELDS_HASH.name() + " and " + CUSTOM.name() + " are."); - } -} diff --git a/modules/core/src/main/java/org/apache/ignite/binary/BinaryTypeConfiguration.java b/modules/core/src/main/java/org/apache/ignite/binary/BinaryTypeConfiguration.java index fea0af737001a..f0d96a6a8f602 100644 --- a/modules/core/src/main/java/org/apache/ignite/binary/BinaryTypeConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/binary/BinaryTypeConfiguration.java @@ -42,6 +42,9 @@ public class BinaryTypeConfiguration { /** Serializer. */ private BinarySerializer serializer; + /** Identity. */ + private BinaryTypeIdentity identity; + /** Enum flag. */ private boolean isEnum; @@ -145,6 +148,24 @@ public void setSerializer(BinarySerializer serializer) { this.serializer = serializer; } + /** + * Gets identity. + * + * @return Identity. + */ + public BinaryTypeIdentity getIdentity() { + return identity; + } + + /** + * Sets type identity. + * + * @param identity Identity. + */ + public void setIdentity(BinaryTypeIdentity identity) { + this.identity = identity; + } + /** * Gets whether this is enum type. * diff --git a/modules/core/src/main/java/org/apache/ignite/binary/BinaryObjectHashCodeResolver.java b/modules/core/src/main/java/org/apache/ignite/binary/BinaryTypeIdentity.java similarity index 80% rename from modules/core/src/main/java/org/apache/ignite/binary/BinaryObjectHashCodeResolver.java rename to modules/core/src/main/java/org/apache/ignite/binary/BinaryTypeIdentity.java index bb22bee6c366d..d7d252ea4bb72 100644 --- a/modules/core/src/main/java/org/apache/ignite/binary/BinaryObjectHashCodeResolver.java +++ b/modules/core/src/main/java/org/apache/ignite/binary/BinaryTypeIdentity.java @@ -17,17 +17,15 @@ package org.apache.ignite.binary; -import org.apache.ignite.internal.binary.BinaryObjectExImpl; - /** * Method to compute hash codes for new binary objects. */ -public interface BinaryObjectHashCodeResolver { +public interface BinaryTypeIdentity { /** - * @param builder Binary object builder. + * @param obj Binary object builder. * @return Hash code value. */ - public int hash(BinaryObjectBuilder builder); + public int hash(BinaryObject obj); /** * Compare binary objects for equality in consistence with how hash code is computed. @@ -36,5 +34,5 @@ public interface BinaryObjectHashCodeResolver { * @param o2 Second object. * @return */ - public boolean equals(BinaryObjectExImpl o1, BinaryObjectExImpl o2); + public boolean equals(BinaryObject o1, BinaryObject o2); } diff --git a/modules/core/src/main/java/org/apache/ignite/cache/CacheKeyConfiguration.java b/modules/core/src/main/java/org/apache/ignite/cache/CacheKeyConfiguration.java index 63dcba2eb041c..f1178475b7536 100644 --- a/modules/core/src/main/java/org/apache/ignite/cache/CacheKeyConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/cache/CacheKeyConfiguration.java @@ -19,11 +19,7 @@ import java.io.Serializable; import java.lang.reflect.Field; -import java.util.List; -import org.apache.ignite.binary.BinaryKeyHashingMode; -import org.apache.ignite.binary.BinaryObjectHashCodeResolver; import org.apache.ignite.cache.affinity.AffinityKeyMapped; -import org.apache.ignite.internal.binary.FieldsListHashCodeResolver; import org.apache.ignite.internal.util.typedef.internal.S; /** @@ -39,15 +35,6 @@ public class CacheKeyConfiguration implements Serializable { /** Affinity key field name. */ private String affKeyFieldName; - /** Key hashing mode. */ - private BinaryKeyHashingMode binHashingMode; - - /** Fields to build binary objects' hash code upon. */ - private List binHashCodeFields; - - /** Class name for hash code resolver to automatically compute hash codes for newly built binary objects. */ - private String binHashCodeRslvrClsName; - /** * Creates an empty cache key configuration that should be populated via setters. */ @@ -117,74 +104,6 @@ public void setAffinityKeyFieldName(String affKeyFieldName) { this.affKeyFieldName = affKeyFieldName; } - /** - * Gets hashing mode for binary keys. - * - * @return Binary hashing mode. - * @see BinaryKeyHashingMode - */ - public BinaryKeyHashingMode getBinaryHashingMode() { - return binHashingMode; - } - - /** - * Sets hashing more for binary keys. - * - * @param binHashingMode Binary hashing mode. - * @see BinaryKeyHashingMode - */ - public void setBinaryHashingMode(BinaryKeyHashingMode binHashingMode) { - this.binHashingMode = binHashingMode; - } - - /** - * Gets list of fields for fields list hash code resolver. - * - * @return List of fields for fields list hash code resolver. - * @see #binHashingMode - * @see BinaryKeyHashingMode#FIELDS_HASH - * @see FieldsListHashCodeResolver - */ - public List getBinaryHashCodeFields() { - return binHashCodeFields; - } - - /** - * Sets list of fields for fields list hash code resolver. - * - * @param binHashCodeFields List of fields for fields list hash code resolver. - * @see #binHashingMode - * @see BinaryKeyHashingMode#FIELDS_HASH - * @see FieldsListHashCodeResolver - */ - public void setBinaryHashCodeFields(List binHashCodeFields) { - this.binHashCodeFields = binHashCodeFields; - } - - /** - * Gets binary objects hash code resolver class name. - * - * @return binary objects hash code resolver class name. - * @see #binHashingMode - * @see BinaryKeyHashingMode#CUSTOM - * @see BinaryObjectHashCodeResolver - */ - public String getBinaryHashCodeResolverClassName() { - return binHashCodeRslvrClsName; - } - - /** - * Sets binary objects hash code resolver class name. - * - * @param binHashCodeRslvrClsName resolver class name. - * @see #binHashingMode - * @see BinaryKeyHashingMode#CUSTOM - * @see BinaryObjectHashCodeResolver - */ - public void setBinHashCodeResolverClassName(String binHashCodeRslvrClsName) { - this.binHashCodeRslvrClsName = binHashCodeRslvrClsName; - } - /** {@inheritDoc} */ @Override public String toString() { return S.toString(CacheKeyConfiguration.class, this); diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/BinaryConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/BinaryConfiguration.java index 30d77de58e0d8..40ce9d63713e6 100644 --- a/modules/core/src/main/java/org/apache/ignite/configuration/BinaryConfiguration.java +++ b/modules/core/src/main/java/org/apache/ignite/configuration/BinaryConfiguration.java @@ -23,6 +23,7 @@ import org.apache.ignite.binary.BinaryNameMapper; import org.apache.ignite.binary.BinarySerializer; import org.apache.ignite.binary.BinaryTypeConfiguration; +import org.apache.ignite.binary.BinaryTypeIdentity; import org.apache.ignite.internal.util.typedef.internal.S; /** @@ -42,6 +43,9 @@ public class BinaryConfiguration { /** Serializer. */ private BinarySerializer serializer; + /** Identity. */ + private BinaryTypeIdentity identity; + /** Types. */ private Collection typeCfgs; @@ -115,6 +119,24 @@ public void setSerializer(BinarySerializer serializer) { this.serializer = serializer; } + /** + * Gets identity. + * + * @return Identity. + */ + public BinaryTypeIdentity getIdentity() { + return identity; + } + + /** + * Sets type identity. + * + * @param identity Identity. + */ + public void setIdentity(BinaryTypeIdentity identity) { + this.identity = identity; + } + /** * Gets types configuration. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java index a92c2b2947a66..fd0dc162f52df 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java @@ -24,11 +24,9 @@ import org.apache.ignite.binary.BinaryBasicNameMapper; import org.apache.ignite.binary.BinaryIdMapper; import org.apache.ignite.binary.BinaryInvalidTypeException; -import org.apache.ignite.binary.BinaryKeyHashingMode; import org.apache.ignite.binary.BinaryNameMapper; -import org.apache.ignite.binary.BinaryObjectBuilder; import org.apache.ignite.binary.BinaryObjectException; -import org.apache.ignite.binary.BinaryObjectHashCodeResolver; +import org.apache.ignite.binary.BinaryTypeIdentity; import org.apache.ignite.binary.BinaryReflectiveSerializer; import org.apache.ignite.binary.BinarySerializer; import org.apache.ignite.binary.BinaryType; @@ -39,8 +37,6 @@ import org.apache.ignite.configuration.BinaryConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.igfs.IgfsPath; -import org.apache.ignite.internal.binary.builder.BinaryObjectBuilderImpl; -import org.apache.ignite.internal.processors.cache.CacheObjectImpl; import org.apache.ignite.internal.processors.cache.binary.BinaryMetadataKey; import org.apache.ignite.internal.processors.closure.GridClosureProcessor; import org.apache.ignite.internal.processors.datastructures.CollocatedQueueItemKey; @@ -227,10 +223,7 @@ public class BinaryContext { private final ConcurrentMap cls2Mappers = new ConcurrentHashMap8<>(0); /** Affinity key field names. */ - private final ConcurrentMap hashingModes = new ConcurrentHashMap8<>(0); - - /** Affinity key field names. */ - private final ConcurrentMap hashCodeRslvrs = new ConcurrentHashMap8<>(0); + private final ConcurrentMap typeIdentities = new ConcurrentHashMap8<>(0); /** */ private BinaryMetadataHandler metaHnd; @@ -407,6 +400,7 @@ public void configure(BinaryMarshaller marsh, IgniteConfiguration cfg) throws Bi binaryCfg.getNameMapper(), binaryCfg.getIdMapper(), binaryCfg.getSerializer(), + binaryCfg.getIdentity(), binaryCfg.getTypeConfigurations() ); @@ -416,6 +410,7 @@ public void configure(BinaryMarshaller marsh, IgniteConfiguration cfg) throws Bi /** * @param globalIdMapper ID mapper. * @param globalSerializer Serializer. + * @param globalIdentity Identity. * @param typeCfgs Type configurations. * @throws BinaryObjectException In case of error. */ @@ -423,24 +418,18 @@ private void configure( BinaryNameMapper globalNameMapper, BinaryIdMapper globalIdMapper, BinarySerializer globalSerializer, + BinaryTypeIdentity globalIdentity, Collection typeCfgs ) throws BinaryObjectException { TypeDescriptors descs = new TypeDescriptors(); Map affFields = new HashMap<>(); - Map cfgHashingModes = new HashMap<>(); - - Map keyCfgs = new HashMap<>(); + Map identities = new HashMap<>(); if (!F.isEmpty(igniteCfg.getCacheKeyConfiguration())) { - for (CacheKeyConfiguration keyCfg : igniteCfg.getCacheKeyConfiguration()) { + for (CacheKeyConfiguration keyCfg : igniteCfg.getCacheKeyConfiguration()) affFields.put(keyCfg.getTypeName(), keyCfg.getAffinityKeyFieldName()); - - keyCfgs.put(keyCfg.getTypeName(), keyCfg); - - cfgHashingModes.put(keyCfg.getTypeName(), keyCfg.getBinaryHashingMode()); - } } if (typeCfgs != null) { @@ -456,37 +445,31 @@ private void configure( if (typeCfg.getIdMapper() != null) idMapper = typeCfg.getIdMapper(); - BinaryNameMapper nameMapper = globalNameMapper; - - if (typeCfg.getNameMapper() != null) - nameMapper = typeCfg.getNameMapper(); + BinaryNameMapper nameMapper = U.firstNotNull(typeCfg.getNameMapper(), globalNameMapper); BinaryInternalMapper mapper = resolveMapper(nameMapper, idMapper); // Resolve serializer. - BinarySerializer serializer = globalSerializer; + BinarySerializer serializer = U.firstNotNull(typeCfg.getSerializer(), globalSerializer); - BinaryKeyHashingMode hashingMode = cfgHashingModes.get(clsName); - - if (typeCfg.getSerializer() != null) - serializer = typeCfg.getSerializer(); + BinaryTypeIdentity identity = U.firstNotNull(typeCfg.getIdentity(), globalIdentity); if (clsName.endsWith(".*")) { String pkgName = clsName.substring(0, clsName.length() - 2); for (String clsName0 : classesInPackage(pkgName)) - descs.add(clsName0, mapper, serializer, hashingMode, affFields.get(clsName0), + descs.add(clsName0, mapper, serializer, identity, affFields.get(clsName0), typeCfg.isEnum(), true); } else - descs.add(clsName, mapper, serializer, hashingMode, affFields.get(clsName), + descs.add(clsName, mapper, serializer, identity, affFields.get(clsName), typeCfg.isEnum(), false); } } for (TypeDescriptor desc : descs.descriptors()) - registerUserType(desc.clsName, desc.mapper, desc.serializer, desc.keyHashingMode, desc.affKeyFieldName, - desc.isEnum, keyCfgs.get(desc.clsName)); + registerUserType(desc.clsName, desc.mapper, desc.serializer, desc.identity, desc.affKeyFieldName, + desc.isEnum); BinaryInternalMapper globalMapper = resolveMapper(globalNameMapper, globalIdMapper); @@ -1110,20 +1093,18 @@ public BinaryClassDescriptor registerPredefinedType(Class cls, int id, String * @param clsName Class name. * @param mapper ID mapper. * @param serializer Serializer. - * @param keyHashingMode Key hashing mode. + * @param identity Type identity. * @param affKeyFieldName Affinity key field name. * @param isEnum If enum. - * @param keyConfiguration Key configuration, if applicable. * @throws BinaryObjectException In case of error. */ @SuppressWarnings("ErrorNotRethrown") public void registerUserType(String clsName, BinaryInternalMapper mapper, @Nullable BinarySerializer serializer, - @Nullable BinaryKeyHashingMode keyHashingMode, + @Nullable BinaryTypeIdentity identity, @Nullable String affKeyFieldName, - boolean isEnum, - @Nullable CacheKeyConfiguration keyConfiguration) + boolean isEnum) throws BinaryObjectException { assert mapper != null; @@ -1147,18 +1128,9 @@ public void registerUserType(String clsName, if (typeId2Mapper.put(id, mapper) != null) dup(clsName, id); - if (keyHashingMode != null) { - if (hashingModes.put(id, keyHashingMode) != null) + if (identity != null) { + if (typeIdentities.put(id, identity) != null) dup(clsName, id); - - if (keyHashingMode.isConfigurable()) { - assert keyConfiguration != null; - - BinaryObjectHashCodeResolver rslvr = keyHashingMode.createResolver(keyConfiguration); - - if (hashCodeRslvrs.put(id, rslvr) != null) - dup(clsName, id); - } } if (affKeyFieldName != null) { @@ -1262,18 +1234,10 @@ public String affinityKeyFieldName(int typeId) { /** * @param typeId Type ID. - * @return Key hashing mode. + * @return Type identity. */ - public BinaryKeyHashingMode keyHashingMode(int typeId) { - return hashingModes.get(typeId); - } - - /** - * @param typeId Type ID. - * @return Key hashing mode. - */ - public BinaryObjectHashCodeResolver hashCodeResolver(int typeId) { - return hashCodeRslvrs.get(typeId); + public BinaryTypeIdentity identity(int typeId) { + return typeIdentities.get(typeId); } /** @@ -1385,7 +1349,7 @@ private static class TypeDescriptors { * @param clsName Class name. * @param mapper Mapper. * @param serializer Serializer. - * @param hashingMode Key hashing mode. + * @param identity Key hashing mode. * @param affKeyFieldName Affinity key field name. * @param isEnum Enum flag. * @param canOverride Whether this descriptor can be override. @@ -1394,7 +1358,7 @@ private static class TypeDescriptors { private void add(String clsName, BinaryInternalMapper mapper, BinarySerializer serializer, - BinaryKeyHashingMode hashingMode, + BinaryTypeIdentity identity, String affKeyFieldName, boolean isEnum, boolean canOverride) @@ -1402,7 +1366,7 @@ private void add(String clsName, TypeDescriptor desc = new TypeDescriptor(clsName, mapper, serializer, - hashingMode, + identity, affKeyFieldName, isEnum, canOverride); @@ -1438,8 +1402,8 @@ private static class TypeDescriptor { /** Serializer. */ private BinarySerializer serializer; - /** Key hashing mode. */ - private BinaryKeyHashingMode keyHashingMode; + /** Type identity. */ + private BinaryTypeIdentity identity; /** Affinity key field name. */ private String affKeyFieldName; @@ -1452,22 +1416,21 @@ private static class TypeDescriptor { /** * Constructor. - * * @param clsName Class name. * @param mapper ID mapper. * @param serializer Serializer. - * @param keyHashingMode Key hashing mode. + * @param identity Key hashing mode. * @param affKeyFieldName Affinity key field name. * @param isEnum Enum type. * @param canOverride Whether this descriptor can be override. */ private TypeDescriptor(String clsName, BinaryInternalMapper mapper, - BinarySerializer serializer, BinaryKeyHashingMode keyHashingMode, String affKeyFieldName, boolean isEnum, + BinarySerializer serializer, BinaryTypeIdentity identity, String affKeyFieldName, boolean isEnum, boolean canOverride) { this.clsName = clsName; this.mapper = mapper; this.serializer = serializer; - this.keyHashingMode = keyHashingMode; + this.identity = identity; this.affKeyFieldName = affKeyFieldName; this.isEnum = isEnum; this.canOverride = canOverride; @@ -1485,7 +1448,7 @@ private void override(TypeDescriptor other) throws BinaryObjectException { if (canOverride) { mapper = other.mapper; serializer = other.serializer; - keyHashingMode = other.keyHashingMode; + identity = other.identity; affKeyFieldName = other.affKeyFieldName; isEnum = other.isEnum; canOverride = other.canOverride; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java index 886fb609975e3..b5858d0e3fbde 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectExImpl.java @@ -23,11 +23,10 @@ import java.util.Iterator; import java.util.Map; import org.apache.ignite.IgniteException; -import org.apache.ignite.binary.BinaryKeyHashingMode; import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.binary.BinaryObjectBuilder; import org.apache.ignite.binary.BinaryObjectException; -import org.apache.ignite.binary.BinaryObjectHashCodeResolver; +import org.apache.ignite.binary.BinaryTypeIdentity; import org.apache.ignite.binary.BinaryType; import org.apache.ignite.internal.binary.builder.BinaryObjectBuilderImpl; import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory; @@ -137,15 +136,14 @@ public boolean equals(Object other) { BinaryObjectExImpl otherPo = (BinaryObjectExImpl)other; - BinaryKeyHashingMode hashingMode = context().keyHashingMode(typeId()); + BinaryTypeIdentity identity = context().identity(typeId()); - if (hashingMode == BinaryKeyHashingMode.FIELDS_HASH || hashingMode == BinaryKeyHashingMode.CUSTOM) { - BinaryObjectHashCodeResolver rslvr = context().hashCodeResolver(typeId()); + // Equivalence relation has to be symmetrical, so it should not matter which identity we choose + if (identity == null) + identity = context().identity(otherPo.typeId()); - assert rslvr != null; - - return rslvr.equals(this, otherPo); - } + if (identity != null) + return identity.equals(this, otherPo); if (length() != otherPo.length() || typeId() != otherPo.typeId()) return false; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinarySchema.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinarySchema.java index 125719e195fb9..78ac8177fd0be 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinarySchema.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinarySchema.java @@ -24,6 +24,7 @@ import java.io.ObjectInput; import java.io.ObjectOutput; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.List; @@ -52,6 +53,9 @@ public class BinarySchema implements Externalizable { /** IDs depending on order. */ private int[] ids; + /** Sorted IDs to compute hash code. */ + private int[] idsSorted; + /** Interned names of associated fields. */ private String[] names; @@ -269,6 +273,15 @@ public int[] fieldIds() { return ids; } + /** + * Gets sorted field ids array to compute hash codes. + * + * @return Sorted field ids. + */ + public int[] fieldIdsSorted() { + return idsSorted; + } + /** * Parse values. * @@ -371,6 +384,12 @@ private void initialize(List fieldIds) { for (int i = 0; i < fieldIds.size(); i++) ids[i] = fieldIds.get(i); + idsSorted = new int[fieldIds.size()]; + + System.arraycopy(ids, 0, idsSorted, 0, ids.length); + + Arrays.sort(idsSorted); + names = new String[fieldIds.size()]; if (fieldIds.size() <= 4) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java index 25d87ffe802be..c3af36b857d32 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java @@ -308,7 +308,7 @@ public static boolean isCompactFooter(short flags) { * @param flag Flag. * @return {@code True} if flag is set in flags. */ - static boolean isFlagSet(short flags, short flag) { + public static boolean isFlagSet(short flags, short flag) { return (flags & flag) == flag; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/FieldsListHashCodeResolver.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/FieldsListIdentity.java similarity index 50% rename from modules/core/src/main/java/org/apache/ignite/internal/binary/FieldsListHashCodeResolver.java rename to modules/core/src/main/java/org/apache/ignite/internal/binary/FieldsListIdentity.java index 67608b782e631..0622933d7689c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/FieldsListHashCodeResolver.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/FieldsListIdentity.java @@ -18,35 +18,33 @@ package org.apache.ignite.internal.binary; import java.util.List; -import org.apache.ignite.binary.BinaryObjectBuilder; -import org.apache.ignite.binary.BinaryObjectHashCodeResolver; +import org.apache.ignite.binary.BinaryObject; +import org.apache.ignite.binary.BinaryTypeIdentity; +import org.apache.ignite.internal.binary.builder.BinaryObjectBuilderImpl; import org.apache.ignite.internal.util.typedef.F; /** * Default implementation of fields based hash code resolver. */ -public final class FieldsListHashCodeResolver implements BinaryObjectHashCodeResolver { +public final class FieldsListIdentity implements BinaryTypeIdentity { /** * Fields based on whose values hash code should be computed. */ - private final List fieldNames; + private List fieldNames; - /** - * @param fieldNames Fields based on whose values hash code should be computed. - */ - public FieldsListHashCodeResolver(List fieldNames) { - if (F.isEmpty(fieldNames)) - throw new IllegalArgumentException("Empty fields list for FieldsListHashCodeResolver"); + /** {@inheritDoc} */ + @Override public int hash(BinaryObject obj) { + assert obj instanceof BinaryObjectExImpl; - this.fieldNames = fieldNames; - } + BinaryObjectExImpl exObj = (BinaryObjectExImpl) obj; + + // Let's create only one reader inside builder instead of new one on each call of field() + BinaryObjectBuilderImpl bldr = (BinaryObjectBuilderImpl) exObj.toBuilder(); - /** {@inheritDoc} */ - @Override public int hash(BinaryObjectBuilder builder) { int hash = 0; for (String fieldName : fieldNames) { - Object val = builder.getField(fieldName); + Object val = bldr.getField(fieldName); hash = 31 * hash + (val != null ? val.hashCode() : 0); } @@ -55,22 +53,43 @@ public FieldsListHashCodeResolver(List fieldNames) { } /** {@inheritDoc} */ - @Override public boolean equals(BinaryObjectExImpl o1, BinaryObjectExImpl o2) { + @Override public boolean equals(BinaryObject o1, BinaryObject o2) { if (o1 == o2) return true; if (o1 == null || o2 == null) return false; - for (String fieldName : fieldNames) { - Object v1 = o1.field(fieldName); + if (!(o1 instanceof BinaryObjectExImpl) || !(o2 instanceof BinaryObjectExImpl)) + return false; + + BinaryObjectExImpl exObj1 = (BinaryObjectExImpl) o1; + + BinaryObjectBuilderImpl bldr1 = (BinaryObjectBuilderImpl) exObj1.toBuilder(); - Object v2 = o2.field(fieldName); + BinaryObjectExImpl exObj2 = (BinaryObjectExImpl) o2; - if (!F.eq(v1, v2)) + BinaryObjectBuilderImpl bldr2 = (BinaryObjectBuilderImpl) exObj2.toBuilder(); + + for (String fld : fieldNames) { + if (!F.eq(bldr1.getField(fld), bldr2.getField(fld))) return false; } return true; } + + /** + * @return Fields list to hash/compare objects based upon. + */ + public List getFieldNames() { + return fieldNames; + } + + /** + * @param fieldNames Fields list to hash/compare objects based upon. + */ + public void setFieldNames(List fieldNames) { + this.fieldNames = fieldNames; + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/FullIdentity.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/FullIdentity.java new file mode 100644 index 0000000000000..6b6f16ab27c58 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/FullIdentity.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import org.apache.ignite.IgniteBinary; +import org.apache.ignite.binary.BinaryObject; +import org.apache.ignite.binary.BinaryTypeIdentity; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.A; + +/** + * Hash and compare instances of {@link BinaryObjectExImpl} for equality using values of all their fields. + * This has been introduced primarily for use by DML engine to free the user from the need to worry about hash codes. + * This hashing method should be used only in cases when you are sure that compared objects have full field sets. + * Objects built via {@link IgniteBinary#toBinary} and those constructed inside DML engine satisfy that requirement. + */ +public class FullIdentity implements BinaryTypeIdentity { + /** {@inheritDoc} */ + @Override public int hash(BinaryObject obj) { + A.notNull(obj, "Can't compute hash code for null object"); + + assert obj instanceof BinaryObjectExImpl; + + BinaryObjectExImpl exObj = (BinaryObjectExImpl) obj; + + BinarySchema schema = exObj.createSchema(); + + int hash = 0; + + for (int fieldId : schema.fieldIdsSorted()) { + Object val = exObj.field(fieldId); + + hash = 31 * hash + (val != null ? val.hashCode() : 0); + } + + return hash; + } + + /** {@inheritDoc} */ + @Override public boolean equals(BinaryObject o1, BinaryObject o2) { + if (o1 == o2) + return true; + + if (o1 == null || o2 == null) + return false; + + if (!(o1 instanceof BinaryObjectExImpl) || !(o2 instanceof BinaryObjectExImpl)) + return false; + + BinaryObjectExImpl exObj1 = (BinaryObjectExImpl) o1; + + BinarySchema schema1 = exObj1.createSchema(); + + BinaryObjectExImpl exObj2 = (BinaryObjectExImpl) o2; + + for (int fldId : schema1.fieldIdsSorted()) { + if (!F.eq(exObj1.field(fldId), exObj2.field(fldId))) + return false; + } + + return true; + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java index 9e93818f30587..32e95f7cc79cf 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java @@ -18,14 +18,14 @@ package org.apache.ignite.internal.binary.builder; import org.apache.ignite.binary.BinaryInvalidTypeException; -import org.apache.ignite.binary.BinaryKeyHashingMode; import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.binary.BinaryObjectBuilder; import org.apache.ignite.binary.BinaryObjectException; -import org.apache.ignite.binary.BinaryObjectHashCodeResolver; +import org.apache.ignite.binary.BinaryTypeIdentity; import org.apache.ignite.binary.BinaryType; import org.apache.ignite.internal.binary.BinaryMetadata; import org.apache.ignite.internal.binary.BinaryObjectImpl; +import org.apache.ignite.internal.binary.BinaryPrimitives; import org.apache.ignite.internal.binary.BinaryWriterExImpl; import org.apache.ignite.internal.binary.GridBinaryMarshaller; import org.apache.ignite.internal.binary.BinaryContext; @@ -38,7 +38,6 @@ import org.apache.ignite.lang.IgniteBiTuple; import org.jetbrains.annotations.Nullable; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -189,7 +188,23 @@ public BinaryObjectBuilderImpl(BinaryObjectImpl obj) { byte[] arr = writer.array(); - return new BinaryObjectImpl(ctx, arr, 0); + BinaryObject res = new BinaryObjectImpl(ctx, arr, 0); + + short flags = BinaryPrimitives.readShort(arr, GridBinaryMarshaller.FLAGS_POS); + + if (BinaryUtils.isFlagSet(flags, BinaryUtils.FLAG_EMPTY_HASH_CODE)) { + BinaryTypeIdentity identity = ctx.identity(typeId()); + + if (identity != null) { + // Reset missing hash code flag + BinaryPrimitives.writeShort(arr, GridBinaryMarshaller.FLAGS_POS, + (short) (flags & ~BinaryUtils.FLAG_EMPTY_HASH_CODE)); + + BinaryPrimitives.writeInt(arr, GridBinaryMarshaller.HASH_CODE_POS, identity.hash(res)); + } + } + + return res; } } @@ -338,41 +353,6 @@ else if (readCache == null) { reader.position(start + BinaryUtils.length(reader, start)); } - // Let us not mess with hash codes of objects based on other objects, m'kay? - if (reader == null && !isHashCodeSet) { - BinaryKeyHashingMode hashingMode = ctx.keyHashingMode(typeId()); - - if (hashingMode != null) { - switch (hashingMode) { - case DEFAULT: - // No-op. - break; - - case BYTES_HASH: - // Not considering #start as we're dealing only with #reader free cases. - // At this point, no footer has been written yet, so we are looking at everything - // past the header and down to the end of all data. - byte[] data = Arrays.copyOfRange(writer.array(), hdrLen, writer.out().position()); - - hashCode(Arrays.hashCode(data)); - break; - - case FIELDS_HASH: - case CUSTOM: - BinaryObjectHashCodeResolver rslvr = ctx.hashCodeResolver(typeId()); - - assert rslvr != null; - - hashCode(rslvr.hash(this)); - break; - - default: - throw new BinaryObjectException("Unexpected hashing mode [typeId=" + typeId() + ", typeName=" + - typeName + ']'); - } - } - } - //noinspection NumberEquality writer.postWrite(true, registeredType, hashCode, isHashCodeSet); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java index 53170f9d51d28..c96b82ed023e0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java @@ -38,12 +38,11 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteBinary; import org.apache.ignite.IgniteCache; -import org.apache.ignite.binary.BinaryKeyHashingMode; import org.apache.ignite.binary.BinaryNameMapper; import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.binary.BinaryObjectBuilder; import org.apache.ignite.binary.BinaryObjectException; -import org.apache.ignite.binary.BinaryObjectHashCodeResolver; +import org.apache.ignite.binary.BinaryTypeIdentity; import org.apache.ignite.binary.BinaryReader; import org.apache.ignite.binary.BinaryTypeConfiguration; import org.apache.ignite.binary.BinaryWriter; @@ -60,9 +59,10 @@ import org.apache.ignite.internal.IgniteKernal; import org.apache.ignite.internal.binary.BinaryContext; import org.apache.ignite.internal.binary.BinaryMarshaller; -import org.apache.ignite.internal.binary.BinaryObjectExImpl; import org.apache.ignite.internal.binary.BinaryObjectImpl; import org.apache.ignite.internal.binary.BinaryObjectOffheapImpl; +import org.apache.ignite.internal.binary.FieldsListIdentity; +import org.apache.ignite.internal.binary.FullIdentity; import org.apache.ignite.internal.processors.cache.GridCacheAdapter; import org.apache.ignite.internal.processors.cache.GridCacheEntryEx; import org.apache.ignite.internal.processors.cache.IgniteCacheProxy; @@ -129,15 +129,28 @@ public abstract class GridCacheBinaryObjectsAbstractSelfTest extends GridCommonA List binTypes = new ArrayList<>(); binTypes.add(new BinaryTypeConfiguration() {{ - setTypeName("BytesHashedKey"); + setTypeName("FullyHashedKey"); + setIdentity(new FullIdentity()); + }}); + + binTypes.add(new BinaryTypeConfiguration() {{ + setTypeName("FullyHashedKeyField"); + setIdentity(new FullIdentity()); }}); binTypes.add(new BinaryTypeConfiguration() {{ setTypeName("FieldsHashedKey"); + + FieldsListIdentity id = new FieldsListIdentity(); + id.setFieldNames(Arrays.asList("fld1", "fld3")); + + setIdentity(id); }}); binTypes.add(new BinaryTypeConfiguration() {{ setTypeName("CustomHashedKey"); + + setIdentity(new Identity()); }}); BinaryConfiguration binCfg = new BinaryConfiguration(); @@ -145,21 +158,11 @@ public abstract class GridCacheBinaryObjectsAbstractSelfTest extends GridCommonA cfg.setBinaryConfiguration(binCfg); - CacheKeyConfiguration arrHashConfiguration = new CacheKeyConfiguration("BytesHashedKey", "fld1") {{ - setBinaryHashingMode(BinaryKeyHashingMode.BYTES_HASH); - }}; - - CacheKeyConfiguration fieldsHashConfiguration = new CacheKeyConfiguration("FieldsHashedKey", "fld1") {{ - setBinaryHashingMode(BinaryKeyHashingMode.FIELDS_HASH); + CacheKeyConfiguration arrHashConfiguration = new CacheKeyConfiguration("FullyHashedKey", "fld1"); - setBinaryHashCodeFields(Arrays.asList("fld1", "fld3")); - }}; + CacheKeyConfiguration fieldsHashConfiguration = new CacheKeyConfiguration("FieldsHashedKey", "fld1"); - CacheKeyConfiguration customHashConfiguration = new CacheKeyConfiguration("CustomHashedKey", "fld1") {{ - setBinaryHashingMode(BinaryKeyHashingMode.CUSTOM); - - setBinHashCodeResolverClassName(HashCodeResolver.class.getName()); - }}; + CacheKeyConfiguration customHashConfiguration = new CacheKeyConfiguration("CustomHashedKey", "fld1"); cfg.setCacheKeyConfiguration(arrHashConfiguration, fieldsHashConfiguration, customHashConfiguration); @@ -990,15 +993,22 @@ public void testPutWithoutHashCode() throws Exception { * */ @SuppressWarnings("unchecked") - public void testPutWithArrayHashing() { + public void testPutWithFullHashing() { IgniteCache c = binKeysCache(); { - BinaryObjectBuilder bldr = grid(0).binary().builder("BytesHashedKey"); + BinaryObjectBuilder bldr = grid(0).binary().builder("FullyHashedKey"); bldr.setField("fld1", 5); bldr.setField("fld2", "abc"); + BinaryObjectBuilder bldr2 = grid(0).binary().builder("FullyHashedKeyField"); + + bldr2.setField("a", "x"); + bldr2.setField("b", 345L); + + bldr.setField("fld3", bldr2.build()); + BinaryObject binKey = bldr.build(); c.put(binKey, "zzz"); @@ -1006,11 +1016,18 @@ public void testPutWithArrayHashing() { // Now let's build an identical key for get { - BinaryObjectBuilder bldr = grid(0).binary().builder("BytesHashedKey"); + BinaryObjectBuilder bldr = grid(0).binary().builder("FullyHashedKey"); bldr.setField("fld1", 5); bldr.setField("fld2", "abc"); + BinaryObjectBuilder bldr2 = grid(0).binary().builder("FullyHashedKeyField"); + + bldr2.setField("a", "x"); + bldr2.setField("b", 345L); + + bldr.setField("fld3", bldr2.build()); + BinaryObject binKey = bldr.build(); assertEquals("zzz", c.get(binKey)); @@ -1390,14 +1407,14 @@ private static class TestStore extends CacheStoreAdapter { /** * */ - private final static class HashCodeResolver implements BinaryObjectHashCodeResolver { + private final static class Identity implements BinaryTypeIdentity { /** {@inheritDoc} */ - @Override public int hash(BinaryObjectBuilder builder) { - return (Integer) builder.getField("fld1") * 31 / 5; + @Override public int hash(BinaryObject builder) { + return (Integer) builder.field("fld1") * 31 / 5; } /** {@inheritDoc} */ - @Override public boolean equals(BinaryObjectExImpl o1, BinaryObjectExImpl o2) { + @Override public boolean equals(BinaryObject o1, BinaryObject o2) { return o1 == o2 || (o1 != null && o2 != null && F.eq(o1.field("fld1"), o2.field("fld1"))); } From 774a802a62abe3a652f4de3c40b115661a700df9 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Tue, 11 Oct 2016 18:48:26 +0300 Subject: [PATCH 4/9] IGNITE-4011 More optimal identities --- .../ignite/internal/binary/BinaryContext.java | 13 ++++++++++++ .../internal/binary/BinaryFieldImpl.java | 1 - .../internal/binary/FieldsListIdentity.java | 20 ++++++++++--------- .../ignite/internal/binary/FullIdentity.java | 13 ++++++++++-- 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java index fd0dc162f52df..f33ffd1986dac 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java @@ -1215,6 +1215,19 @@ public BinaryFieldImpl createField(int typeId, String fieldName) { return new BinaryFieldImpl(typeId, schemaReg, fieldName, fieldId); } + /** + * Create binary field. + * + * @param typeId Type ID. + * @param fieldId Field id. + * @return Binary field. + */ + public BinaryFieldImpl createField(int typeId, int fieldId) { + BinarySchemaRegistry schemaReg = schemaRegistry(typeId); + + return new BinaryFieldImpl(typeId, schemaReg, null, fieldId); + } + /** * @param typeId Type ID. * @return Meta data. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java index 78ed17a5054c7..532262ae00ff3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java @@ -50,7 +50,6 @@ public class BinaryFieldImpl implements BinaryField { public BinaryFieldImpl(int typeId, BinarySchemaRegistry schemas, String fieldName, int fieldId) { assert typeId != 0; assert schemas != null; - assert fieldName != null; assert fieldId != 0; this.typeId = typeId; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/FieldsListIdentity.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/FieldsListIdentity.java index 0622933d7689c..847eac60f4d03 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/FieldsListIdentity.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/FieldsListIdentity.java @@ -38,13 +38,10 @@ public final class FieldsListIdentity implements BinaryTypeIdentity { BinaryObjectExImpl exObj = (BinaryObjectExImpl) obj; - // Let's create only one reader inside builder instead of new one on each call of field() - BinaryObjectBuilderImpl bldr = (BinaryObjectBuilderImpl) exObj.toBuilder(); - int hash = 0; for (String fieldName : fieldNames) { - Object val = bldr.getField(fieldName); + Object val = fieldValue(exObj, fieldName); hash = 31 * hash + (val != null ? val.hashCode() : 0); } @@ -65,14 +62,10 @@ public final class FieldsListIdentity implements BinaryTypeIdentity { BinaryObjectExImpl exObj1 = (BinaryObjectExImpl) o1; - BinaryObjectBuilderImpl bldr1 = (BinaryObjectBuilderImpl) exObj1.toBuilder(); - BinaryObjectExImpl exObj2 = (BinaryObjectExImpl) o2; - BinaryObjectBuilderImpl bldr2 = (BinaryObjectBuilderImpl) exObj2.toBuilder(); - for (String fld : fieldNames) { - if (!F.eq(bldr1.getField(fld), bldr2.getField(fld))) + if (!F.eq(fieldValue(exObj1, fld), fieldValue(exObj2, fld))) return false; } @@ -92,4 +85,13 @@ public List getFieldNames() { public void setFieldNames(List fieldNames) { this.fieldNames = fieldNames; } + + /** + * @param exObj Object to get the field value from. + * @param fieldName Field id. + * @return Field value. + */ + private static Object fieldValue(BinaryObjectExImpl exObj, String fieldName) { + return exObj.context().createField(exObj.typeId(), fieldName).value(exObj); + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/FullIdentity.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/FullIdentity.java index 6b6f16ab27c58..58eef180532b5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/binary/FullIdentity.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/FullIdentity.java @@ -43,7 +43,7 @@ public class FullIdentity implements BinaryTypeIdentity { int hash = 0; for (int fieldId : schema.fieldIdsSorted()) { - Object val = exObj.field(fieldId); + Object val = exObj.context().createField(exObj.typeId(), fieldId).value(obj); hash = 31 * hash + (val != null ? val.hashCode() : 0); } @@ -69,10 +69,19 @@ public class FullIdentity implements BinaryTypeIdentity { BinaryObjectExImpl exObj2 = (BinaryObjectExImpl) o2; for (int fldId : schema1.fieldIdsSorted()) { - if (!F.eq(exObj1.field(fldId), exObj2.field(fldId))) + if (!F.eq(fieldValue(exObj1, fldId), fieldValue(exObj2, fldId))) return false; } return true; } + + /** + * @param exObj Object to get the field value from. + * @param fieldId Field id. + * @return Field value. + */ + private static Object fieldValue(BinaryObjectExImpl exObj, int fieldId) { + return exObj.context().createField(exObj.typeId(), fieldId).value(exObj); + } } From cea9f865702969ad930136356d36289aaaa9fe14 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Wed, 12 Oct 2016 17:22:40 +0300 Subject: [PATCH 5/9] IGNITE-4011 Binary identity benchmarks. --- .../jmh/cache/JmhCacheAbstractBenchmark.java | 2 +- .../jmh/cache/JmhCacheBenchmark.java | 20 ++-- .../jmh/cache/JmhCacheBinKeysBenchmark.java | 98 +++++++++++++++++++ .../JmhCacheBinKeysFullIdentityBenchmark.java | 32 ++++++ .../jmh/runner/JmhIdeBenchmarkRunner.java | 2 + 5 files changed, 145 insertions(+), 9 deletions(-) create mode 100644 modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhCacheBinKeysBenchmark.java create mode 100644 modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhCacheBinKeysFullIdentityBenchmark.java diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhCacheAbstractBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhCacheAbstractBenchmark.java index 709ab777ff42d..7261fdc5e8c1f 100644 --- a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhCacheAbstractBenchmark.java +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhCacheAbstractBenchmark.java @@ -116,7 +116,7 @@ public void setup() throws Exception { node = Ignition.start(clientCfg); } - cache = node.cache(null); + cache = node.cache(null).withKeepBinary(); } /** diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhCacheBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhCacheBenchmark.java index f55d16cfe5358..22013f2f1c86a 100644 --- a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhCacheBenchmark.java +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhCacheBenchmark.java @@ -37,14 +37,13 @@ public class JmhCacheBenchmark extends JmhCacheAbstractBenchmark { * * @throws Exception If failed. */ - public void setup() throws Exception { super.setup(); - IgniteDataStreamer dataLdr = node.dataStreamer(cache.getName()); + IgniteDataStreamer dataLdr = node.dataStreamer(cache.getName()); for (int i = 0; i < CNT; i++) - dataLdr.addData(i, new IntValue(i)); + dataLdr.addData(createKey(i), new IntValue(i)); dataLdr.close(); @@ -60,7 +59,7 @@ public void setup() throws Exception { public void put() throws Exception { int key = ThreadLocalRandom.current().nextInt(CNT); - cache.put(key, new IntValue(key)); + cache.put(createKey(key), new IntValue(key)); } /** @@ -72,7 +71,7 @@ public void put() throws Exception { public Object get() throws Exception { int key = ThreadLocalRandom.current().nextInt(CNT); - return cache.get(key); + return cache.get(createKey(key)); } /** @@ -133,13 +132,18 @@ private static void run(String benchmark, int threads, boolean client, CacheAtom .jvmArguments( "-Xms4g", "-Xmx4g", - "-XX:+UnlockCommercialFeatures", - "-XX:+FlightRecorder", - "-XX:StartFlightRecording=delay=30s,dumponexit=true,settings=alloc,filename=" + output + ".jfr", JmhIdeBenchmarkRunner.createProperty(PROP_ATOMICITY_MODE, atomicityMode), JmhIdeBenchmarkRunner.createProperty(PROP_WRITE_SYNC_MODE, writeSyncMode), JmhIdeBenchmarkRunner.createProperty(PROP_DATA_NODES, 2), JmhIdeBenchmarkRunner.createProperty(PROP_CLIENT_MODE, client)) .run(); } + + /** + * @param key int key. + * @return Key object. + */ + protected Object createKey(int key) { + return key; + } } diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhCacheBinKeysBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhCacheBinKeysBenchmark.java new file mode 100644 index 0000000000000..83b9260a286b8 --- /dev/null +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhCacheBinKeysBenchmark.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.benchmarks.jmh.cache; + +import java.util.Collections; +import org.apache.ignite.binary.BinaryObject; +import org.apache.ignite.binary.BinaryObjectBuilder; +import org.apache.ignite.binary.BinaryTypeConfiguration; +import org.apache.ignite.binary.BinaryTypeIdentity; +import org.apache.ignite.configuration.BinaryConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.binary.BinaryMarshaller; + +/** + * + */ +@SuppressWarnings("unchecked") +public class JmhCacheBinKeysBenchmark extends JmhCacheBenchmark { + /** */ + private BinaryTypeIdentity keyIdentity; + + /** {@inheritDoc} */ + @Override public void setup() throws Exception { + keyIdentity = identity(); + + super.setup(); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration configuration(String gridName) { + IgniteConfiguration cfg = super.configuration(gridName); + + cfg.setMarshaller(new BinaryMarshaller()); + + BinaryConfiguration binCfg = new BinaryConfiguration(); + + binCfg.setTypeConfigurations(Collections.singletonList( + new BinaryTypeConfiguration() {{ + setTypeName("Int3FieldsKey"); + + setIdentity(keyIdentity); + }} + )); + + cfg.setBinaryConfiguration(binCfg); + + return cfg; + } + + /** + * @return New identity to hash/compare keys with, {@code null} for default behavior. + */ + protected BinaryTypeIdentity identity() { + return null; + } + + /** + * @param k Base value to initialize key fields upon. + * @return Binary key having 3 fields. + */ + @SuppressWarnings("ResultOfMethodCallIgnored") + @Override protected BinaryObject createKey(int k) { + BinaryObjectBuilder bldr = node.binary().builder("Int3FieldsKey"); + + Long l = (long) k; + + bldr.setField("f1", 1); + bldr.setField("f2", "suffix"); + bldr.setField("f3", l); + + if (keyIdentity == null) { + int hash = 0; + + hash = 31 * hash + 1; + hash = 31 * hash + ("suffix".hashCode()); + hash = 31 * hash + l.hashCode(); + + bldr.hashCode(hash); + } + + return bldr.build(); + } +} diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhCacheBinKeysFullIdentityBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhCacheBinKeysFullIdentityBenchmark.java new file mode 100644 index 0000000000000..e82583a36de7c --- /dev/null +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/cache/JmhCacheBinKeysFullIdentityBenchmark.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.benchmarks.jmh.cache; + +import org.apache.ignite.binary.BinaryTypeIdentity; +import org.apache.ignite.internal.binary.FullIdentity; + +/** + * + */ +@SuppressWarnings("unchecked") +public class JmhCacheBinKeysFullIdentityBenchmark extends JmhCacheBinKeysBenchmark { + /** {@inheritDoc} */ + @Override protected BinaryTypeIdentity identity() { + return new FullIdentity(); + } +} diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/runner/JmhIdeBenchmarkRunner.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/runner/JmhIdeBenchmarkRunner.java index 0cad0883460b2..1ea602e9c1ecc 100644 --- a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/runner/JmhIdeBenchmarkRunner.java +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/runner/JmhIdeBenchmarkRunner.java @@ -186,6 +186,8 @@ public OptionsBuilder optionsBuilder() { builder.timeUnit(outputTimeUnit); builder.threads(threads); + builder.shouldFailOnError(true); + if (benchmarkModes != null) { for (Mode benchmarkMode : benchmarkModes) builder.getBenchModes().add(benchmarkMode); From 9fa7d0680b772e3cde87e2d48631647f79d0167a Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Wed, 12 Oct 2016 19:27:53 +0300 Subject: [PATCH 6/9] IGNITE-4011 Binary identity benchmarks for Yardstick. --- .../config/benchmark-bin-identity.properties | 94 +++++++++++++++ .../config/ignite-bin-multicast-config.xml | 78 +++++++++++++ .../cache/IgniteBinaryIdentityBenchmark.java | 108 ++++++++++++++++++ .../IgniteBinaryIdentityGetBenchmark.java | 34 ++++++ .../IgniteBinaryIdentityPutBenchmark.java | 35 ++++++ .../IgniteFullBinaryIdentityGetBenchmark.java | 30 +++++ .../IgniteFullBinaryIdentityPutBenchmark.java | 30 +++++ ...gniteLegacyBinaryIdentityGetBenchmark.java | 30 +++++ ...gniteLegacyBinaryIdentityPutBenchmark.java | 30 +++++ 9 files changed, 469 insertions(+) create mode 100644 modules/yardstick/config/benchmark-bin-identity.properties create mode 100644 modules/yardstick/config/ignite-bin-multicast-config.xml create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteBinaryIdentityBenchmark.java create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteBinaryIdentityGetBenchmark.java create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteBinaryIdentityPutBenchmark.java create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteFullBinaryIdentityGetBenchmark.java create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteFullBinaryIdentityPutBenchmark.java create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteLegacyBinaryIdentityGetBenchmark.java create mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteLegacyBinaryIdentityPutBenchmark.java diff --git a/modules/yardstick/config/benchmark-bin-identity.properties b/modules/yardstick/config/benchmark-bin-identity.properties new file mode 100644 index 0000000000000..0b7857f838ad1 --- /dev/null +++ b/modules/yardstick/config/benchmark-bin-identity.properties @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Contains all multicast benchmarks +# + +now0=`date +'%H%M%S'` + +# JVM options. +JVM_OPTS=${JVM_OPTS}" -DIGNITE_QUIET=false" + +# Uncomment to enable concurrent garbage collection (GC) if you encounter long GC pauses. +JVM_OPTS=${JVM_OPTS}" \ +-Xloggc:./gc${now0}.log \ +-XX:+PrintGCDetails \ +-verbose:gc \ +-XX:+UseParNewGC \ +-XX:+UseConcMarkSweepGC \ +-XX:+UseTLAB \ +-XX:NewSize=128m \ +-XX:MaxNewSize=128m \ +-XX:MaxTenuringThreshold=0 \ +-XX:SurvivorRatio=1024 \ +-XX:+UseCMSInitiatingOccupancyOnly \ +-XX:CMSInitiatingOccupancyFraction=60 \ +" + +#Ignite version +ver="RELEASE-" + +# List of default probes. +# Add DStatProbe or VmStatProbe if your OS supports it (e.g. if running on Linux). +BENCHMARK_DEFAULT_PROBES=ThroughputLatencyProbe,PercentileProbe,DStatProbe + +# Packages where the specified benchmark is searched by reflection mechanism. +BENCHMARK_PACKAGES=org.yardstickframework,org.apache.ignite.yardstick + +# Restart servers for each benchmark. +RESTART_SERVERS=true + +# Probe point writer class name. +# BENCHMARK_WRITER= + +# Comma-separated list of the hosts to run BenchmarkServers on. +SERVER_HOSTS=localhost,localhost,localhost + +# Comma-separated list of the hosts to run BenchmarkDrivers on. +DRIVER_HOSTS=localhost + +# Remote username. +# REMOTE_USER= + +# Number of nodes, used to wait for the specified number of nodes to start. +nodesNum=$((`echo ${SERVER_HOSTS} | tr ',' '\n' | wc -l` + `echo ${DRIVER_HOSTS} | tr ',' '\n' | wc -l`)) + +# Backups count. +b=1 + +# Warmup. +w=60 + +# Duration. +d=60 + +# Threads count. +t=64 + +# Sync mode. +sm=PRIMARY_SYNC + +# Jobs. +j=10 + +# Run configuration which contains all benchmarks. +# Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute). +CONFIGS="\ +-cfg ${SCRIPT_DIR}/../config/ignite-bin-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteLegacyBinaryIdentityGetBenchmark -sn IgniteNode -ds ${ver}legacy-bin-id-atomic-get-1-backup,\ +-cfg ${SCRIPT_DIR}/../config/ignite-bin-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteLegacyBinaryIdentityPutBenchmark -sn IgniteNode -ds ${ver}legacy-bin-id-atomic-put-1-backup,\ +-cfg ${SCRIPT_DIR}/../config/ignite-bin-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteFullBinaryIdentityGetBenchmark -sn IgniteNode -ds ${ver}full-bin-id-atomic-get-1-backup,\ +-cfg ${SCRIPT_DIR}/../config/ignite-bin-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteFullBinaryIdentityPutBenchmark -sn IgniteNode -ds ${ver}full-bin-id-atomic-put-1-backup\ +" diff --git a/modules/yardstick/config/ignite-bin-multicast-config.xml b/modules/yardstick/config/ignite-bin-multicast-config.xml new file mode 100644 index 0000000000000..7efa1950833d9 --- /dev/null +++ b/modules/yardstick/config/ignite-bin-multicast-config.xml @@ -0,0 +1,78 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteBinaryIdentityBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteBinaryIdentityBenchmark.java new file mode 100644 index 0000000000000..6fb53b73ad676 --- /dev/null +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteBinaryIdentityBenchmark.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.yardstick.cache; + +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.binary.BinaryObject; +import org.apache.ignite.binary.BinaryObjectBuilder; +import org.apache.ignite.yardstick.cache.model.SampleValue; +import org.yardstickframework.BenchmarkConfiguration; + +import static org.yardstickframework.BenchmarkUtils.println; + +/** + * Ignite benchmark that performs get operations. + */ +abstract class IgniteBinaryIdentityBenchmark extends IgniteCacheAbstractBenchmark { + /** {@inheritDoc} */ + @Override public void setUp(BenchmarkConfiguration cfg) throws Exception { + super.setUp(cfg); + + println(cfg, "Populating data..."); + + long start = System.nanoTime(); + + try (IgniteDataStreamer dataLdr = ignite().dataStreamer(cache.getName())) { + for (int i = 0; i < args.range() && !Thread.currentThread().isInterrupted();) { + dataLdr.addData(createKey(i), new SampleValue(i)); + + if (++i % 100000 == 0) + println(cfg, "Items populated: " + i); + } + } + + println(cfg, "Finished populating data in " + ((System.nanoTime() - start) / 1_000_000) + " ms."); + } + + /** {@inheritDoc} */ + @Override protected IgniteCache cache() { + return ignite().cache("atomic").withKeepBinary(); + } + + /** + * @param key Base key value. + * @return Binary key. + */ + abstract BinaryObject createKey(int key); + + /** + * @param key Key field value. + * @return Binary object without hash code explicitly set at build time. + */ + BinaryObject createFullIdentityBinaryKey(int key) { + BinaryObjectBuilder bldr = ignite().binary().builder("BinaryKeyWithFullIdentity"); + + setBuilderFields(bldr, key); + + return bldr.build(); + } + + /** + * @param key Key field value. + * @return Binary object with hash code explicitly set at build time. + */ + @SuppressWarnings("ResultOfMethodCallIgnored") + BinaryObject createLegacyIdentityBinaryKey(int key) { + BinaryObjectBuilder bldr = ignite().binary().builder("BinaryKeyWithLegacyIdentity"); + + setBuilderFields(bldr, key); + + int hash = 0; + + hash = 31 * hash + 1; + hash = 31 * hash + ("SomeString".hashCode()); + hash = 31 * hash + Long.valueOf(key).hashCode(); + + bldr.hashCode(hash); + + return bldr.build(); + } + + /** + * @param builder Builder. + * @param key Key field value. + */ + private static void setBuilderFields(BinaryObjectBuilder builder, int key) { + builder.setField("f1", 1); + + builder.setField("f2", "SomeString"); + + builder.setField("f3", (long) key); + } +} diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteBinaryIdentityGetBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteBinaryIdentityGetBenchmark.java new file mode 100644 index 0000000000000..1af1846c270bb --- /dev/null +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteBinaryIdentityGetBenchmark.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.yardstick.cache; + +import java.util.Map; + +/** + * Test GETs with binary hashed key. + */ +public abstract class IgniteBinaryIdentityGetBenchmark extends IgniteBinaryIdentityBenchmark { + /** {@inheritDoc} */ + @Override public boolean test(Map ctx) throws Exception { + int key = nextRandom(args.range()); + + cache.get(createKey(key)); + + return true; + } +} diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteBinaryIdentityPutBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteBinaryIdentityPutBenchmark.java new file mode 100644 index 0000000000000..58895a0afac31 --- /dev/null +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteBinaryIdentityPutBenchmark.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.yardstick.cache; + +import java.util.Map; +import org.apache.ignite.yardstick.cache.model.SampleValue; + +/** + * Test PUTs with binary hashed key. + */ +public abstract class IgniteBinaryIdentityPutBenchmark extends IgniteBinaryIdentityBenchmark { + /** {@inheritDoc} */ + @Override public boolean test(Map ctx) throws Exception { + int key = nextRandom(args.range()); + + cache.put(createKey(key), new SampleValue(key)); + + return true; + } +} diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteFullBinaryIdentityGetBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteFullBinaryIdentityGetBenchmark.java new file mode 100644 index 0000000000000..2ad3b1f59b511 --- /dev/null +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteFullBinaryIdentityGetBenchmark.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.yardstick.cache; + +import org.apache.ignite.binary.BinaryObject; + +/** + * Test GETs with binary hashed key. + */ +public class IgniteFullBinaryIdentityGetBenchmark extends IgniteBinaryIdentityGetBenchmark { + /** {@inheritDoc} */ + @Override BinaryObject createKey(int key) { + return createFullIdentityBinaryKey(key); + } +} diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteFullBinaryIdentityPutBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteFullBinaryIdentityPutBenchmark.java new file mode 100644 index 0000000000000..db6ed25b6cfd6 --- /dev/null +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteFullBinaryIdentityPutBenchmark.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.yardstick.cache; + +import org.apache.ignite.binary.BinaryObject; + +/** + * Test PUTs with binary hashed key. + */ +public class IgniteFullBinaryIdentityPutBenchmark extends IgniteBinaryIdentityPutBenchmark { + /** {@inheritDoc} */ + @Override BinaryObject createKey(int key) { + return createFullIdentityBinaryKey(key); + } +} diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteLegacyBinaryIdentityGetBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteLegacyBinaryIdentityGetBenchmark.java new file mode 100644 index 0000000000000..1b36b246dbdad --- /dev/null +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteLegacyBinaryIdentityGetBenchmark.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.yardstick.cache; + +import org.apache.ignite.binary.BinaryObject; + +/** + * Test GETs with binary hashed key. + */ +public class IgniteLegacyBinaryIdentityGetBenchmark extends IgniteBinaryIdentityGetBenchmark { + /** {@inheritDoc} */ + @Override BinaryObject createKey(int key) { + return createLegacyIdentityBinaryKey(key); + } +} diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteLegacyBinaryIdentityPutBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteLegacyBinaryIdentityPutBenchmark.java new file mode 100644 index 0000000000000..246fc28a27358 --- /dev/null +++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/IgniteLegacyBinaryIdentityPutBenchmark.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.yardstick.cache; + +import org.apache.ignite.binary.BinaryObject; + +/** + * Test PUTs with binary hashed key. + */ +public class IgniteLegacyBinaryIdentityPutBenchmark extends IgniteBinaryIdentityPutBenchmark { + /** {@inheritDoc} */ + @Override BinaryObject createKey(int key) { + return createLegacyIdentityBinaryKey(key); + } +} From 696141f95b9bff668461d9853d8fd058b0b8a9dd Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Thu, 13 Oct 2016 14:53:22 +0300 Subject: [PATCH 7/9] IGNITE-4011 Binary identity benchmarks for Yardstick - configs fixed. --- .../config/benchmark-bin-identity.properties | 8 ++++---- .../config/ignite-bin-multicast-config.xml | 20 +++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/modules/yardstick/config/benchmark-bin-identity.properties b/modules/yardstick/config/benchmark-bin-identity.properties index 0b7857f838ad1..da0c2a7b69066 100644 --- a/modules/yardstick/config/benchmark-bin-identity.properties +++ b/modules/yardstick/config/benchmark-bin-identity.properties @@ -73,7 +73,7 @@ b=1 w=60 # Duration. -d=60 +d=300 # Threads count. t=64 @@ -87,8 +87,8 @@ j=10 # Run configuration which contains all benchmarks. # Note that each benchmark is set to run for 300 seconds (5 mins) with warm-up set to 60 seconds (1 minute). CONFIGS="\ --cfg ${SCRIPT_DIR}/../config/ignite-bin-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteLegacyBinaryIdentityGetBenchmark -sn IgniteNode -ds ${ver}legacy-bin-id-atomic-get-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-bin-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteLegacyBinaryIdentityPutBenchmark -sn IgniteNode -ds ${ver}legacy-bin-id-atomic-put-1-backup,\ -cfg ${SCRIPT_DIR}/../config/ignite-bin-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteFullBinaryIdentityGetBenchmark -sn IgniteNode -ds ${ver}full-bin-id-atomic-get-1-backup,\ --cfg ${SCRIPT_DIR}/../config/ignite-bin-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteFullBinaryIdentityPutBenchmark -sn IgniteNode -ds ${ver}full-bin-id-atomic-put-1-backup\ +-cfg ${SCRIPT_DIR}/../config/ignite-bin-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteFullBinaryIdentityPutBenchmark -sn IgniteNode -ds ${ver}full-bin-id-atomic-put-1-backup,\ +-cfg ${SCRIPT_DIR}/../config/ignite-bin-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteLegacyBinaryIdentityGetBenchmark -sn IgniteNode -ds ${ver}legacy-bin-id-atomic-get-1-backup,\ +-cfg ${SCRIPT_DIR}/../config/ignite-bin-multicast-config.xml -nn ${nodesNum} -b ${b} -w ${w} -d ${d} -t ${t} -sm ${sm} -dn IgniteLegacyBinaryIdentityPutBenchmark -sn IgniteNode -ds ${ver}legacy-bin-id-atomic-put-1-backup\ " diff --git a/modules/yardstick/config/ignite-bin-multicast-config.xml b/modules/yardstick/config/ignite-bin-multicast-config.xml index 7efa1950833d9..ff05f5bd50390 100644 --- a/modules/yardstick/config/ignite-bin-multicast-config.xml +++ b/modules/yardstick/config/ignite-bin-multicast-config.xml @@ -24,9 +24,17 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation=" http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.5.xsd"> - + + + + + + + + + + - @@ -66,13 +74,5 @@ - - - - - - - - From e8c0a01f1de2fddebe8b246fc52905ae3a2dcb47 Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Thu, 13 Oct 2016 17:24:29 +0300 Subject: [PATCH 8/9] Fixed comment --- .../main/java/org/apache/ignite/binary/BinaryTypeIdentity.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/binary/BinaryTypeIdentity.java b/modules/core/src/main/java/org/apache/ignite/binary/BinaryTypeIdentity.java index d7d252ea4bb72..86713248d1fdf 100644 --- a/modules/core/src/main/java/org/apache/ignite/binary/BinaryTypeIdentity.java +++ b/modules/core/src/main/java/org/apache/ignite/binary/BinaryTypeIdentity.java @@ -18,7 +18,7 @@ package org.apache.ignite.binary; /** - * Method to compute hash codes for new binary objects. + * Method to compute hash codes for new binary objects and compare them for equality. */ public interface BinaryTypeIdentity { /** From bafb7684074a589476526d78a5128054d6ac367f Mon Sep 17 00:00:00 2001 From: Alexander Paschenko Date: Fri, 14 Oct 2016 17:54:06 +0300 Subject: [PATCH 9/9] IGNITE-4011 Added test --- ...ridCacheBinaryObjectsAbstractSelfTest.java | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java index c96b82ed023e0..27082404c2a3b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheBinaryObjectsAbstractSelfTest.java @@ -153,6 +153,15 @@ public abstract class GridCacheBinaryObjectsAbstractSelfTest extends GridCommonA setIdentity(new Identity()); }}); + binTypes.add(new BinaryTypeConfiguration() {{ + setTypeName("ComplexBinaryFieldsListHashedKey"); + + FieldsListIdentity id = new FieldsListIdentity(); + id.setFieldNames(Arrays.asList("secondField", "thirdField")); + + setIdentity(id); + }}); + BinaryConfiguration binCfg = new BinaryConfiguration(); binCfg.setTypeConfigurations(binTypes); @@ -1034,6 +1043,27 @@ public void testPutWithFullHashing() { } } + /** + * + */ + @SuppressWarnings("unchecked") + public void testCrossFormatObjectsIdentity() { + IgniteCache c = binKeysCache(); + + c.put(new ComplexKey(), "zzz"); + + // Now let's build an identical key for get + BinaryObjectBuilder bldr = grid(0).binary().builder("ComplexBinaryFieldsListHashedKey"); + + bldr.setField("firstField", 365); + bldr.setField("secondField", "value"); + bldr.setField("thirdField", 0x1020304050607080L); + + BinaryObject binKey = bldr.build(); + + assertEquals("zzz", c.get(binKey)); + } + /** * */ @@ -1419,4 +1449,37 @@ private final static class Identity implements BinaryTypeIdentity { } } + + /** + * Key to test puts and gets with + */ + @SuppressWarnings({"ConstantConditions", "unused"}) + private final static class ComplexKey { + /** */ + private final Integer firstField = 1; + + /** */ + private final String secondField = "value"; + + /** */ + private final Long thirdField = 0x1020304050607080L; + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ComplexKey that = (ComplexKey) o; + + return secondField.equals(that.secondField) && + thirdField.equals(that.thirdField); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + int res = secondField.hashCode(); + res = 31 * res + thirdField.hashCode(); + return res; + } + } }