diff --git a/hazelcast/src/main/java/com/hazelcast/internal/serialization/impl/compact/Schema.java b/hazelcast/src/main/java/com/hazelcast/internal/serialization/impl/compact/Schema.java index 089a6562bf3b..0ca45757912f 100644 --- a/hazelcast/src/main/java/com/hazelcast/internal/serialization/impl/compact/Schema.java +++ b/hazelcast/src/main/java/com/hazelcast/internal/serialization/impl/compact/Schema.java @@ -208,6 +208,37 @@ public void readData(ObjectDataInput in) throws IOException { init(); } + public static List readSchemas(DataInput in) throws IOException { + int schemaCount = in.readInt(); + List schemas = new ArrayList<>(schemaCount); + for (int i = 0; i < schemaCount; i++) { + String typeName = in.readUTF(); + int fieldCount = in.readInt(); + List fields = new ArrayList<>(fieldCount); + for (int j = 0; j < fieldCount; j++) { + String name = in.readUTF(); + FieldKind kind = FieldKind.get(in.readInt()); + FieldDescriptor descriptor = new FieldDescriptor(name, kind); + fields.add(descriptor); + } + var schema = new Schema(typeName, fields); + schemas.add(schema); + } + return schemas; + } + + public static void writeSchemas(DataOutput out, Collection schemas) throws IOException { + out.writeInt(schemas.size()); + for (Schema schema : schemas) { + out.writeUTF(schema.getTypeName()); + out.writeInt(schema.getFieldCount()); + for (FieldDescriptor descriptor : schema.getFields()) { + out.writeUTF(descriptor.getFieldName()); + out.writeInt(descriptor.getKind().getId()); + } + } + } + @Override public int getFactoryId() { return SchemaDataSerializerHook.F_ID; @@ -240,34 +271,4 @@ public int hashCode() { return (int) schemaId; } - public static List readSchemas(DataInput in) throws IOException { - int schemaCount = in.readInt(); - ArrayList schemas = new ArrayList<>(schemaCount); - for (int i = 0; i < schemaCount; i++) { - String typeName = in.readUTF(); - int fieldCount = in.readInt(); - ArrayList fields = new ArrayList<>(fieldCount); - for (int j = 0; j < fieldCount; j++) { - String name = in.readUTF(); - FieldKind kind = FieldKind.get(in.readInt()); - FieldDescriptor descriptor = new FieldDescriptor(name, kind); - fields.add(descriptor); - } - Schema schema = new Schema(typeName, fields); - schemas.add(schema); - } - return schemas; - } - - public static void writeSchemas(DataOutput out, Collection schemas) throws IOException { - out.writeInt(schemas.size()); - for (Schema schema : schemas) { - out.writeUTF(schema.getTypeName()); - out.writeInt(schema.getFieldCount()); - for (FieldDescriptor descriptor : schema.getFields()) { - out.writeUTF(descriptor.getFieldName()); - out.writeInt(descriptor.getKind().getId()); - } - } - } } diff --git a/hazelcast/src/test/java/com/hazelcast/test/compatibility/SamplingNodeExtension.java b/hazelcast/src/test/java/com/hazelcast/test/compatibility/SamplingNodeExtension.java index 52330bbe769f..9b3991523a70 100644 --- a/hazelcast/src/test/java/com/hazelcast/test/compatibility/SamplingNodeExtension.java +++ b/hazelcast/src/test/java/com/hazelcast/test/compatibility/SamplingNodeExtension.java @@ -39,6 +39,7 @@ import com.hazelcast.internal.networking.OutboundHandler; import com.hazelcast.internal.serialization.InternalSerializationService; import com.hazelcast.internal.serialization.SerializationService; +import com.hazelcast.internal.serialization.impl.AbstractSerializationService; import com.hazelcast.internal.serialization.impl.compact.schema.MemberSchemaService; import com.hazelcast.internal.server.ServerConnection; import com.hazelcast.internal.server.ServerContext; @@ -74,13 +75,13 @@ public SamplingNodeExtension(NodeExtension nodeExtension) { @Override public InternalSerializationService createSerializationService() { InternalSerializationService serializationService = nodeExtension.createSerializationService(); - return new SamplingSerializationService(serializationService); + return new SamplingSerializationService((AbstractSerializationService) serializationService); } @Override public InternalSerializationService createCompatibilitySerializationService() { InternalSerializationService serializationService = nodeExtension.createCompatibilitySerializationService(); - return new SamplingSerializationService(serializationService); + return new SamplingSerializationService((AbstractSerializationService) serializationService); } @Override diff --git a/hazelcast/src/test/java/com/hazelcast/test/compatibility/SamplingSerializationService.java b/hazelcast/src/test/java/com/hazelcast/test/compatibility/SamplingSerializationService.java index 6fe219ece57b..a7702e6e9f43 100644 --- a/hazelcast/src/test/java/com/hazelcast/test/compatibility/SamplingSerializationService.java +++ b/hazelcast/src/test/java/com/hazelcast/test/compatibility/SamplingSerializationService.java @@ -22,9 +22,12 @@ import com.hazelcast.internal.serialization.Data; import com.hazelcast.internal.serialization.DataType; import com.hazelcast.internal.serialization.InternalSerializationService; +import com.hazelcast.internal.serialization.impl.AbstractSerializationService; import com.hazelcast.internal.serialization.impl.HeapData; import com.hazelcast.internal.serialization.impl.InternalGenericRecord; +import com.hazelcast.internal.serialization.impl.SerializerAdapter; import com.hazelcast.internal.serialization.impl.compact.CompactGenericRecord; +import com.hazelcast.internal.serialization.impl.compact.CompactStreamSerializer; import com.hazelcast.internal.serialization.impl.compact.Schema; import com.hazelcast.internal.serialization.impl.portable.PortableContext; import com.hazelcast.jet.config.JobConfig; @@ -60,7 +63,8 @@ public class SamplingSerializationService implements InternalSerializationServic static final ConcurrentMap> SERIALIZED_SAMPLES_PER_CLASS_NAME = new ConcurrentHashMap<>(1000); // cache classes for which samples have already been captured - static final Set SAMPLED_CLASSES = newSetFromMap(new ConcurrentHashMap(1000)); + static final Set SAMPLED_CLASSES = newSetFromMap(new ConcurrentHashMap<>(1000)); + static final ConcurrentMap SAMPLED_CLASSES_SCHEMAS = new ConcurrentHashMap<>(1000); private static final int MAX_SERIALIZED_SAMPLES_PER_CLASS = 5; // utility strings to locate test classes commonly used as user objects @@ -78,10 +82,14 @@ public class SamplingSerializationService implements InternalSerializationServic .map(Class::getName) .collect(Collectors.toSet()); - protected final InternalSerializationService delegate; + protected final AbstractSerializationService delegate; + protected final CompactStreamSerializer compactStreamSerializer; - public SamplingSerializationService(InternalSerializationService delegate) { + public SamplingSerializationService(AbstractSerializationService delegate) { this.delegate = delegate; + + SerializerAdapter adapter = delegate.getCompactSerializer(true); + compactStreamSerializer = (CompactStreamSerializer) adapter.getImpl(); } @Override @@ -250,7 +258,7 @@ public void dispose() { } // record the given object, then return it - protected static T sampleObject(T obj, byte[] serializedObject) { + protected T sampleObject(T obj, byte[] serializedObject) { if (obj == null) { return null; } @@ -270,7 +278,7 @@ private static void addSerializedSample(Object obj, byte[] bytes) { } } - private static boolean shouldAddSerializedSample(Object obj) { + private boolean shouldAddSerializedSample(Object obj) { Class klass = obj.getClass(); if (klass.isPrimitive()) { return false; @@ -282,6 +290,12 @@ private static boolean shouldAddSerializedSample(Object obj) { return false; } + if (delegate.isCompactSerializable(obj)) { + SAMPLED_CLASSES_SCHEMAS.computeIfAbsent(className, cn -> + compactStreamSerializer.extractSchema(obj) + ); + } + if (SAMPLED_CLASSES.contains(className)) { return false; } diff --git a/hazelcast/src/test/java/com/hazelcast/test/compatibility/SamplingRunListener.java b/hazelcast/src/test/java/com/hazelcast/test/compatibility/SamplingTestExecutionListener.java similarity index 53% rename from hazelcast/src/test/java/com/hazelcast/test/compatibility/SamplingRunListener.java rename to hazelcast/src/test/java/com/hazelcast/test/compatibility/SamplingTestExecutionListener.java index 77409f5a88aa..2c1ea592e8de 100644 --- a/hazelcast/src/test/java/com/hazelcast/test/compatibility/SamplingRunListener.java +++ b/hazelcast/src/test/java/com/hazelcast/test/compatibility/SamplingTestExecutionListener.java @@ -16,46 +16,50 @@ package com.hazelcast.test.compatibility; +import com.hazelcast.internal.serialization.impl.compact.Schema; import com.hazelcast.logging.ILogger; import com.hazelcast.logging.Logger; import com.hazelcast.test.TestEnvironment; -import org.junit.runner.Result; -import org.junit.runner.notification.RunListener; +import org.junit.platform.launcher.TestExecutionListener; +import org.junit.platform.launcher.TestPlan; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; +import java.io.ObjectOutputStream; import java.nio.channels.FileChannel; +import java.util.Collection; import java.util.List; import java.util.Map; -import static com.hazelcast.internal.nio.IOUtil.closeResource; -import static com.hazelcast.test.HazelcastTestSupport.randomString; -import static com.hazelcast.test.compatibility.SamplingSerializationService.SERIALIZED_SAMPLES_PER_CLASS_NAME; import static com.hazelcast.internal.util.StringUtil.LINE_SEPARATOR; +import static com.hazelcast.internal.util.StringUtil.isNullOrEmpty; +import static com.hazelcast.test.compatibility.SamplingConf.FILE_NAME; +import static com.hazelcast.test.compatibility.SamplingConf.INDEX_FILE_SUFFIX; +import static com.hazelcast.test.compatibility.SamplingConf.SAMPLES_FILE_SUFFIX; +import static com.hazelcast.test.compatibility.SamplingConf.SCHEMA_FILE_SUFFIX; +import static com.hazelcast.test.compatibility.SamplingSerializationService.SERIALIZED_SAMPLES_PER_CLASS_NAME; /** * When tests run is done, dump serialized object samples to output file */ -public class SamplingRunListener extends RunListener { +public class SamplingTestExecutionListener implements TestExecutionListener { - public static final String FILE_NAME = TestEnvironment.getSerializedClassNamesPath() + randomString(); - public static final String SAMPLES_FILE_SUFFIX = ".samples"; - public static final String INDEX_FILE_SUFFIX = ".index"; - - private static final ILogger LOGGER = Logger.getLogger(SamplingRunListener.class); + private static final ILogger LOGGER = Logger.getLogger(SamplingTestExecutionListener.class); @Override - public void testRunFinished(Result result) - throws Exception { - FileOutputStream serializedSamplesOutput = null; - FileWriter indexOutput = null; - try { - serializedSamplesOutput = new FileOutputStream(FILE_NAME + SAMPLES_FILE_SUFFIX); + public void testPlanExecutionFinished(TestPlan testPlan) { + if (isNullOrEmpty(TestEnvironment.getSerializedClassNamesPath())) { + return; + } + LOGGER.info("Sampling is done, serialized classes count: " + SERIALIZED_SAMPLES_PER_CLASS_NAME.keySet().size()); + try ( + var serializedSamplesOutput = new FileOutputStream(FILE_NAME + SAMPLES_FILE_SUFFIX); + var indexOutput = new FileWriter(FILE_NAME + INDEX_FILE_SUFFIX); + var serializedSchemaOutput = new FileOutputStream(FILE_NAME + SCHEMA_FILE_SUFFIX) + ) { FileChannel samplesOutputChannel = serializedSamplesOutput.getChannel(); // index file line format: className,startOfSample1,lengthOfSample1,startOfSample2,lengthOfSample2,... - indexOutput = new FileWriter(FILE_NAME + INDEX_FILE_SUFFIX); for (Map.Entry> entry : SERIALIZED_SAMPLES_PER_CLASS_NAME.entrySet()) { if (entry.getValue().isEmpty()) { @@ -63,20 +67,25 @@ public void testRunFinished(Result result) } List samples = entry.getValue(); indexOutput.write(entry.getKey()); - for (int i = 0; i < samples.size(); i++) { - byte[] sample = samples.get(i); + for (byte[] sample : samples) { indexOutput.write("," + samplesOutputChannel.position() + "," + sample.length); serializedSamplesOutput.write(sample); } indexOutput.write(LINE_SEPARATOR); } - } catch (FileNotFoundException e) { - LOGGER.severe(e); + + writeSchemas(serializedSchemaOutput); } catch (IOException e) { - LOGGER.severe(e); - } finally { - closeResource(indexOutput); - closeResource(serializedSamplesOutput); + throw new RuntimeException(e); + } + } + + private void writeSchemas(FileOutputStream fileOut) { + Collection schemas = SamplingSerializationService.SAMPLED_CLASSES_SCHEMAS.values(); + try (var out = new ObjectOutputStream(fileOut)) { + Schema.writeSchemas(out, schemas); + } catch (Exception e) { + throw new RuntimeException(e); } } } diff --git a/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/TestNodeRegistry.java b/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/TestNodeRegistry.java index 94e176a68634..82e47a5f0526 100644 --- a/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/TestNodeRegistry.java +++ b/hazelcast/src/test/java/com/hazelcast/test/mocknetwork/TestNodeRegistry.java @@ -16,13 +16,12 @@ package com.hazelcast.test.mocknetwork; +import com.hazelcast.cluster.Address; import com.hazelcast.core.HazelcastInstance; import com.hazelcast.core.LifecycleService; import com.hazelcast.instance.impl.Node; import com.hazelcast.instance.impl.NodeContext; import com.hazelcast.instance.impl.NodeState; -import com.hazelcast.cluster.Address; -import com.hazelcast.test.AssertTask; import com.hazelcast.internal.util.AddressUtil; import java.net.UnknownHostException; @@ -39,7 +38,7 @@ import java.util.concurrent.ConcurrentMap; import static com.hazelcast.test.HazelcastTestSupport.assertTrueEventually; -import static org.junit.Assert.assertEquals; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertFalse; public final class TestNodeRegistry { @@ -61,12 +60,8 @@ public NodeContext createNodeContext(final Address address, Set
initial final Node node = nodes.get(address); if (node != null) { assertFalse(address + " is already registered", node.isRunning()); - assertTrueEventually(new AssertTask() { - @Override - public void run() { - assertEquals(address + " should be SHUT_DOWN", NodeState.SHUT_DOWN, node.getState()); - } - }); + assertTrueEventually(address + " should be SHUT_DOWN", + () -> assertThat(node.getState()).isEqualTo(NodeState.SHUT_DOWN)); nodes.remove(address, node); } return new MockNodeContext(this, address, initiallyBlockedAddresses, nodeExtensionPriorityList); diff --git a/hazelcast/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener b/hazelcast/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener new file mode 100644 index 000000000000..8bfbfff9b4c3 --- /dev/null +++ b/hazelcast/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener @@ -0,0 +1 @@ +com.hazelcast.test.compatibility.SamplingTestExecutionListener \ No newline at end of file diff --git a/pom.xml b/pom.xml index 2e228e7b52b2..842c791d53d7 100644 --- a/pom.xml +++ b/pom.xml @@ -1389,7 +1389,7 @@ generate-compatibility-samples - -Xms128m -Xmx1G + -Xms4G -Xmx8G @@ -1401,7 +1401,7 @@ listener - com.hazelcast.test.compatibility.SamplingRunListener + com.hazelcast.test.compatibility.SamplingTestExecutionListener false