Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add MemorySegment::mismatch #180

Closed
wants to merge 5 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/java.base/share/classes/module-info.java
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,8 @@
jdk.internal.vm.ci,
jdk.incubator.foreign,
jdk.unsupported;
exports jdk.internal.util to
jdk.incubator.foreign;
exports jdk.internal.util.jar to
jdk.jartool;
exports jdk.internal.util.xml to
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,31 @@ static <S extends MemorySegment> Spliterator<S> spliterator(S segment, SequenceL
*/
void copyFrom(MemorySegment src);

/**
* Finds and returns the offset, in bytes, of the first mismatch between
* this segment and a given other segment. The offset is relative to the
* {@link #baseAddress() base address} of each segment and will be in the
* range of 0 (inclusive) up to the {@link #byteSize() size} (in bytes) of
* the smaller memory segment (exclusive).
* <p>
* If the two segments share a common prefix then the returned offset is
* the length of the common prefix and it follows that there is a mismatch
* between the two segments at that offset within the respective segments.
* If one segment is a proper prefix of the other then the returned offset is
* the smaller of the segment sizes, and it follows that the offset is only
* valid for the larger segment. Otherwise, there is no mismatch.
ChrisHegarty marked this conversation as resolved.
Show resolved Hide resolved
*
* @param other the segment to be tested for a mismatch with this segment
* @return the relative offset, in bytes, of the first mismatch between this
* and the given other segment, otherwise -1 if no mismatch
* @throws IllegalStateException if either this segment of the other segment
* have been already closed, or if access occurs from a thread other than the
* thread owning either segment
* @throws UnsupportedOperationException if either this segment or the other
* segment does not feature at least the {@link MemorySegment#READ} access mode
*/
long mismatch(MemorySegment other);

/**
* Wraps this segment in a {@link ByteBuffer}. Some of the properties of the returned buffer are linked to
* the properties of this segment. For instance, if this segment is <em>immutable</em>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import jdk.internal.access.foreign.MemorySegmentProxy;
import jdk.internal.access.foreign.UnmapperProxy;
import jdk.internal.misc.Unsafe;
import jdk.internal.util.ArraysSupport;
import jdk.internal.vm.annotation.ForceInline;
import sun.security.action.GetPropertyAction;

Expand Down Expand Up @@ -131,6 +132,53 @@ public void copyFrom(MemorySegment src) {
base(), min(), size);
}

@Override
public long mismatch(MemorySegment other) {
AbstractMemorySegmentImpl that = (AbstractMemorySegmentImpl)other;
final long thisSize = this.byteSize();
final long thatSize = that.byteSize();
final long minSize = Math.min(thisSize, thatSize);

this.checkRange(0, minSize, false);
ChrisHegarty marked this conversation as resolved.
Show resolved Hide resolved
that.checkRange(0, minSize, false);

if (this == other)
return -1;

long off = 0;
long remaining = minSize;
int i = 0;
while (remaining > 7) {
int size;
if (remaining > Integer.MAX_VALUE) {
size = Integer.MAX_VALUE;
} else {
size = (int) remaining;
}
i = ArraysSupport.vectorizedMismatch(
this.base(),
this.min() + off,
that.base(),
that.min() + off,
size,
ArraysSupport.LOG2_ARRAY_BYTE_INDEX_SCALE);
if (i >= 0) {
return off + i;
}
i = size - ~i;

off += i;
remaining -= i;
}

for (; off < minSize; off++) {
if (UNSAFE.getByte(this.base(), this.min() + off) != UNSAFE.getByte(that.base(), that.min() + off)) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This code could be simplified/rewritten to use MemoryAddress' and VH, instead of unsafe access with object/offset addressing. E.g. you could maintain a MemoryAddresslocal variable in the loop instead of theoffsetand keep increasing that address on each iteration ofArraySupport::vectorizedMismatch`. Then, when you get out of the loop, the address already points at the base of the region to compare, and a simple for loop with an indexed VH should do the job.

return off;
}
}
return thisSize != thatSize ? minSize : -1;
}

@Override
@ForceInline
public final MemoryAddress baseAddress() {
Expand Down
236 changes: 236 additions & 0 deletions test/jdk/java/foreign/TestMismatch.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,236 @@
/*
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/

/*
* @test
* @run testng TestMismatch
*/

import java.lang.invoke.VarHandle;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.IntFunction;
import jdk.incubator.foreign.MemoryAddress;
import jdk.incubator.foreign.MemoryLayouts;
import jdk.incubator.foreign.MemorySegment;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import static java.lang.System.out;
import static jdk.incubator.foreign.MemorySegment.READ;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertThrows;

public class TestMismatch {

final static VarHandle BYTE_HANDLE = MemoryLayouts.JAVA_BYTE.varHandle(byte.class);

// stores a increasing sequence of values into the memory of the given segment
static MemorySegment initializeSegment(MemorySegment segment) {
MemoryAddress addr = segment.baseAddress();
for (int i = 0 ; i < segment.byteSize() ; i++) {
BYTE_HANDLE.set(addr.addOffset(i), (byte)i);
}
return segment;
}

@Test(dataProvider = "slices")
public void testSameValues(MemorySegment ss1, MemorySegment ss2) {
out.format("testSameValues s1:%s, s2:%s\n", ss1, ss2);
MemorySegment s1 = initializeSegment(ss1);
MemorySegment s2 = initializeSegment(ss2);

if (s1.byteSize() == s2.byteSize()) {
assertEquals(s1.mismatch(s2), -1); // identical
assertEquals(s2.mismatch(s1), -1);
} else if (s1.byteSize() > s2.byteSize()) {
assertEquals(s1.mismatch(s2), s2.byteSize()); // proper prefix
assertEquals(s2.mismatch(s1), s2.byteSize());
} else {
assert s1.byteSize() < s2.byteSize();
assertEquals(s1.mismatch(s2), s1.byteSize()); // proper prefix
assertEquals(s2.mismatch(s1), s1.byteSize());
}
}

@Test(dataProvider = "slices")
public void testDifferentValues(MemorySegment s1, MemorySegment s2) {
out.format("testDifferentValues s1:%s, s2:%s\n", s1, s2);
s1 = initializeSegment(s1);
s2 = initializeSegment(s2);

for (long i = s2.byteSize() -1 ; i >= 0; i--) {
long expectedMismatchOffset = i;
BYTE_HANDLE.set(s2.baseAddress().addOffset(i), (byte) 0xFF);

if (s1.byteSize() == s2.byteSize()) {
assertEquals(s1.mismatch(s2), expectedMismatchOffset);
assertEquals(s2.mismatch(s1), expectedMismatchOffset);
} else if (s1.byteSize() > s2.byteSize()) {
assertEquals(s1.mismatch(s2), expectedMismatchOffset);
assertEquals(s2.mismatch(s1), expectedMismatchOffset);
} else {
assert s1.byteSize() < s2.byteSize();
var off = Math.min(s1.byteSize(), expectedMismatchOffset);
assertEquals(s1.mismatch(s2), off); // proper prefix
assertEquals(s2.mismatch(s1), off);
}
}
}
Comment on lines +57 to +99
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How important is it that these tests operate on slices? Looking at the test code, it could have worked equally well if the input parameters were just two sizes, and then you did an explicit allocation (or maybe also receive a segment factory from the provider, so that you can test different segment kinds).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Originally I had a version of the test that did compare specific segments, but it didn't scale well to different sizes and kinds ( we need to test both above and below the 8 byte threshold ). I removed a number of slice sizes, which greatly reduces the combinations. This may be enough, or I can certainly revisit the test's structure.


@Test
public void testEmpty() {
var s1 = MemorySegment.ofArray(new byte[0]);
assertEquals(s1.mismatch(s1), -1);
try (var nativeSegment = MemorySegment.allocateNative(4)) {
var s2 = nativeSegment.asSlice(0, 0);
assertEquals(s1.mismatch(s2), -1);
assertEquals(s2.mismatch(s1), -1);
}
}

@Test
public void testLarge() {
try (var s1 = MemorySegment.allocateNative((long)Integer.MAX_VALUE + 10L);
var s2 = MemorySegment.allocateNative((long)Integer.MAX_VALUE + 10L)) {
assertEquals(s1.mismatch(s1), -1);
assertEquals(s1.mismatch(s2), -1);
assertEquals(s2.mismatch(s1), -1);

for (long i = s2.byteSize() -1 ; i >= Integer.MAX_VALUE - 10L; i--) {
BYTE_HANDLE.set(s2.baseAddress().addOffset(i), (byte) 0xFF);
long expectedMismatchOffset = i;
assertEquals(s1.mismatch(s2), expectedMismatchOffset);
assertEquals(s2.mismatch(s1), expectedMismatchOffset);
}
}
}

static final Class<IllegalStateException> ISE = IllegalStateException.class;
static final Class<UnsupportedOperationException> UOE = UnsupportedOperationException.class;

@Test
public void testClosed() {
var s1 = MemorySegment.ofArray(new byte[4]);
var s2 = MemorySegment.ofArray(new byte[4]);
s1.close();
assertThrows(ISE, () -> s1.mismatch(s1));
assertThrows(ISE, () -> s1.mismatch(s2));
assertThrows(ISE, () -> s2.mismatch(s1));
}

@Test
public void testInsufficientAccessModes() {
var s1 = MemorySegment.ofArray(new byte[4]);
var s2 = MemorySegment.ofArray(new byte[4]);
var s1WithoutRead = s1.withAccessModes(s1.accessModes() & ~READ);
var s2WithoutRead = s2.withAccessModes(s2.accessModes() & ~READ);

assertThrows(UOE, () -> s1.mismatch(s2WithoutRead));
assertThrows(UOE, () -> s1WithoutRead.mismatch(s2));
assertThrows(UOE, () -> s1WithoutRead.mismatch(s2WithoutRead));
}

@Test
public void testThreadAccess() throws Exception {
var segment = MemorySegment.ofArray(new byte[4]);
{
AtomicReference<RuntimeException> exception = new AtomicReference<>();
Runnable action = () -> {
try {
MemorySegment.ofArray(new byte[4]).mismatch(segment);
} catch (RuntimeException e) {
exception.set(e);
}
};
Thread thread = new Thread(action);
thread.start();
thread.join();

RuntimeException e = exception.get();
if (!(e instanceof IllegalStateException)) {
throw e;
}
}
{
AtomicReference<RuntimeException> exception = new AtomicReference<>();
Runnable action = () -> {
try {
segment.mismatch(MemorySegment.ofArray(new byte[4]));
} catch (RuntimeException e) {
exception.set(e);
}
};
Thread thread = new Thread(action);
thread.start();
thread.join();

RuntimeException e = exception.get();
if (!(e instanceof IllegalStateException)) {
throw e;
}
}
}

enum SegmentKind {
NATIVE(MemorySegment::allocateNative),
ARRAY(i -> MemorySegment.ofArray(new byte[i]));

final IntFunction<MemorySegment> segmentFactory;

SegmentKind(IntFunction<MemorySegment> segmentFactory) {
this.segmentFactory = segmentFactory;
}

MemorySegment makeSegment(int elems) {
return segmentFactory.apply(elems);
}
}

@DataProvider(name = "slices")
static Object[][] slices() {
int[] sizes = { 16, 8, 4, 2, 1 };
List<MemorySegment> aSlices = new ArrayList<>();
List<MemorySegment> bSlices = new ArrayList<>();
for (List<MemorySegment> slices : List.of(aSlices, bSlices)) {
for (SegmentKind kind : SegmentKind.values()) {
MemorySegment segment = kind.makeSegment(16);
//compute all slices
for (int size : sizes) {
for (int index = 0 ; index < 16 ; index += size) {
MemorySegment slice = segment.asSlice(index, size);
slices.add(slice);
}
}
}
}
assert aSlices.size() == bSlices.size();
Object[][] sliceArray = new Object[aSlices.size() * bSlices.size()][];
for (int i = 0 ; i < aSlices.size() ; i++) {
for (int j = 0 ; j < bSlices.size() ; j++) {
sliceArray[i * aSlices.size() + j] = new Object[] { aSlices.get(i), bSlices.get(j) };
}
}
return sliceArray;
}
}