diff --git a/jctools-build/src/main/java/org/jctools/queues/atomic/JavaParsingAtomicArrayQueueGenerator.java b/jctools-build/src/main/java/org/jctools/queues/atomic/JavaParsingAtomicArrayQueueGenerator.java index 77321303..efeb3fe6 100644 --- a/jctools-build/src/main/java/org/jctools/queues/atomic/JavaParsingAtomicArrayQueueGenerator.java +++ b/jctools-build/src/main/java/org/jctools/queues/atomic/JavaParsingAtomicArrayQueueGenerator.java @@ -15,6 +15,8 @@ import com.github.javaparser.ast.type.PrimitiveType; import com.github.javaparser.ast.type.Type; +import java.util.Arrays; + /** * This generator takes in an JCTools 'ArrayQueue' Java source file and patches {@link sun.misc.Unsafe} accesses into * atomic {@link java.util.concurrent.atomic.AtomicLongFieldUpdater}. It outputs a Java source file with these patches. @@ -25,11 +27,11 @@ public final class JavaParsingAtomicArrayQueueGenerator extends JavaParsingAtomicQueueGenerator { public static void main(String[] args) throws Exception { - main(JavaParsingAtomicArrayQueueGenerator.class, args); + main(Boolean.parseBoolean(args[0]), JavaParsingAtomicArrayQueueGenerator.class, Arrays.copyOfRange(args, 1, args.length)); } - JavaParsingAtomicArrayQueueGenerator(String sourceFileName) { - super(sourceFileName); + JavaParsingAtomicArrayQueueGenerator(boolean unpadded ,String sourceFileName) { + super(unpadded, sourceFileName); } @Override @@ -70,6 +72,17 @@ public void visit(ClassOrInterfaceDeclaration node, Void arg) { + JavaParsingAtomicArrayQueueGenerator.class.getName(), "which can found in the jctools-build module. The original source file is " + sourceFileName + ".") + node.getJavadocComment().orElse(new JavadocComment("")).getContent()); + + if (unpadded) { + // remove padding fields + for (FieldDeclaration field : node.getFields()) + { + String fieldName = field.getVariables().get(0).getNameAsString(); + if (fieldName.startsWith("b0") || fieldName.startsWith("b1")) { + node.remove(field); + } + } + } } String fieldUpdaterFieldName(String fieldName) { diff --git a/jctools-build/src/main/java/org/jctools/queues/atomic/JavaParsingAtomicLinkedQueueGenerator.java b/jctools-build/src/main/java/org/jctools/queues/atomic/JavaParsingAtomicLinkedQueueGenerator.java index 660c0f39..ab89a835 100644 --- a/jctools-build/src/main/java/org/jctools/queues/atomic/JavaParsingAtomicLinkedQueueGenerator.java +++ b/jctools-build/src/main/java/org/jctools/queues/atomic/JavaParsingAtomicLinkedQueueGenerator.java @@ -13,6 +13,8 @@ import com.github.javaparser.ast.type.PrimitiveType; import com.github.javaparser.ast.type.Type; +import java.util.Arrays; + /** * This generator takes in an JCTools 'LinkedQueue' Java source file and patches {@link sun.misc.Unsafe} accesses into * atomic {@link java.util.concurrent.atomic.AtomicLongFieldUpdater}. It outputs a Java source file with these patches. @@ -22,13 +24,14 @@ */ public final class JavaParsingAtomicLinkedQueueGenerator extends JavaParsingAtomicQueueGenerator { private static final String MPSC_LINKED_ATOMIC_QUEUE_NAME = "MpscLinkedAtomicQueue"; + private static final String MPSC_LINKED_UNPADDED_ATOMIC_QUEUE_NAME = "MpscLinkedAtomicUnpaddedQueue"; public static void main(String[] args) throws Exception { - main(JavaParsingAtomicLinkedQueueGenerator.class, args); + main(Boolean.parseBoolean(args[0]), JavaParsingAtomicLinkedQueueGenerator.class, Arrays.copyOfRange(args, 1, args.length)); } - JavaParsingAtomicLinkedQueueGenerator(String sourceFileName) { - super(sourceFileName); + JavaParsingAtomicLinkedQueueGenerator(boolean unpadded, String sourceFileName) { + super(unpadded, sourceFileName); } @Override @@ -40,7 +43,8 @@ public void visit(ConstructorDeclaration n, Void arg) { if (nameAsString.equals("WeakIterator")) return; n.setName(translateQueueName(nameAsString)); - if (MPSC_LINKED_ATOMIC_QUEUE_NAME.equals(nameAsString)) { + String atomicQueueName = unpadded ? MPSC_LINKED_UNPADDED_ATOMIC_QUEUE_NAME : MPSC_LINKED_ATOMIC_QUEUE_NAME; + if (atomicQueueName.equals(nameAsString)) { // Special case for MPSC because the Unsafe variant has a static factory method and a protected constructor. n.setModifier(Keyword.PROTECTED, false); n.setModifier(Keyword.PUBLIC, true); @@ -56,7 +60,8 @@ public void visit(ClassOrInterfaceDeclaration node, Void arg) { String nameAsString = node.getNameAsString(); if (nameAsString.contains("Queue")) node.setName(translateQueueName(nameAsString)); - if (MPSC_LINKED_ATOMIC_QUEUE_NAME.equals(nameAsString)) { + String atomicQueueName = unpadded ? MPSC_LINKED_UNPADDED_ATOMIC_QUEUE_NAME : MPSC_LINKED_ATOMIC_QUEUE_NAME; + if (atomicQueueName.equals(nameAsString)) { /* * Special case for MPSC */ @@ -80,6 +85,16 @@ public void visit(ClassOrInterfaceDeclaration node, Void arg) { + JavaParsingAtomicLinkedQueueGenerator.class.getName(), "which can found in the jctools-build module. The original source file is " + sourceFileName + ".") + node.getJavadocComment().orElse(new JavadocComment("")).getContent()); + + if (unpadded) { + // remove padding fields + for (FieldDeclaration field : node.getFields()) { + String fieldName = field.getVariables().get(0).getNameAsString(); + if (fieldName.startsWith("b0") || fieldName.startsWith("b1")) { + node.remove(field); + } + } + } } @Override diff --git a/jctools-build/src/main/java/org/jctools/queues/atomic/JavaParsingAtomicQueueGenerator.java b/jctools-build/src/main/java/org/jctools/queues/atomic/JavaParsingAtomicQueueGenerator.java index b3c1b5a2..7b08a3ea 100644 --- a/jctools-build/src/main/java/org/jctools/queues/atomic/JavaParsingAtomicQueueGenerator.java +++ b/jctools-build/src/main/java/org/jctools/queues/atomic/JavaParsingAtomicQueueGenerator.java @@ -59,11 +59,12 @@ abstract class JavaParsingAtomicQueueGenerator extends VoidVisitorAdapter protected static final String INDENT_LEVEL = " "; protected final String sourceFileName; + protected final boolean unpadded; - static void main(Class generatorClass, String[] args) throws Exception { + static void main(boolean unpadded, Class generatorClass, String[] args) throws Exception { if (args.length < 2) { - throw new IllegalArgumentException("Usage: outputDirectory inputSourceFiles"); + throw new IllegalArgumentException("Usage: unpadded outputDirectory inputSourceFiles"); } File outputDirectory = new File(args[0]); @@ -72,11 +73,15 @@ static void main(Class generatorClass File file = new File(args[i]); System.out.println("Processing " + file); CompilationUnit cu = new JavaParser().parse(file).getResult().get(); - JavaParsingAtomicQueueGenerator generator = buildGenerator(generatorClass, file.getName()); + JavaParsingAtomicQueueGenerator generator = buildGenerator(generatorClass, unpadded, file.getName()); generator.visit(cu, null); generator.organiseImports(cu); + if (unpadded) { + cleanupPaddingComments(cu); + } + String outputFileName = generator.translateQueueName(file.getName().replace(".java", "")) + ".java"; try (FileWriter writer = new FileWriter(new File(outputDirectory, outputFileName))) { @@ -87,7 +92,21 @@ static void main(Class generatorClass } } - JavaParsingAtomicQueueGenerator(String sourceFileName) { + + + private static void cleanupPaddingComments(CompilationUnit cu) + { + for (Comment comment : cu.getAllContainedComments()) + { + String content = comment.getContent(); + if (content.contains("byte b") || content.contains("drop 8b") || content.contains("drop 16b") ) { + comment.remove(); + } + } + } + + JavaParsingAtomicQueueGenerator(boolean unpadded, String sourceFileName) { + this.unpadded = unpadded; this.sourceFileName = sourceFileName; } @@ -98,7 +117,7 @@ static void main(Class generatorClass public void visit(PackageDeclaration n, Void arg) { super.visit(n, arg); // Change the package of the output - n.setName("org.jctools.queues.atomic"); + n.setName(unpadded? "org.jctools.queues.atomic.unpadded" : "org.jctools.queues.atomic"); } @Override @@ -151,11 +170,11 @@ protected void removeStaticFieldsAndInitialisers(ClassOrInterfaceDeclaration nod protected String translateQueueName(String qName) { if (qName.contains("LinkedQueue") || qName.contains("LinkedArrayQueue")) { - return qName.replace("Linked", "LinkedAtomic"); + return qName.replace("Linked", unpadded ? "LinkedAtomicUnpadded" : "LinkedAtomic"); } if (qName.contains("ArrayQueue")) { - return qName.replace("ArrayQueue", "AtomicArrayQueue"); + return qName.replace("ArrayQueue", unpadded ? "AtomicUnpaddedArrayQueue" : "AtomicArrayQueue"); } throw new IllegalArgumentException("Unexpected queue name: " + qName); @@ -219,7 +238,7 @@ protected void replaceParentClassesForAtomics(ClassOrInterfaceDeclaration n) { // ignore the JDK parent break; case "BaseLinkedQueue": - parent.setName("BaseLinkedAtomicQueue"); + parent.setName(unpadded? "BaseLinkedAtomicUnpaddedQueue" : "BaseLinkedAtomicQueue"); break; case "ConcurrentCircularArrayQueue": parent.setName("AtomicReferenceArrayQueue"); @@ -258,7 +277,7 @@ protected void organiseImports(CompilationUnit cu) { cu.addImport(new ImportDeclaration("java.util.concurrent.atomic", false, true)); cu.addImport(new ImportDeclaration("org.jctools.queues", false, true)); - cu.addImport(staticImportDeclaration("org.jctools.queues.atomic.AtomicQueueUtil")); + cu.addImport(staticImportDeclaration(unpadded? "org.jctools.queues.atomic.unpadded.AtomicQueueUtil" : "org.jctools.queues.atomic.AtomicQueueUtil")); } protected String capitalise(String s) { @@ -388,8 +407,8 @@ protected boolean isRefType(Type in, String className) { return false; } - private static T buildGenerator(Class generatorClass, String fileName) throws Exception { - return generatorClass.getDeclaredConstructor(String.class).newInstance(fileName); + private static T buildGenerator(Class generatorClass, boolean unpadded, String fileName) throws Exception { + return generatorClass.getDeclaredConstructor(boolean.class, String.class).newInstance(unpadded, fileName); } ImportDeclaration staticImportDeclaration(String name) { diff --git a/jctools-core/pom.xml b/jctools-core/pom.xml index 936ac8ae..09521100 100644 --- a/jctools-core/pom.xml +++ b/jctools-core/pom.xml @@ -133,6 +133,8 @@ false org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + + false ${basedir}/src/main/java/org/jctools/queues/atomic @@ -154,6 +156,8 @@ false org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + + false ${basedir}/src/main/java/org/jctools/queues/atomic @@ -173,6 +177,61 @@ + + generate-atomic-unpadded-array-queues + + java + + generate-sources + + true + false + org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + + + true + + ${basedir}/src/main/java/org/jctools/queues/atomic/unpadded + + ${basedir}/src/main/java/org/jctools/queues/SpscArrayQueue.java + ${basedir}/src/main/java/org/jctools/queues/SpmcArrayQueue.java + ${basedir}/src/main/java/org/jctools/queues/MpscArrayQueue.java + ${basedir}/src/main/java/org/jctools/queues/MpmcArrayQueue.java + + + + + generate-atomic-unpadded-linked-queues + + java + + generate-sources + + true + false + org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + + + true + + ${basedir}/src/main/java/org/jctools/queues/atomic/unpadded + + ${basedir}/src/main/java/org/jctools/queues/BaseLinkedQueue.java + ${basedir}/src/main/java/org/jctools/queues/SpscLinkedQueue.java + ${basedir}/src/main/java/org/jctools/queues/MpscLinkedQueue.java + + ${basedir}/src/main/java/org/jctools/queues/BaseSpscLinkedArrayQueue.java + ${basedir}/src/main/java/org/jctools/queues/SpscChunkedArrayQueue.java + ${basedir}/src/main/java/org/jctools/queues/SpscUnboundedArrayQueue.java + ${basedir}/src/main/java/org/jctools/queues/SpscGrowableArrayQueue.java + + ${basedir}/src/main/java/org/jctools/queues/BaseMpscLinkedArrayQueue.java + ${basedir}/src/main/java/org/jctools/queues/MpscChunkedArrayQueue.java + ${basedir}/src/main/java/org/jctools/queues/MpscUnboundedArrayQueue.java + ${basedir}/src/main/java/org/jctools/queues/MpscGrowableArrayQueue.java + + + generate-unpadded-queues diff --git a/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/AtomicQueueUtil.java b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/AtomicQueueUtil.java new file mode 100644 index 00000000..a12980d2 --- /dev/null +++ b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/AtomicQueueUtil.java @@ -0,0 +1,101 @@ +package org.jctools.queues.atomic.unpadded; + +import java.util.concurrent.atomic.AtomicLongArray; +import java.util.concurrent.atomic.AtomicReferenceArray; + +final class AtomicQueueUtil +{ + static E lvRefElement(AtomicReferenceArray buffer, int offset) + { + return buffer.get(offset); + } + + static E lpRefElement(AtomicReferenceArray buffer, int offset) + { + return buffer.get(offset); // no weaker form available + } + + static void spRefElement(AtomicReferenceArray buffer, int offset, E value) + { + buffer.lazySet(offset, value); // no weaker form available + } + + static void soRefElement(AtomicReferenceArray buffer, int offset, Object value) + { + buffer.lazySet(offset, value); + } + + static void svRefElement(AtomicReferenceArray buffer, int offset, E value) + { + buffer.set(offset, value); + } + + static int calcRefElementOffset(long index) + { + return (int) index; + } + + static int calcCircularRefElementOffset(long index, long mask) + { + return (int) (index & mask); + } + + static AtomicReferenceArray allocateRefArray(int capacity) + { + return new AtomicReferenceArray(capacity); + } + + static void spLongElement(AtomicLongArray buffer, int offset, long e) + { + buffer.lazySet(offset, e); + } + + static void soLongElement(AtomicLongArray buffer, int offset, long e) + { + buffer.lazySet(offset, e); + } + + static long lpLongElement(AtomicLongArray buffer, int offset) + { + return buffer.get(offset); + } + + static long lvLongElement(AtomicLongArray buffer, int offset) + { + return buffer.get(offset); + } + + static int calcLongElementOffset(long index) + { + return (int) index; + } + + static int calcCircularLongElementOffset(long index, int mask) + { + return (int) (index & mask); + } + + static AtomicLongArray allocateLongArray(int capacity) + { + return new AtomicLongArray(capacity); + } + + static int length(AtomicReferenceArray buf) + { + return buf.length(); + } + + /** + * This method assumes index is actually (index << 1) because lower bit is used for resize hence the >> 1 + */ + static int modifiedCalcCircularRefElementOffset(long index, long mask) + { + return (int) (index & mask) >> 1; + } + + static int nextArrayOffset(AtomicReferenceArray curr) + { + return length(curr) - 1; + } + +} diff --git a/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/AtomicReferenceArrayQueue.java b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/AtomicReferenceArrayQueue.java new file mode 100644 index 00000000..155c1f48 --- /dev/null +++ b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/AtomicReferenceArrayQueue.java @@ -0,0 +1,158 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jctools.queues.atomic.unpadded; + +import org.jctools.queues.IndexedQueueSizeUtil; +import org.jctools.queues.IndexedQueueSizeUtil.IndexedQueue; +import org.jctools.queues.MessagePassingQueue; +import org.jctools.queues.QueueProgressIndicators; +import org.jctools.queues.SupportsIterator; +import org.jctools.util.Pow2; + +import java.util.AbstractQueue; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.concurrent.atomic.AtomicReferenceArray; + +import static org.jctools.queues.atomic.unpadded.AtomicQueueUtil.calcCircularRefElementOffset; +import static org.jctools.queues.atomic.unpadded.AtomicQueueUtil.lvRefElement; + +abstract class AtomicReferenceArrayQueue extends AbstractQueue implements IndexedQueue, QueueProgressIndicators, MessagePassingQueue, SupportsIterator +{ + protected final AtomicReferenceArray buffer; + protected final int mask; + + public AtomicReferenceArrayQueue(int capacity) + { + int actualCapacity = Pow2.roundToPowerOfTwo(capacity); + this.mask = actualCapacity - 1; + this.buffer = new AtomicReferenceArray(actualCapacity); + } + + @Override + public String toString() + { + return this.getClass().getName(); + } + + @Override + public void clear() + { + while (poll() != null) + { + // toss it away + } + } + + @Override + public final int capacity() + { + return (int) (mask + 1); + } + + /** + * {@inheritDoc} + *

+ */ + @Override + public final int size() + { + return IndexedQueueSizeUtil.size(this, IndexedQueueSizeUtil.PLAIN_DIVISOR); + } + + @Override + public final boolean isEmpty() + { + return IndexedQueueSizeUtil.isEmpty(this); + } + + @Override + public final long currentProducerIndex() + { + return lvProducerIndex(); + } + + @Override + public final long currentConsumerIndex() + { + return lvConsumerIndex(); + } + + /** + * Get an iterator for this queue. This method is thread safe. + *

+ * The iterator provides a best-effort snapshot of the elements in the queue. + * The returned iterator is not guaranteed to return elements in queue order, + * and races with the consumer thread may cause gaps in the sequence of returned elements. + * Like {link #relaxedPoll}, the iterator may not immediately return newly inserted elements. + * + * @return The iterator. + */ + @Override + public final Iterator iterator() { + final long cIndex = lvConsumerIndex(); + final long pIndex = lvProducerIndex(); + + return new WeakIterator(cIndex, pIndex, mask, buffer); + } + + private static class WeakIterator implements Iterator { + + private final long pIndex; + private final int mask; + private final AtomicReferenceArray buffer; + private long nextIndex; + private E nextElement; + + WeakIterator(long cIndex, long pIndex, int mask, AtomicReferenceArray buffer) { + this.nextIndex = cIndex; + this.pIndex = pIndex; + this.mask = mask; + this.buffer = buffer; + nextElement = getNext(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException("remove"); + } + + @Override + public boolean hasNext() { + return nextElement != null; + } + + @Override + public E next() { + final E e = nextElement; + if (e == null) + throw new NoSuchElementException(); + nextElement = getNext(); + return e; + } + + private E getNext() { + final int mask = this.mask; + final AtomicReferenceArray buffer = this.buffer; + while (nextIndex < pIndex) { + int offset = calcCircularRefElementOffset(nextIndex++, mask); + E e = lvRefElement(buffer, offset); + if (e != null) { + return e; + } + } + return null; + } + } +} diff --git a/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/BaseLinkedAtomicUnpaddedQueue.java b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/BaseLinkedAtomicUnpaddedQueue.java new file mode 100644 index 00000000..f270d141 --- /dev/null +++ b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/BaseLinkedAtomicUnpaddedQueue.java @@ -0,0 +1,328 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jctools.queues.atomic.unpadded; + +import java.util.AbstractQueue; +import java.util.Iterator; +import java.util.Queue; +import java.util.concurrent.atomic.*; +import org.jctools.queues.*; +import static org.jctools.queues.atomic.unpadded.AtomicQueueUtil.*; + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseLinkedQueue.java. + */ +abstract class BaseLinkedAtomicUnpaddedQueuePad0 extends AbstractQueue implements MessagePassingQueue { +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseLinkedQueue.java. + */ +abstract class BaseLinkedAtomicUnpaddedQueueProducerNodeRef extends BaseLinkedAtomicUnpaddedQueuePad0 { + + private static final AtomicReferenceFieldUpdater P_NODE_UPDATER = AtomicReferenceFieldUpdater.newUpdater(BaseLinkedAtomicUnpaddedQueueProducerNodeRef.class, LinkedQueueAtomicNode.class, "producerNode"); + + private volatile LinkedQueueAtomicNode producerNode; + + final void spProducerNode(LinkedQueueAtomicNode newValue) { + P_NODE_UPDATER.lazySet(this, newValue); + } + + final void soProducerNode(LinkedQueueAtomicNode newValue) { + P_NODE_UPDATER.lazySet(this, newValue); + } + + final LinkedQueueAtomicNode lvProducerNode() { + return producerNode; + } + + final boolean casProducerNode(LinkedQueueAtomicNode expect, LinkedQueueAtomicNode newValue) { + return P_NODE_UPDATER.compareAndSet(this, expect, newValue); + } + + final LinkedQueueAtomicNode lpProducerNode() { + return producerNode; + } + + protected final LinkedQueueAtomicNode xchgProducerNode(LinkedQueueAtomicNode newValue) { + return P_NODE_UPDATER.getAndSet(this, newValue); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseLinkedQueue.java. + */ +abstract class BaseLinkedAtomicUnpaddedQueuePad1 extends BaseLinkedAtomicUnpaddedQueueProducerNodeRef { +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseLinkedQueue.java. + */ +abstract class BaseLinkedAtomicUnpaddedQueueConsumerNodeRef extends BaseLinkedAtomicUnpaddedQueuePad1 { + + private static final AtomicReferenceFieldUpdater C_NODE_UPDATER = AtomicReferenceFieldUpdater.newUpdater(BaseLinkedAtomicUnpaddedQueueConsumerNodeRef.class, LinkedQueueAtomicNode.class, "consumerNode"); + + private volatile LinkedQueueAtomicNode consumerNode; + + final void spConsumerNode(LinkedQueueAtomicNode newValue) { + C_NODE_UPDATER.lazySet(this, newValue); + } + + @SuppressWarnings("unchecked") + final LinkedQueueAtomicNode lvConsumerNode() { + return consumerNode; + } + + final LinkedQueueAtomicNode lpConsumerNode() { + return consumerNode; + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseLinkedQueue.java. + */ +abstract class BaseLinkedAtomicUnpaddedQueuePad2 extends BaseLinkedAtomicUnpaddedQueueConsumerNodeRef { +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseLinkedQueue.java. + * + * A base data structure for concurrent linked queues. For convenience also pulled in common single consumer + * methods since at this time there's no plan to implement MC. + */ +abstract class BaseLinkedAtomicUnpaddedQueue extends BaseLinkedAtomicUnpaddedQueuePad2 { + + @Override + public final Iterator iterator() { + throw new UnsupportedOperationException(); + } + + @Override + public String toString() { + return this.getClass().getName(); + } + + protected final LinkedQueueAtomicNode newNode() { + return new LinkedQueueAtomicNode(); + } + + protected final LinkedQueueAtomicNode newNode(E e) { + return new LinkedQueueAtomicNode(e); + } + + /** + * {@inheritDoc}
+ *

+ * IMPLEMENTATION NOTES:
+ * This is an O(n) operation as we run through all the nodes and count them.
+ * The accuracy of the value returned by this method is subject to races with producer/consumer threads. In + * particular when racing with the consumer thread this method may under estimate the size.
+ * + * @see java.util.Queue#size() + */ + @Override + public final int size() { + // Read consumer first, this is important because if the producer is node is 'older' than the consumer + // the consumer may overtake it (consume past it) invalidating the 'snapshot' notion of size. + LinkedQueueAtomicNode chaserNode = lvConsumerNode(); + LinkedQueueAtomicNode producerNode = lvProducerNode(); + int size = 0; + // must chase the nodes all the way to the producer node, but there's no need to count beyond expected head. + while (// don't go passed producer node + chaserNode != producerNode && // stop at last node + chaserNode != null && // stop at max int + size < Integer.MAX_VALUE) { + LinkedQueueAtomicNode next; + next = chaserNode.lvNext(); + // check if this node has been consumed, if so return what we have + if (next == chaserNode) { + return size; + } + chaserNode = next; + size++; + } + return size; + } + + /** + * {@inheritDoc}
+ *

+ * IMPLEMENTATION NOTES:
+ * Queue is empty when producerNode is the same as consumerNode. An alternative implementation would be to + * observe the producerNode.value is null, which also means an empty queue because only the + * consumerNode.value is allowed to be null. + * + * @see MessagePassingQueue#isEmpty() + */ + @Override + public boolean isEmpty() { + LinkedQueueAtomicNode consumerNode = lvConsumerNode(); + LinkedQueueAtomicNode producerNode = lvProducerNode(); + return consumerNode == producerNode; + } + + protected E getSingleConsumerNodeValue(LinkedQueueAtomicNode currConsumerNode, LinkedQueueAtomicNode nextNode) { + // we have to null out the value because we are going to hang on to the node + final E nextValue = nextNode.getAndNullValue(); + // Fix up the next ref of currConsumerNode to prevent promoted nodes from keeping new ones alive. + // We use a reference to self instead of null because null is already a meaningful value (the next of + // producer node is null). + currConsumerNode.soNext(currConsumerNode); + spConsumerNode(nextNode); + // currConsumerNode is now no longer referenced and can be collected + return nextValue; + } + + /** + * {@inheritDoc}
+ *

+ * IMPLEMENTATION NOTES:
+ * Poll is allowed from a SINGLE thread.
+ * Poll is potentially blocking here as the {@link Queue#poll()} does not allow returning {@code null} if the queue is not + * empty. This is very different from the original Vyukov guarantees. See {@link #relaxedPoll()} for the original + * semantics.
+ * Poll reads {@code consumerNode.next} and: + *

    + *
  1. If it is {@code null} AND the queue is empty return {@code null}, if queue is not empty spin wait for + * value to become visible. + *
  2. If it is not {@code null} set it as the consumer node and return it's now evacuated value. + *
+ * This means the consumerNode.value is always {@code null}, which is also the starting point for the queue. + * Because {@code null} values are not allowed to be offered this is the only node with it's value set to + * {@code null} at any one time. + * + * @see MessagePassingQueue#poll() + * @see java.util.Queue#poll() + */ + @Override + public E poll() { + final LinkedQueueAtomicNode currConsumerNode = lpConsumerNode(); + LinkedQueueAtomicNode nextNode = currConsumerNode.lvNext(); + if (nextNode != null) { + return getSingleConsumerNodeValue(currConsumerNode, nextNode); + } else if (currConsumerNode != lvProducerNode()) { + nextNode = spinWaitForNextNode(currConsumerNode); + // got the next node... + return getSingleConsumerNodeValue(currConsumerNode, nextNode); + } + return null; + } + + /** + * {@inheritDoc}
+ *

+ * IMPLEMENTATION NOTES:
+ * Peek is allowed from a SINGLE thread.
+ * Peek is potentially blocking here as the {@link Queue#peek()} does not allow returning {@code null} if the queue is not + * empty. This is very different from the original Vyukov guarantees. See {@link #relaxedPeek()} for the original + * semantics.
+ * Poll reads the next node from the consumerNode and: + *

    + *
  1. If it is {@code null} AND the queue is empty return {@code null}, if queue is not empty spin wait for + * value to become visible. + *
  2. If it is not {@code null} return it's value. + *
+ * + * @see MessagePassingQueue#peek() + * @see java.util.Queue#peek() + */ + @Override + public E peek() { + final LinkedQueueAtomicNode currConsumerNode = lpConsumerNode(); + LinkedQueueAtomicNode nextNode = currConsumerNode.lvNext(); + if (nextNode != null) { + return nextNode.lpValue(); + } else if (currConsumerNode != lvProducerNode()) { + nextNode = spinWaitForNextNode(currConsumerNode); + // got the next node... + return nextNode.lpValue(); + } + return null; + } + + LinkedQueueAtomicNode spinWaitForNextNode(LinkedQueueAtomicNode currNode) { + LinkedQueueAtomicNode nextNode; + while ((nextNode = currNode.lvNext()) == null) { + // spin, we are no longer wait free + } + return nextNode; + } + + @Override + public E relaxedPoll() { + final LinkedQueueAtomicNode currConsumerNode = lpConsumerNode(); + final LinkedQueueAtomicNode nextNode = currConsumerNode.lvNext(); + if (nextNode != null) { + return getSingleConsumerNodeValue(currConsumerNode, nextNode); + } + return null; + } + + @Override + public E relaxedPeek() { + final LinkedQueueAtomicNode nextNode = lpConsumerNode().lvNext(); + if (nextNode != null) { + return nextNode.lpValue(); + } + return null; + } + + @Override + public boolean relaxedOffer(E e) { + return offer(e); + } + + @Override + public int drain(Consumer c, int limit) { + if (null == c) + throw new IllegalArgumentException("c is null"); + if (limit < 0) + throw new IllegalArgumentException("limit is negative: " + limit); + if (limit == 0) + return 0; + LinkedQueueAtomicNode chaserNode = this.lpConsumerNode(); + for (int i = 0; i < limit; i++) { + final LinkedQueueAtomicNode nextNode = chaserNode.lvNext(); + if (nextNode == null) { + return i; + } + // we have to null out the value because we are going to hang on to the node + final E nextValue = getSingleConsumerNodeValue(chaserNode, nextNode); + chaserNode = nextNode; + c.accept(nextValue); + } + return limit; + } + + @Override + public int drain(Consumer c) { + return MessagePassingQueueUtil.drain(this, c); + } + + @Override + public void drain(Consumer c, WaitStrategy wait, ExitCondition exit) { + MessagePassingQueueUtil.drain(this, c, wait, exit); + } + + @Override + public int capacity() { + return UNBOUNDED_CAPACITY; + } +} diff --git a/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/BaseMpscLinkedAtomicUnpaddedArrayQueue.java b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/BaseMpscLinkedAtomicUnpaddedArrayQueue.java new file mode 100644 index 00000000..0ee152f4 --- /dev/null +++ b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/BaseMpscLinkedAtomicUnpaddedArrayQueue.java @@ -0,0 +1,650 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jctools.queues.atomic.unpadded; + +import org.jctools.queues.IndexedQueueSizeUtil.IndexedQueue; +import org.jctools.util.PortableJvmInfo; +import org.jctools.util.Pow2; +import org.jctools.util.RangeUtil; +import java.util.AbstractQueue; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.concurrent.atomic.*; +import org.jctools.queues.*; +import static org.jctools.queues.atomic.unpadded.AtomicQueueUtil.*; + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseMpscLinkedArrayQueue.java. + */ +abstract class BaseMpscLinkedAtomicUnpaddedArrayQueuePad1 extends AbstractQueue implements IndexedQueue { +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseMpscLinkedArrayQueue.java. + */ +abstract class BaseMpscLinkedAtomicUnpaddedArrayQueueProducerFields extends BaseMpscLinkedAtomicUnpaddedArrayQueuePad1 { + + private static final AtomicLongFieldUpdater P_INDEX_UPDATER = AtomicLongFieldUpdater.newUpdater(BaseMpscLinkedAtomicUnpaddedArrayQueueProducerFields.class, "producerIndex"); + + private volatile long producerIndex; + + @Override + public final long lvProducerIndex() { + return producerIndex; + } + + final void soProducerIndex(long newValue) { + P_INDEX_UPDATER.lazySet(this, newValue); + } + + final boolean casProducerIndex(long expect, long newValue) { + return P_INDEX_UPDATER.compareAndSet(this, expect, newValue); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseMpscLinkedArrayQueue.java. + */ +abstract class BaseMpscLinkedAtomicUnpaddedArrayQueuePad2 extends BaseMpscLinkedAtomicUnpaddedArrayQueueProducerFields { +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseMpscLinkedArrayQueue.java. + */ +abstract class BaseMpscLinkedAtomicUnpaddedArrayQueueConsumerFields extends BaseMpscLinkedAtomicUnpaddedArrayQueuePad2 { + + private static final AtomicLongFieldUpdater C_INDEX_UPDATER = AtomicLongFieldUpdater.newUpdater(BaseMpscLinkedAtomicUnpaddedArrayQueueConsumerFields.class, "consumerIndex"); + + private volatile long consumerIndex; + + protected long consumerMask; + + protected AtomicReferenceArray consumerBuffer; + + @Override + public final long lvConsumerIndex() { + return consumerIndex; + } + + final long lpConsumerIndex() { + return consumerIndex; + } + + final void soConsumerIndex(long newValue) { + C_INDEX_UPDATER.lazySet(this, newValue); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseMpscLinkedArrayQueue.java. + */ +abstract class BaseMpscLinkedAtomicUnpaddedArrayQueuePad3 extends BaseMpscLinkedAtomicUnpaddedArrayQueueConsumerFields { +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseMpscLinkedArrayQueue.java. + */ +abstract class BaseMpscLinkedAtomicUnpaddedArrayQueueColdProducerFields extends BaseMpscLinkedAtomicUnpaddedArrayQueuePad3 { + + private static final AtomicLongFieldUpdater P_LIMIT_UPDATER = AtomicLongFieldUpdater.newUpdater(BaseMpscLinkedAtomicUnpaddedArrayQueueColdProducerFields.class, "producerLimit"); + + private volatile long producerLimit; + + protected long producerMask; + + protected AtomicReferenceArray producerBuffer; + + final long lvProducerLimit() { + return producerLimit; + } + + final boolean casProducerLimit(long expect, long newValue) { + return P_LIMIT_UPDATER.compareAndSet(this, expect, newValue); + } + + final void soProducerLimit(long newValue) { + P_LIMIT_UPDATER.lazySet(this, newValue); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseMpscLinkedArrayQueue.java. + * + * An MPSC array queue which starts at initialCapacity and grows to maxCapacity in linked chunks + * of the initial size. The queue grows only when the current buffer is full and elements are not copied on + * resize, instead a link to the new buffer is stored in the old buffer for the consumer to follow. + */ +abstract class BaseMpscLinkedAtomicUnpaddedArrayQueue extends BaseMpscLinkedAtomicUnpaddedArrayQueueColdProducerFields implements MessagePassingQueue, QueueProgressIndicators { + + // No post padding here, subclasses must add + private static final Object JUMP = new Object(); + + private static final Object BUFFER_CONSUMED = new Object(); + + private static final int CONTINUE_TO_P_INDEX_CAS = 0; + + private static final int RETRY = 1; + + private static final int QUEUE_FULL = 2; + + private static final int QUEUE_RESIZE = 3; + + /** + * @param initialCapacity the queue initial capacity. If chunk size is fixed this will be the chunk size. + * Must be 2 or more. + */ + public BaseMpscLinkedAtomicUnpaddedArrayQueue(final int initialCapacity) { + RangeUtil.checkGreaterThanOrEqual(initialCapacity, 2, "initialCapacity"); + int p2capacity = Pow2.roundToPowerOfTwo(initialCapacity); + // leave lower bit of mask clear + long mask = (p2capacity - 1) << 1; + // need extra element to point at next array + AtomicReferenceArray buffer = allocateRefArray(p2capacity + 1); + producerBuffer = buffer; + producerMask = mask; + consumerBuffer = buffer; + consumerMask = mask; + // we know it's all empty to start with + soProducerLimit(mask); + } + + @Override + public int size() { + return IndexedQueueSizeUtil.size(this, IndexedQueueSizeUtil.IGNORE_PARITY_DIVISOR); + } + + @Override + public boolean isEmpty() { + // Order matters! + // Loading consumer before producer allows for producer increments after consumer index is read. + // This ensures this method is conservative in it's estimate. Note that as this is an MPMC there is + // nothing we can do to make this an exact method. + return ((lvConsumerIndex() - lvProducerIndex()) / 2 == 0); + } + + @Override + public String toString() { + return this.getClass().getName(); + } + + @Override + public boolean offer(final E e) { + if (null == e) { + throw new NullPointerException(); + } + long mask; + AtomicReferenceArray buffer; + long pIndex; + while (true) { + long producerLimit = lvProducerLimit(); + pIndex = lvProducerIndex(); + // lower bit is indicative of resize, if we see it we spin until it's cleared + if ((pIndex & 1) == 1) { + continue; + } + // pIndex is even (lower bit is 0) -> actual index is (pIndex >> 1) + // mask/buffer may get changed by resizing -> only use for array access after successful CAS. + mask = this.producerMask; + buffer = this.producerBuffer; + // a successful CAS ties the ordering, lv(pIndex) - [mask/buffer] -> cas(pIndex) + // assumption behind this optimization is that queue is almost always empty or near empty + if (producerLimit <= pIndex) { + int result = offerSlowPath(mask, pIndex, producerLimit); + switch(result) { + case CONTINUE_TO_P_INDEX_CAS: + break; + case RETRY: + continue; + case QUEUE_FULL: + return false; + case QUEUE_RESIZE: + resize(mask, buffer, pIndex, e, null); + return true; + } + } + if (casProducerIndex(pIndex, pIndex + 2)) { + break; + } + } + // INDEX visible before ELEMENT + final int offset = modifiedCalcCircularRefElementOffset(pIndex, mask); + // release element e + soRefElement(buffer, offset, e); + return true; + } + + /** + * {@inheritDoc} + *

+ * This implementation is correct for single consumer thread use only. + */ + @SuppressWarnings("unchecked") + @Override + public E poll() { + final AtomicReferenceArray buffer = consumerBuffer; + final long cIndex = lpConsumerIndex(); + final long mask = consumerMask; + final int offset = modifiedCalcCircularRefElementOffset(cIndex, mask); + Object e = lvRefElement(buffer, offset); + if (e == null) { + long pIndex = lvProducerIndex(); + // isEmpty? + if ((cIndex - pIndex) / 2 == 0) { + return null; + } + // poll() == null iff queue is empty, null element is not strong enough indicator, so we must + // spin until element is visible. + do { + e = lvRefElement(buffer, offset); + } while (e == null); + } + if (e == JUMP) { + final AtomicReferenceArray nextBuffer = nextBuffer(buffer, mask); + return newBufferPoll(nextBuffer, cIndex); + } + // release element null + soRefElement(buffer, offset, null); + // release cIndex + soConsumerIndex(cIndex + 2); + return (E) e; + } + + /** + * {@inheritDoc} + *

+ * This implementation is correct for single consumer thread use only. + */ + @SuppressWarnings("unchecked") + @Override + public E peek() { + final AtomicReferenceArray buffer = consumerBuffer; + final long cIndex = lpConsumerIndex(); + final long mask = consumerMask; + final int offset = modifiedCalcCircularRefElementOffset(cIndex, mask); + Object e = lvRefElement(buffer, offset); + if (e == null) { + long pIndex = lvProducerIndex(); + // isEmpty? + if ((cIndex - pIndex) / 2 == 0) { + return null; + } + // peek() == null iff queue is empty, null element is not strong enough indicator, so we must + // spin until element is visible. + do { + e = lvRefElement(buffer, offset); + } while (e == null); + } + if (e == JUMP) { + return newBufferPeek(nextBuffer(buffer, mask), cIndex); + } + return (E) e; + } + + /** + * We do not inline resize into this method because we do not resize on fill. + */ + private int offerSlowPath(long mask, long pIndex, long producerLimit) { + final long cIndex = lvConsumerIndex(); + long bufferCapacity = getCurrentBufferCapacity(mask); + if (cIndex + bufferCapacity > pIndex) { + if (!casProducerLimit(producerLimit, cIndex + bufferCapacity)) { + // retry from top + return RETRY; + } else { + // continue to pIndex CAS + return CONTINUE_TO_P_INDEX_CAS; + } + } else // full and cannot grow + if (availableInQueue(pIndex, cIndex) <= 0) { + // offer should return false; + return QUEUE_FULL; + } else // grab index for resize -> set lower bit + if (casProducerIndex(pIndex, pIndex + 1)) { + // trigger a resize + return QUEUE_RESIZE; + } else { + // failed resize attempt, retry from top + return RETRY; + } + } + + /** + * @return available elements in queue * 2 + */ + protected abstract long availableInQueue(long pIndex, long cIndex); + + @SuppressWarnings("unchecked") + private AtomicReferenceArray nextBuffer(final AtomicReferenceArray buffer, final long mask) { + final int offset = nextArrayOffset(mask); + final AtomicReferenceArray nextBuffer = (AtomicReferenceArray) lvRefElement(buffer, offset); + consumerBuffer = nextBuffer; + consumerMask = (length(nextBuffer) - 2) << 1; + soRefElement(buffer, offset, BUFFER_CONSUMED); + return nextBuffer; + } + + private static int nextArrayOffset(long mask) { + return modifiedCalcCircularRefElementOffset(mask + 2, Long.MAX_VALUE); + } + + private E newBufferPoll(AtomicReferenceArray nextBuffer, long cIndex) { + final int offset = modifiedCalcCircularRefElementOffset(cIndex, consumerMask); + final E n = lvRefElement(nextBuffer, offset); + if (n == null) { + throw new IllegalStateException("new buffer must have at least one element"); + } + soRefElement(nextBuffer, offset, null); + soConsumerIndex(cIndex + 2); + return n; + } + + private E newBufferPeek(AtomicReferenceArray nextBuffer, long cIndex) { + final int offset = modifiedCalcCircularRefElementOffset(cIndex, consumerMask); + final E n = lvRefElement(nextBuffer, offset); + if (null == n) { + throw new IllegalStateException("new buffer must have at least one element"); + } + return n; + } + + @Override + public long currentProducerIndex() { + return lvProducerIndex() / 2; + } + + @Override + public long currentConsumerIndex() { + return lvConsumerIndex() / 2; + } + + @Override + public abstract int capacity(); + + @Override + public boolean relaxedOffer(E e) { + return offer(e); + } + + @SuppressWarnings("unchecked") + @Override + public E relaxedPoll() { + final AtomicReferenceArray buffer = consumerBuffer; + final long cIndex = lpConsumerIndex(); + final long mask = consumerMask; + final int offset = modifiedCalcCircularRefElementOffset(cIndex, mask); + Object e = lvRefElement(buffer, offset); + if (e == null) { + return null; + } + if (e == JUMP) { + final AtomicReferenceArray nextBuffer = nextBuffer(buffer, mask); + return newBufferPoll(nextBuffer, cIndex); + } + soRefElement(buffer, offset, null); + soConsumerIndex(cIndex + 2); + return (E) e; + } + + @SuppressWarnings("unchecked") + @Override + public E relaxedPeek() { + final AtomicReferenceArray buffer = consumerBuffer; + final long cIndex = lpConsumerIndex(); + final long mask = consumerMask; + final int offset = modifiedCalcCircularRefElementOffset(cIndex, mask); + Object e = lvRefElement(buffer, offset); + if (e == JUMP) { + return newBufferPeek(nextBuffer(buffer, mask), cIndex); + } + return (E) e; + } + + @Override + public int fill(Supplier s) { + // result is a long because we want to have a safepoint check at regular intervals + long result = 0; + final int capacity = capacity(); + do { + final int filled = fill(s, PortableJvmInfo.RECOMENDED_OFFER_BATCH); + if (filled == 0) { + return (int) result; + } + result += filled; + } while (result <= capacity); + return (int) result; + } + + @Override + public int fill(Supplier s, int limit) { + if (null == s) + throw new IllegalArgumentException("supplier is null"); + if (limit < 0) + throw new IllegalArgumentException("limit is negative:" + limit); + if (limit == 0) + return 0; + long mask; + AtomicReferenceArray buffer; + long pIndex; + int claimedSlots; + while (true) { + long producerLimit = lvProducerLimit(); + pIndex = lvProducerIndex(); + // lower bit is indicative of resize, if we see it we spin until it's cleared + if ((pIndex & 1) == 1) { + continue; + } + // pIndex is even (lower bit is 0) -> actual index is (pIndex >> 1) + // NOTE: mask/buffer may get changed by resizing -> only use for array access after successful CAS. + // Only by virtue offloading them between the lvProducerIndex and a successful casProducerIndex are they + // safe to use. + mask = this.producerMask; + buffer = this.producerBuffer; + // a successful CAS ties the ordering, lv(pIndex) -> [mask/buffer] -> cas(pIndex) + // we want 'limit' slots, but will settle for whatever is visible to 'producerLimit' + // -> producerLimit >= batchIndex + long batchIndex = Math.min(producerLimit, pIndex + 2l * limit); + if (pIndex >= producerLimit) { + int result = offerSlowPath(mask, pIndex, producerLimit); + switch(result) { + case CONTINUE_TO_P_INDEX_CAS: + // offer slow path verifies only one slot ahead, we cannot rely on indication here + case RETRY: + continue; + case QUEUE_FULL: + return 0; + case QUEUE_RESIZE: + resize(mask, buffer, pIndex, null, s); + return 1; + } + } + // claim limit slots at once + if (casProducerIndex(pIndex, batchIndex)) { + claimedSlots = (int) ((batchIndex - pIndex) / 2); + break; + } + } + for (int i = 0; i < claimedSlots; i++) { + final int offset = modifiedCalcCircularRefElementOffset(pIndex + 2l * i, mask); + soRefElement(buffer, offset, s.get()); + } + return claimedSlots; + } + + @Override + public void fill(Supplier s, WaitStrategy wait, ExitCondition exit) { + MessagePassingQueueUtil.fill(this, s, wait, exit); + } + + @Override + public int drain(Consumer c) { + return drain(c, capacity()); + } + + @Override + public int drain(Consumer c, int limit) { + return MessagePassingQueueUtil.drain(this, c, limit); + } + + @Override + public void drain(Consumer c, WaitStrategy wait, ExitCondition exit) { + MessagePassingQueueUtil.drain(this, c, wait, exit); + } + + /** + * Get an iterator for this queue. This method is thread safe. + *

+ * The iterator provides a best-effort snapshot of the elements in the queue. + * The returned iterator is not guaranteed to return elements in queue order, + * and races with the consumer thread may cause gaps in the sequence of returned elements. + * Like {link #relaxedPoll}, the iterator may not immediately return newly inserted elements. + * + * @return The iterator. + */ + @Override + public Iterator iterator() { + return new WeakIterator(consumerBuffer, lvConsumerIndex(), lvProducerIndex()); + } + + /** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseMpscLinkedArrayQueue.java. + */ + private static class WeakIterator implements Iterator { + + private final long pIndex; + + private long nextIndex; + + private E nextElement; + + private AtomicReferenceArray currentBuffer; + + private int mask; + + WeakIterator(AtomicReferenceArray currentBuffer, long cIndex, long pIndex) { + this.pIndex = pIndex >> 1; + this.nextIndex = cIndex >> 1; + setBuffer(currentBuffer); + nextElement = getNext(); + } + + @Override + public void remove() { + throw new UnsupportedOperationException("remove"); + } + + @Override + public boolean hasNext() { + return nextElement != null; + } + + @Override + public E next() { + final E e = nextElement; + if (e == null) { + throw new NoSuchElementException(); + } + nextElement = getNext(); + return e; + } + + private void setBuffer(AtomicReferenceArray buffer) { + this.currentBuffer = buffer; + this.mask = length(buffer) - 2; + } + + private E getNext() { + while (nextIndex < pIndex) { + long index = nextIndex++; + E e = lvRefElement(currentBuffer, calcCircularRefElementOffset(index, mask)); + // skip removed/not yet visible elements + if (e == null) { + continue; + } + // not null && not JUMP -> found next element + if (e != JUMP) { + return e; + } + // need to jump to the next buffer + int nextBufferIndex = mask + 1; + Object nextBuffer = lvRefElement(currentBuffer, calcRefElementOffset(nextBufferIndex)); + if (nextBuffer == BUFFER_CONSUMED || nextBuffer == null) { + // Consumer may have passed us, or the next buffer is not visible yet: drop out early + return null; + } + setBuffer((AtomicReferenceArray) nextBuffer); + // now with the new array retry the load, it can't be a JUMP, but we need to repeat same index + e = lvRefElement(currentBuffer, calcCircularRefElementOffset(index, mask)); + // skip removed/not yet visible elements + if (e == null) { + continue; + } else { + return e; + } + } + return null; + } + } + + private void resize(long oldMask, AtomicReferenceArray oldBuffer, long pIndex, E e, Supplier s) { + assert (e != null && s == null) || (e == null || s != null); + int newBufferLength = getNextBufferSize(oldBuffer); + final AtomicReferenceArray newBuffer; + try { + newBuffer = allocateRefArray(newBufferLength); + } catch (OutOfMemoryError oom) { + assert lvProducerIndex() == pIndex + 1; + soProducerIndex(pIndex); + throw oom; + } + producerBuffer = newBuffer; + final int newMask = (newBufferLength - 2) << 1; + producerMask = newMask; + final int offsetInOld = modifiedCalcCircularRefElementOffset(pIndex, oldMask); + final int offsetInNew = modifiedCalcCircularRefElementOffset(pIndex, newMask); + // element in new array + soRefElement(newBuffer, offsetInNew, e == null ? s.get() : e); + // buffer linked + soRefElement(oldBuffer, nextArrayOffset(oldMask), newBuffer); + // ASSERT code + final long cIndex = lvConsumerIndex(); + final long availableInQueue = availableInQueue(pIndex, cIndex); + RangeUtil.checkPositive(availableInQueue, "availableInQueue"); + // Invalidate racing CASs + // We never set the limit beyond the bounds of a buffer + soProducerLimit(pIndex + Math.min(newMask, availableInQueue)); + // make resize visible to the other producers + soProducerIndex(pIndex + 2); + // INDEX visible before ELEMENT, consistent with consumer expectation + // make resize visible to consumer + soRefElement(oldBuffer, offsetInOld, JUMP); + } + + /** + * @return next buffer size(inclusive of next array pointer) + */ + protected abstract int getNextBufferSize(AtomicReferenceArray buffer); + + /** + * @return current buffer capacity for elements (excluding next pointer and jump entry) * 2 + */ + protected abstract long getCurrentBufferCapacity(long mask); +} diff --git a/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/BaseSpscLinkedAtomicUnpaddedArrayQueue.java b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/BaseSpscLinkedAtomicUnpaddedArrayQueue.java new file mode 100644 index 00000000..518a6563 --- /dev/null +++ b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/BaseSpscLinkedAtomicUnpaddedArrayQueue.java @@ -0,0 +1,351 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jctools.queues.atomic.unpadded; + +import org.jctools.queues.IndexedQueueSizeUtil.IndexedQueue; +import org.jctools.util.PortableJvmInfo; +import java.util.AbstractQueue; +import java.util.Iterator; +import java.util.concurrent.atomic.*; +import org.jctools.queues.*; +import static org.jctools.queues.atomic.unpadded.AtomicQueueUtil.*; + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseSpscLinkedArrayQueue.java. + */ +abstract class BaseSpscLinkedAtomicUnpaddedArrayQueuePrePad extends AbstractQueue implements IndexedQueue { +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseSpscLinkedArrayQueue.java. + */ +abstract class BaseSpscLinkedAtomicUnpaddedArrayQueueConsumerColdFields extends BaseSpscLinkedAtomicUnpaddedArrayQueuePrePad { + + protected long consumerMask; + + protected AtomicReferenceArray consumerBuffer; +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseSpscLinkedArrayQueue.java. + */ +abstract class BaseSpscLinkedAtomicUnpaddedArrayQueueConsumerField extends BaseSpscLinkedAtomicUnpaddedArrayQueueConsumerColdFields { + + private static final AtomicLongFieldUpdater C_INDEX_UPDATER = AtomicLongFieldUpdater.newUpdater(BaseSpscLinkedAtomicUnpaddedArrayQueueConsumerField.class, "consumerIndex"); + + private volatile long consumerIndex; + + @Override + public final long lvConsumerIndex() { + return consumerIndex; + } + + final long lpConsumerIndex() { + return consumerIndex; + } + + final void soConsumerIndex(long newValue) { + C_INDEX_UPDATER.lazySet(this, newValue); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseSpscLinkedArrayQueue.java. + */ +abstract class BaseSpscLinkedAtomicUnpaddedArrayQueueL2Pad extends BaseSpscLinkedAtomicUnpaddedArrayQueueConsumerField { +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseSpscLinkedArrayQueue.java. + */ +abstract class BaseSpscLinkedAtomicUnpaddedArrayQueueProducerFields extends BaseSpscLinkedAtomicUnpaddedArrayQueueL2Pad { + + private static final AtomicLongFieldUpdater P_INDEX_UPDATER = AtomicLongFieldUpdater.newUpdater(BaseSpscLinkedAtomicUnpaddedArrayQueueProducerFields.class, "producerIndex"); + + private volatile long producerIndex; + + @Override + public final long lvProducerIndex() { + return producerIndex; + } + + final void soProducerIndex(long newValue) { + P_INDEX_UPDATER.lazySet(this, newValue); + } + + final long lpProducerIndex() { + return producerIndex; + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseSpscLinkedArrayQueue.java. + */ +abstract class BaseSpscLinkedAtomicUnpaddedArrayQueueProducerColdFields extends BaseSpscLinkedAtomicUnpaddedArrayQueueProducerFields { + + protected long producerBufferLimit; + + // fixed for chunked and unbounded + protected long producerMask; + + protected AtomicReferenceArray producerBuffer; +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is BaseSpscLinkedArrayQueue.java. + */ +abstract class BaseSpscLinkedAtomicUnpaddedArrayQueue extends BaseSpscLinkedAtomicUnpaddedArrayQueueProducerColdFields implements MessagePassingQueue, QueueProgressIndicators { + + private static final Object JUMP = new Object(); + + @Override + public final Iterator iterator() { + throw new UnsupportedOperationException(); + } + + @Override + public final int size() { + return IndexedQueueSizeUtil.size(this, IndexedQueueSizeUtil.PLAIN_DIVISOR); + } + + @Override + public final boolean isEmpty() { + return IndexedQueueSizeUtil.isEmpty(this); + } + + @Override + public String toString() { + return this.getClass().getName(); + } + + @Override + public long currentProducerIndex() { + return lvProducerIndex(); + } + + @Override + public long currentConsumerIndex() { + return lvConsumerIndex(); + } + + protected final void soNext(AtomicReferenceArray curr, AtomicReferenceArray next) { + int offset = nextArrayOffset(curr); + soRefElement(curr, offset, next); + } + + @SuppressWarnings("unchecked") + protected final AtomicReferenceArray lvNextArrayAndUnlink(AtomicReferenceArray curr) { + final int offset = nextArrayOffset(curr); + final AtomicReferenceArray nextBuffer = (AtomicReferenceArray) lvRefElement(curr, offset); + // prevent GC nepotism + soRefElement(curr, offset, null); + return nextBuffer; + } + + @Override + public boolean relaxedOffer(E e) { + return offer(e); + } + + @Override + public E relaxedPoll() { + return poll(); + } + + @Override + public E relaxedPeek() { + return peek(); + } + + @Override + public int drain(Consumer c) { + return MessagePassingQueueUtil.drain(this, c); + } + + @Override + public int fill(Supplier s) { + // result is a long because we want to have a safepoint check at regular intervals + long result = 0; + final int capacity = capacity(); + do { + final int filled = fill(s, PortableJvmInfo.RECOMENDED_OFFER_BATCH); + if (filled == 0) { + return (int) result; + } + result += filled; + } while (result <= capacity); + return (int) result; + } + + @Override + public int drain(Consumer c, int limit) { + return MessagePassingQueueUtil.drain(this, c, limit); + } + + @Override + public int fill(Supplier s, int limit) { + if (null == s) + throw new IllegalArgumentException("supplier is null"); + if (limit < 0) + throw new IllegalArgumentException("limit is negative:" + limit); + if (limit == 0) + return 0; + for (int i = 0; i < limit; i++) { + // local load of field to avoid repeated loads after volatile reads + final AtomicReferenceArray buffer = producerBuffer; + final long index = lpProducerIndex(); + final long mask = producerMask; + final int offset = calcCircularRefElementOffset(index, mask); + // expected hot path + if (index < producerBufferLimit) { + writeToQueue(buffer, s.get(), index, offset); + } else { + if (!offerColdPath(buffer, mask, index, offset, null, s)) { + return i; + } + } + } + return limit; + } + + @Override + public void drain(Consumer c, WaitStrategy wait, ExitCondition exit) { + MessagePassingQueueUtil.drain(this, c, wait, exit); + } + + @Override + public void fill(Supplier s, WaitStrategy wait, ExitCondition exit) { + MessagePassingQueueUtil.fill(this, s, wait, exit); + } + + /** + * {@inheritDoc} + *

+ * This implementation is correct for single producer thread use only. + */ + @Override + public boolean offer(final E e) { + // Objects.requireNonNull(e); + if (null == e) { + throw new NullPointerException(); + } + // local load of field to avoid repeated loads after volatile reads + final AtomicReferenceArray buffer = producerBuffer; + final long index = lpProducerIndex(); + final long mask = producerMask; + final int offset = calcCircularRefElementOffset(index, mask); + // expected hot path + if (index < producerBufferLimit) { + writeToQueue(buffer, e, index, offset); + return true; + } + return offerColdPath(buffer, mask, index, offset, e, null); + } + + abstract boolean offerColdPath(AtomicReferenceArray buffer, long mask, long pIndex, int offset, E v, Supplier s); + + /** + * {@inheritDoc} + *

+ * This implementation is correct for single consumer thread use only. + */ + @SuppressWarnings("unchecked") + @Override + public E poll() { + // local load of field to avoid repeated loads after volatile reads + final AtomicReferenceArray buffer = consumerBuffer; + final long index = lpConsumerIndex(); + final long mask = consumerMask; + final int offset = calcCircularRefElementOffset(index, mask); + final Object e = lvRefElement(buffer, offset); + boolean isNextBuffer = e == JUMP; + if (null != e && !isNextBuffer) { + // this ensures correctness on 32bit platforms + soConsumerIndex(index + 1); + soRefElement(buffer, offset, null); + return (E) e; + } else if (isNextBuffer) { + return newBufferPoll(buffer, index); + } + return null; + } + + /** + * {@inheritDoc} + *

+ * This implementation is correct for single consumer thread use only. + */ + @SuppressWarnings("unchecked") + @Override + public E peek() { + final AtomicReferenceArray buffer = consumerBuffer; + final long index = lpConsumerIndex(); + final long mask = consumerMask; + final int offset = calcCircularRefElementOffset(index, mask); + final Object e = lvRefElement(buffer, offset); + if (e == JUMP) { + return newBufferPeek(buffer, index); + } + return (E) e; + } + + final void linkOldToNew(final long currIndex, final AtomicReferenceArray oldBuffer, final int offset, final AtomicReferenceArray newBuffer, final int offsetInNew, final E e) { + soRefElement(newBuffer, offsetInNew, e); + // link to next buffer and add next indicator as element of old buffer + soNext(oldBuffer, newBuffer); + soRefElement(oldBuffer, offset, JUMP); + // index is visible after elements (isEmpty/poll ordering) + // this ensures atomic write of long on 32bit platforms + soProducerIndex(currIndex + 1); + } + + final void writeToQueue(final AtomicReferenceArray buffer, final E e, final long index, final int offset) { + soRefElement(buffer, offset, e); + // this ensures atomic write of long on 32bit platforms + soProducerIndex(index + 1); + } + + private E newBufferPeek(final AtomicReferenceArray buffer, final long index) { + AtomicReferenceArray nextBuffer = lvNextArrayAndUnlink(buffer); + consumerBuffer = nextBuffer; + final long mask = length(nextBuffer) - 2; + consumerMask = mask; + final int offset = calcCircularRefElementOffset(index, mask); + return lvRefElement(nextBuffer, offset); + } + + private E newBufferPoll(final AtomicReferenceArray buffer, final long index) { + AtomicReferenceArray nextBuffer = lvNextArrayAndUnlink(buffer); + consumerBuffer = nextBuffer; + final long mask = length(nextBuffer) - 2; + consumerMask = mask; + final int offset = calcCircularRefElementOffset(index, mask); + final E n = lvRefElement(nextBuffer, offset); + if (null == n) { + throw new IllegalStateException("new buffer must have at least one element"); + } else { + // this ensures correctness on 32bit platforms + soConsumerIndex(index + 1); + soRefElement(nextBuffer, offset, null); + return n; + } + } +} diff --git a/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/LinkedQueueAtomicNode.java b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/LinkedQueueAtomicNode.java new file mode 100644 index 00000000..fa923e6a --- /dev/null +++ b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/LinkedQueueAtomicNode.java @@ -0,0 +1,69 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jctools.queues.atomic.unpadded; + +import java.util.concurrent.atomic.AtomicReference; + +public final class LinkedQueueAtomicNode extends AtomicReference> +{ + /** */ + private static final long serialVersionUID = 2404266111789071508L; + private E value; + + LinkedQueueAtomicNode() + { + } + + LinkedQueueAtomicNode(E val) + { + spValue(val); + } + + /** + * Gets the current value and nulls out the reference to it from this node. + * + * @return value + */ + public E getAndNullValue() + { + E temp = lpValue(); + spValue(null); + return temp; + } + + public E lpValue() + { + return value; + } + + public void spValue(E newValue) + { + value = newValue; + } + + public void soNext(LinkedQueueAtomicNode n) + { + lazySet(n); + } + + public void spNext(LinkedQueueAtomicNode n) + { + lazySet(n); + } + + public LinkedQueueAtomicNode lvNext() + { + return get(); + } +} diff --git a/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/MpmcAtomicUnpaddedArrayQueue.java b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/MpmcAtomicUnpaddedArrayQueue.java new file mode 100644 index 00000000..92e4fcdb --- /dev/null +++ b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/MpmcAtomicUnpaddedArrayQueue.java @@ -0,0 +1,510 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jctools.queues.atomic.unpadded; + +import org.jctools.util.RangeUtil; +import java.util.concurrent.atomic.*; +import org.jctools.queues.*; +import static org.jctools.queues.atomic.unpadded.AtomicQueueUtil.*; + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is MpmcArrayQueue.java. + */ +abstract class MpmcAtomicUnpaddedArrayQueueL1Pad extends SequencedAtomicReferenceArrayQueue { + + MpmcAtomicUnpaddedArrayQueueL1Pad(int capacity) { + super(capacity); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is MpmcArrayQueue.java. + */ +abstract class MpmcAtomicUnpaddedArrayQueueProducerIndexField extends MpmcAtomicUnpaddedArrayQueueL1Pad { + + private static final AtomicLongFieldUpdater P_INDEX_UPDATER = AtomicLongFieldUpdater.newUpdater(MpmcAtomicUnpaddedArrayQueueProducerIndexField.class, "producerIndex"); + + private volatile long producerIndex; + + MpmcAtomicUnpaddedArrayQueueProducerIndexField(int capacity) { + super(capacity); + } + + @Override + public final long lvProducerIndex() { + return producerIndex; + } + + final boolean casProducerIndex(long expect, long newValue) { + return P_INDEX_UPDATER.compareAndSet(this, expect, newValue); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is MpmcArrayQueue.java. + */ +abstract class MpmcAtomicUnpaddedArrayQueueL2Pad extends MpmcAtomicUnpaddedArrayQueueProducerIndexField { + + MpmcAtomicUnpaddedArrayQueueL2Pad(int capacity) { + super(capacity); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is MpmcArrayQueue.java. + */ +abstract class MpmcAtomicUnpaddedArrayQueueConsumerIndexField extends MpmcAtomicUnpaddedArrayQueueL2Pad { + + private static final AtomicLongFieldUpdater C_INDEX_UPDATER = AtomicLongFieldUpdater.newUpdater(MpmcAtomicUnpaddedArrayQueueConsumerIndexField.class, "consumerIndex"); + + private volatile long consumerIndex; + + MpmcAtomicUnpaddedArrayQueueConsumerIndexField(int capacity) { + super(capacity); + } + + @Override + public final long lvConsumerIndex() { + return consumerIndex; + } + + final boolean casConsumerIndex(long expect, long newValue) { + return C_INDEX_UPDATER.compareAndSet(this, expect, newValue); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is MpmcArrayQueue.java. + */ +abstract class MpmcAtomicUnpaddedArrayQueueL3Pad extends MpmcAtomicUnpaddedArrayQueueConsumerIndexField { + + MpmcAtomicUnpaddedArrayQueueL3Pad(int capacity) { + super(capacity); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is MpmcArrayQueue.java. + * + * A Multi-Producer-Multi-Consumer queue based on a {@link org.jctools.queues.ConcurrentCircularArrayQueue}. This + * implies that any and all threads may call the offer/poll/peek methods and correctness is maintained.
+ * This implementation follows patterns documented on the package level for False Sharing protection.
+ * The algorithm for offer/poll is an adaptation of the one put forward by D. Vyukov (See here). The original + * algorithm uses an array of structs which should offer nice locality properties but is sadly not possible in + * Java (waiting on Value Types or similar). The alternative explored here utilizes 2 arrays, one for each + * field of the struct. There is a further alternative in the experimental project which uses iteration phase + * markers to achieve the same algo and is closer structurally to the original, but sadly does not perform as + * well as this implementation.
+ *

+ * Tradeoffs to keep in mind: + *

    + *
  1. Padding for false sharing: counter fields and queue fields are all padded as well as either side of + * both arrays. We are trading memory to avoid false sharing(active and passive). + *
  2. 2 arrays instead of one: The algorithm requires an extra array of longs matching the size of the + * elements array. This is doubling/tripling the memory allocated for the buffer. + *
  3. Power of 2 capacity: Actual elements buffer (and sequence buffer) is the closest power of 2 larger or + * equal to the requested capacity. + *
+ */ +public class MpmcAtomicUnpaddedArrayQueue extends MpmcAtomicUnpaddedArrayQueueL3Pad { + + public static final int MAX_LOOK_AHEAD_STEP = Integer.getInteger("jctools.mpmc.max.lookahead.step", 4096); + + private final int lookAheadStep; + + public MpmcAtomicUnpaddedArrayQueue(final int capacity) { + super(RangeUtil.checkGreaterThanOrEqual(capacity, 2, "capacity")); + lookAheadStep = Math.max(2, Math.min(capacity() / 4, MAX_LOOK_AHEAD_STEP)); + } + + @Override + public boolean offer(final E e) { + if (null == e) { + throw new NullPointerException(); + } + final int mask = this.mask; + final long capacity = mask + 1; + final AtomicLongArray sBuffer = sequenceBuffer; + long pIndex; + int seqOffset; + long seq; + // start with bogus value, hope we don't need it + long cIndex = Long.MIN_VALUE; + do { + pIndex = lvProducerIndex(); + seqOffset = calcCircularLongElementOffset(pIndex, mask); + seq = lvLongElement(sBuffer, seqOffset); + // consumer has not moved this seq forward, it's as last producer left + if (seq < pIndex) { + // Extra check required to ensure [Queue.offer == false iff queue is full] + if (// test against cached cIndex + pIndex - capacity >= cIndex && // test against latest cIndex + pIndex - capacity >= (cIndex = lvConsumerIndex())) { + return false; + } else { + // (+) hack to make it go around again without CAS + seq = pIndex + 1; + } + } + } while (// another producer has moved the sequence(or +) + seq > pIndex || // failed to increment + !casProducerIndex(pIndex, pIndex + 1)); + // casProducerIndex ensures correct construction + spRefElement(buffer, calcCircularRefElementOffset(pIndex, mask), e); + // seq++; + soLongElement(sBuffer, seqOffset, pIndex + 1); + return true; + } + + /** + * {@inheritDoc} + *

+ * Because return null indicates queue is empty we cannot simply rely on next element visibility for poll + * and must test producer index when next element is not visible. + */ + @Override + public E poll() { + // local load of field to avoid repeated loads after volatile reads + final AtomicLongArray sBuffer = sequenceBuffer; + final int mask = this.mask; + long cIndex; + long seq; + int seqOffset; + long expectedSeq; + // start with bogus value, hope we don't need it + long pIndex = -1; + do { + cIndex = lvConsumerIndex(); + seqOffset = calcCircularLongElementOffset(cIndex, mask); + seq = lvLongElement(sBuffer, seqOffset); + expectedSeq = cIndex + 1; + if (seq < expectedSeq) { + // slot has not been moved by producer + if (// test against cached pIndex + cIndex >= pIndex && // update pIndex if we must + cIndex == (pIndex = lvProducerIndex())) { + // strict empty check, this ensures [Queue.poll() == null iff isEmpty()] + return null; + } else { + // trip another go around + seq = expectedSeq + 1; + } + } + } while (// another consumer beat us to it + seq > expectedSeq || // failed the CAS + !casConsumerIndex(cIndex, cIndex + 1)); + final int offset = calcCircularRefElementOffset(cIndex, mask); + final E e = lpRefElement(buffer, offset); + spRefElement(buffer, offset, null); + // i.e. seq += capacity + soLongElement(sBuffer, seqOffset, cIndex + mask + 1); + return e; + } + + @Override + public E peek() { + // local load of field to avoid repeated loads after volatile reads + final AtomicLongArray sBuffer = sequenceBuffer; + final int mask = this.mask; + long cIndex; + long seq; + int seqOffset; + long expectedSeq; + // start with bogus value, hope we don't need it + long pIndex = -1; + E e; + while (true) { + cIndex = lvConsumerIndex(); + seqOffset = calcCircularLongElementOffset(cIndex, mask); + seq = lvLongElement(sBuffer, seqOffset); + expectedSeq = cIndex + 1; + if (seq < expectedSeq) { + // slot has not been moved by producer + if (// test against cached pIndex + cIndex >= pIndex && // update pIndex if we must + cIndex == (pIndex = lvProducerIndex())) { + // strict empty check, this ensures [Queue.poll() == null iff isEmpty()] + return null; + } + } else if (seq == expectedSeq) { + final int offset = calcCircularRefElementOffset(cIndex, mask); + e = lvRefElement(buffer, offset); + if (lvConsumerIndex() == cIndex) + return e; + } + } + } + + @Override + public boolean relaxedOffer(E e) { + if (null == e) { + throw new NullPointerException(); + } + final int mask = this.mask; + final AtomicLongArray sBuffer = sequenceBuffer; + long pIndex; + int seqOffset; + long seq; + do { + pIndex = lvProducerIndex(); + seqOffset = calcCircularLongElementOffset(pIndex, mask); + seq = lvLongElement(sBuffer, seqOffset); + if (seq < pIndex) { + // slot not cleared by consumer yet + return false; + } + } while (// another producer has moved the sequence + seq > pIndex || // failed to increment + !casProducerIndex(pIndex, pIndex + 1)); + // casProducerIndex ensures correct construction + spRefElement(buffer, calcCircularRefElementOffset(pIndex, mask), e); + soLongElement(sBuffer, seqOffset, pIndex + 1); + return true; + } + + @Override + public E relaxedPoll() { + final AtomicLongArray sBuffer = sequenceBuffer; + final int mask = this.mask; + long cIndex; + int seqOffset; + long seq; + long expectedSeq; + do { + cIndex = lvConsumerIndex(); + seqOffset = calcCircularLongElementOffset(cIndex, mask); + seq = lvLongElement(sBuffer, seqOffset); + expectedSeq = cIndex + 1; + if (seq < expectedSeq) { + return null; + } + } while (// another consumer beat us to it + seq > expectedSeq || // failed the CAS + !casConsumerIndex(cIndex, cIndex + 1)); + final int offset = calcCircularRefElementOffset(cIndex, mask); + final E e = lpRefElement(buffer, offset); + spRefElement(buffer, offset, null); + soLongElement(sBuffer, seqOffset, cIndex + mask + 1); + return e; + } + + @Override + public E relaxedPeek() { + // local load of field to avoid repeated loads after volatile reads + final AtomicLongArray sBuffer = sequenceBuffer; + final int mask = this.mask; + long cIndex; + long seq; + int seqOffset; + long expectedSeq; + E e; + do { + cIndex = lvConsumerIndex(); + seqOffset = calcCircularLongElementOffset(cIndex, mask); + seq = lvLongElement(sBuffer, seqOffset); + expectedSeq = cIndex + 1; + if (seq < expectedSeq) { + return null; + } else if (seq == expectedSeq) { + final int offset = calcCircularRefElementOffset(cIndex, mask); + e = lvRefElement(buffer, offset); + if (lvConsumerIndex() == cIndex) + return e; + } + } while (true); + } + + @Override + public int drain(Consumer c, int limit) { + if (null == c) + throw new IllegalArgumentException("c is null"); + if (limit < 0) + throw new IllegalArgumentException("limit is negative: " + limit); + if (limit == 0) + return 0; + final AtomicLongArray sBuffer = sequenceBuffer; + final int mask = this.mask; + final AtomicReferenceArray buffer = this.buffer; + final int maxLookAheadStep = Math.min(this.lookAheadStep, limit); + int consumed = 0; + while (consumed < limit) { + final int remaining = limit - consumed; + final int lookAheadStep = Math.min(remaining, maxLookAheadStep); + final long cIndex = lvConsumerIndex(); + final long lookAheadIndex = cIndex + lookAheadStep - 1; + final int lookAheadSeqOffset = calcCircularLongElementOffset(lookAheadIndex, mask); + final long lookAheadSeq = lvLongElement(sBuffer, lookAheadSeqOffset); + final long expectedLookAheadSeq = lookAheadIndex + 1; + if (lookAheadSeq == expectedLookAheadSeq && casConsumerIndex(cIndex, expectedLookAheadSeq)) { + for (int i = 0; i < lookAheadStep; i++) { + final long index = cIndex + i; + final int seqOffset = calcCircularLongElementOffset(index, mask); + final int offset = calcCircularRefElementOffset(index, mask); + final long expectedSeq = index + 1; + while (lvLongElement(sBuffer, seqOffset) != expectedSeq) { + } + final E e = lpRefElement(buffer, offset); + spRefElement(buffer, offset, null); + soLongElement(sBuffer, seqOffset, index + mask + 1); + c.accept(e); + } + consumed += lookAheadStep; + } else { + if (lookAheadSeq < expectedLookAheadSeq) { + if (notAvailable(cIndex, mask, sBuffer, cIndex + 1)) { + return consumed; + } + } + return consumed + drainOneByOne(c, remaining); + } + } + return limit; + } + + private int drainOneByOne(Consumer c, int limit) { + final AtomicLongArray sBuffer = sequenceBuffer; + final int mask = this.mask; + final AtomicReferenceArray buffer = this.buffer; + long cIndex; + int seqOffset; + long seq; + long expectedSeq; + for (int i = 0; i < limit; i++) { + do { + cIndex = lvConsumerIndex(); + seqOffset = calcCircularLongElementOffset(cIndex, mask); + seq = lvLongElement(sBuffer, seqOffset); + expectedSeq = cIndex + 1; + if (seq < expectedSeq) { + return i; + } + } while (// another consumer beat us to it + seq > expectedSeq || // failed the CAS + !casConsumerIndex(cIndex, cIndex + 1)); + final int offset = calcCircularRefElementOffset(cIndex, mask); + final E e = lpRefElement(buffer, offset); + spRefElement(buffer, offset, null); + soLongElement(sBuffer, seqOffset, cIndex + mask + 1); + c.accept(e); + } + return limit; + } + + @Override + public int fill(Supplier s, int limit) { + if (null == s) + throw new IllegalArgumentException("supplier is null"); + if (limit < 0) + throw new IllegalArgumentException("limit is negative:" + limit); + if (limit == 0) + return 0; + final AtomicLongArray sBuffer = sequenceBuffer; + final int mask = this.mask; + final AtomicReferenceArray buffer = this.buffer; + final int maxLookAheadStep = Math.min(this.lookAheadStep, limit); + int produced = 0; + while (produced < limit) { + final int remaining = limit - produced; + final int lookAheadStep = Math.min(remaining, maxLookAheadStep); + final long pIndex = lvProducerIndex(); + final long lookAheadIndex = pIndex + lookAheadStep - 1; + final int lookAheadSeqOffset = calcCircularLongElementOffset(lookAheadIndex, mask); + final long lookAheadSeq = lvLongElement(sBuffer, lookAheadSeqOffset); + final long expectedLookAheadSeq = lookAheadIndex; + if (lookAheadSeq == expectedLookAheadSeq && casProducerIndex(pIndex, expectedLookAheadSeq + 1)) { + for (int i = 0; i < lookAheadStep; i++) { + final long index = pIndex + i; + final int seqOffset = calcCircularLongElementOffset(index, mask); + final int offset = calcCircularRefElementOffset(index, mask); + while (lvLongElement(sBuffer, seqOffset) != index) { + } + // Ordered store ensures correct construction + soRefElement(buffer, offset, s.get()); + soLongElement(sBuffer, seqOffset, index + 1); + } + produced += lookAheadStep; + } else { + if (lookAheadSeq < expectedLookAheadSeq) { + if (notAvailable(pIndex, mask, sBuffer, pIndex)) { + return produced; + } + } + return produced + fillOneByOne(s, remaining); + } + } + return limit; + } + + private boolean notAvailable(long index, int mask, AtomicLongArray sBuffer, long expectedSeq) { + final int seqOffset = calcCircularLongElementOffset(index, mask); + final long seq = lvLongElement(sBuffer, seqOffset); + if (seq < expectedSeq) { + return true; + } + return false; + } + + private int fillOneByOne(Supplier s, int limit) { + final AtomicLongArray sBuffer = sequenceBuffer; + final int mask = this.mask; + final AtomicReferenceArray buffer = this.buffer; + long pIndex; + int seqOffset; + long seq; + for (int i = 0; i < limit; i++) { + do { + pIndex = lvProducerIndex(); + seqOffset = calcCircularLongElementOffset(pIndex, mask); + seq = lvLongElement(sBuffer, seqOffset); + if (seq < pIndex) { + // slot not cleared by consumer yet + return i; + } + } while (// another producer has moved the sequence + seq > pIndex || // failed to increment + !casProducerIndex(pIndex, pIndex + 1)); + // Ordered store ensures correct construction + soRefElement(buffer, calcCircularRefElementOffset(pIndex, mask), s.get()); + soLongElement(sBuffer, seqOffset, pIndex + 1); + } + return limit; + } + + @Override + public int drain(Consumer c) { + return MessagePassingQueueUtil.drain(this, c); + } + + @Override + public int fill(Supplier s) { + return MessagePassingQueueUtil.fillBounded(this, s); + } + + @Override + public void drain(Consumer c, WaitStrategy w, ExitCondition exit) { + MessagePassingQueueUtil.drain(this, c, w, exit); + } + + @Override + public void fill(Supplier s, WaitStrategy wait, ExitCondition exit) { + MessagePassingQueueUtil.fill(this, s, wait, exit); + } +} diff --git a/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/MpscAtomicUnpaddedArrayQueue.java b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/MpscAtomicUnpaddedArrayQueue.java new file mode 100644 index 00000000..c4b82c84 --- /dev/null +++ b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/MpscAtomicUnpaddedArrayQueue.java @@ -0,0 +1,476 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jctools.queues.atomic.unpadded; + +import java.util.concurrent.atomic.*; +import org.jctools.queues.*; +import static org.jctools.queues.atomic.unpadded.AtomicQueueUtil.*; + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is MpscArrayQueue.java. + */ +abstract class MpscAtomicUnpaddedArrayQueueL1Pad extends AtomicReferenceArrayQueue { + + MpscAtomicUnpaddedArrayQueueL1Pad(int capacity) { + super(capacity); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is MpscArrayQueue.java. + */ +abstract class MpscAtomicUnpaddedArrayQueueProducerIndexField extends MpscAtomicUnpaddedArrayQueueL1Pad { + + private static final AtomicLongFieldUpdater P_INDEX_UPDATER = AtomicLongFieldUpdater.newUpdater(MpscAtomicUnpaddedArrayQueueProducerIndexField.class, "producerIndex"); + + private volatile long producerIndex; + + MpscAtomicUnpaddedArrayQueueProducerIndexField(int capacity) { + super(capacity); + } + + @Override + public final long lvProducerIndex() { + return producerIndex; + } + + final boolean casProducerIndex(long expect, long newValue) { + return P_INDEX_UPDATER.compareAndSet(this, expect, newValue); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is MpscArrayQueue.java. + */ +abstract class MpscAtomicUnpaddedArrayQueueMidPad extends MpscAtomicUnpaddedArrayQueueProducerIndexField { + + MpscAtomicUnpaddedArrayQueueMidPad(int capacity) { + super(capacity); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is MpscArrayQueue.java. + */ +abstract class MpscAtomicUnpaddedArrayQueueProducerLimitField extends MpscAtomicUnpaddedArrayQueueMidPad { + + private static final AtomicLongFieldUpdater P_LIMIT_UPDATER = AtomicLongFieldUpdater.newUpdater(MpscAtomicUnpaddedArrayQueueProducerLimitField.class, "producerLimit"); + + // First unavailable index the producer may claim up to before rereading the consumer index + private volatile long producerLimit; + + MpscAtomicUnpaddedArrayQueueProducerLimitField(int capacity) { + super(capacity); + this.producerLimit = capacity; + } + + final long lvProducerLimit() { + return producerLimit; + } + + final void soProducerLimit(long newValue) { + P_LIMIT_UPDATER.lazySet(this, newValue); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is MpscArrayQueue.java. + */ +abstract class MpscAtomicUnpaddedArrayQueueL2Pad extends MpscAtomicUnpaddedArrayQueueProducerLimitField { + + MpscAtomicUnpaddedArrayQueueL2Pad(int capacity) { + super(capacity); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is MpscArrayQueue.java. + */ +abstract class MpscAtomicUnpaddedArrayQueueConsumerIndexField extends MpscAtomicUnpaddedArrayQueueL2Pad { + + private static final AtomicLongFieldUpdater C_INDEX_UPDATER = AtomicLongFieldUpdater.newUpdater(MpscAtomicUnpaddedArrayQueueConsumerIndexField.class, "consumerIndex"); + + private volatile long consumerIndex; + + MpscAtomicUnpaddedArrayQueueConsumerIndexField(int capacity) { + super(capacity); + } + + @Override + public final long lvConsumerIndex() { + return consumerIndex; + } + + final long lpConsumerIndex() { + return consumerIndex; + } + + final void soConsumerIndex(long newValue) { + C_INDEX_UPDATER.lazySet(this, newValue); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is MpscArrayQueue.java. + */ +abstract class MpscAtomicUnpaddedArrayQueueL3Pad extends MpscAtomicUnpaddedArrayQueueConsumerIndexField { + + MpscAtomicUnpaddedArrayQueueL3Pad(int capacity) { + super(capacity); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is MpscArrayQueue.java. + * + * A Multi-Producer-Single-Consumer queue based on a {@link org.jctools.queues.ConcurrentCircularArrayQueue}. This + * implies that any thread may call the offer method, but only a single thread may call poll/peek for correctness to + * maintained.
+ * This implementation follows patterns documented on the package level for False Sharing protection.
+ * This implementation is using the Fast Flow + * method for polling from the queue (with minor change to correctly publish the index) and an extension of + * the Leslie Lamport concurrent queue algorithm (originated by Martin Thompson) on the producer side. + */ +public class MpscAtomicUnpaddedArrayQueue extends MpscAtomicUnpaddedArrayQueueL3Pad { + + public MpscAtomicUnpaddedArrayQueue(final int capacity) { + super(capacity); + } + + /** + * {@link #offer}} if {@link #size()} is less than threshold. + * + * @param e the object to offer onto the queue, not null + * @param threshold the maximum allowable size + * @return true if the offer is successful, false if queue size exceeds threshold + * @since 1.0.1 + */ + public boolean offerIfBelowThreshold(final E e, int threshold) { + if (null == e) { + throw new NullPointerException(); + } + final int mask = this.mask; + final long capacity = mask + 1; + long producerLimit = lvProducerLimit(); + long pIndex; + do { + pIndex = lvProducerIndex(); + long available = producerLimit - pIndex; + long size = capacity - available; + if (size >= threshold) { + final long cIndex = lvConsumerIndex(); + size = pIndex - cIndex; + if (size >= threshold) { + // the size exceeds threshold + return false; + } else { + // update producer limit to the next index that we must recheck the consumer index + producerLimit = cIndex + capacity; + // this is racy, but the race is benign + soProducerLimit(producerLimit); + } + } + } while (!casProducerIndex(pIndex, pIndex + 1)); + /* + * NOTE: the new producer index value is made visible BEFORE the element in the array. If we relied on + * the index visibility to poll() we would need to handle the case where the element is not visible. + */ + // Won CAS, move on to storing + final int offset = calcCircularRefElementOffset(pIndex, mask); + soRefElement(buffer, offset, e); + // AWESOME :) + return true; + } + + /** + * {@inheritDoc}
+ *

+ * IMPLEMENTATION NOTES:
+ * Lock free offer using a single CAS. As class name suggests access is permitted to many threads + * concurrently. + * + * @see java.util.Queue#offer + * @see org.jctools.queues.MessagePassingQueue#offer + */ + @Override + public boolean offer(final E e) { + if (null == e) { + throw new NullPointerException(); + } + // use a cached view on consumer index (potentially updated in loop) + final int mask = this.mask; + long producerLimit = lvProducerLimit(); + long pIndex; + do { + pIndex = lvProducerIndex(); + if (pIndex >= producerLimit) { + final long cIndex = lvConsumerIndex(); + producerLimit = cIndex + mask + 1; + if (pIndex >= producerLimit) { + // FULL :( + return false; + } else { + // update producer limit to the next index that we must recheck the consumer index + // this is racy, but the race is benign + soProducerLimit(producerLimit); + } + } + } while (!casProducerIndex(pIndex, pIndex + 1)); + /* + * NOTE: the new producer index value is made visible BEFORE the element in the array. If we relied on + * the index visibility to poll() we would need to handle the case where the element is not visible. + */ + // Won CAS, move on to storing + final int offset = calcCircularRefElementOffset(pIndex, mask); + soRefElement(buffer, offset, e); + // AWESOME :) + return true; + } + + /** + * A wait free alternative to offer which fails on CAS failure. + * + * @param e new element, not null + * @return 1 if next element cannot be filled, -1 if CAS failed, 0 if successful + */ + public final int failFastOffer(final E e) { + if (null == e) { + throw new NullPointerException(); + } + final int mask = this.mask; + final long capacity = mask + 1; + final long pIndex = lvProducerIndex(); + long producerLimit = lvProducerLimit(); + if (pIndex >= producerLimit) { + final long cIndex = lvConsumerIndex(); + producerLimit = cIndex + capacity; + if (pIndex >= producerLimit) { + // FULL :( + return 1; + } else { + // update producer limit to the next index that we must recheck the consumer index + soProducerLimit(producerLimit); + } + } + // look Ma, no loop! + if (!casProducerIndex(pIndex, pIndex + 1)) { + // CAS FAIL :( + return -1; + } + // Won CAS, move on to storing + final int offset = calcCircularRefElementOffset(pIndex, mask); + soRefElement(buffer, offset, e); + // AWESOME :) + return 0; + } + + /** + * {@inheritDoc} + *

+ * IMPLEMENTATION NOTES:
+ * Lock free poll using ordered loads/stores. As class name suggests access is limited to a single thread. + * + * @see java.util.Queue#poll + * @see org.jctools.queues.MessagePassingQueue#poll + */ + @Override + public E poll() { + final long cIndex = lpConsumerIndex(); + final int offset = calcCircularRefElementOffset(cIndex, mask); + // Copy field to avoid re-reading after volatile load + final AtomicReferenceArray buffer = this.buffer; + // If we can't see the next available element we can't poll + E e = lvRefElement(buffer, offset); + if (null == e) { + /* + * NOTE: Queue may not actually be empty in the case of a producer (P1) being interrupted after + * winning the CAS on offer but before storing the element in the queue. Other producers may go on + * to fill up the queue after this element. + */ + if (cIndex != lvProducerIndex()) { + do { + e = lvRefElement(buffer, offset); + } while (e == null); + } else { + return null; + } + } + spRefElement(buffer, offset, null); + soConsumerIndex(cIndex + 1); + return e; + } + + /** + * {@inheritDoc} + *

+ * IMPLEMENTATION NOTES:
+ * Lock free peek using ordered loads. As class name suggests access is limited to a single thread. + * + * @see java.util.Queue#poll + * @see org.jctools.queues.MessagePassingQueue#poll + */ + @Override + public E peek() { + // Copy field to avoid re-reading after volatile load + final AtomicReferenceArray buffer = this.buffer; + final long cIndex = lpConsumerIndex(); + final int offset = calcCircularRefElementOffset(cIndex, mask); + E e = lvRefElement(buffer, offset); + if (null == e) { + /* + * NOTE: Queue may not actually be empty in the case of a producer (P1) being interrupted after + * winning the CAS on offer but before storing the element in the queue. Other producers may go on + * to fill up the queue after this element. + */ + if (cIndex != lvProducerIndex()) { + do { + e = lvRefElement(buffer, offset); + } while (e == null); + } else { + return null; + } + } + return e; + } + + @Override + public boolean relaxedOffer(E e) { + return offer(e); + } + + @Override + public E relaxedPoll() { + final AtomicReferenceArray buffer = this.buffer; + final long cIndex = lpConsumerIndex(); + final int offset = calcCircularRefElementOffset(cIndex, mask); + // If we can't see the next available element we can't poll + E e = lvRefElement(buffer, offset); + if (null == e) { + return null; + } + spRefElement(buffer, offset, null); + soConsumerIndex(cIndex + 1); + return e; + } + + @Override + public E relaxedPeek() { + final AtomicReferenceArray buffer = this.buffer; + final int mask = this.mask; + final long cIndex = lpConsumerIndex(); + return lvRefElement(buffer, calcCircularRefElementOffset(cIndex, mask)); + } + + @Override + public int drain(final Consumer c, final int limit) { + if (null == c) + throw new IllegalArgumentException("c is null"); + if (limit < 0) + throw new IllegalArgumentException("limit is negative: " + limit); + if (limit == 0) + return 0; + final AtomicReferenceArray buffer = this.buffer; + final int mask = this.mask; + final long cIndex = lpConsumerIndex(); + for (int i = 0; i < limit; i++) { + final long index = cIndex + i; + final int offset = calcCircularRefElementOffset(index, mask); + final E e = lvRefElement(buffer, offset); + if (null == e) { + return i; + } + spRefElement(buffer, offset, null); + // ordered store -> atomic and ordered for size() + soConsumerIndex(index + 1); + c.accept(e); + } + return limit; + } + + @Override + public int fill(Supplier s, int limit) { + if (null == s) + throw new IllegalArgumentException("supplier is null"); + if (limit < 0) + throw new IllegalArgumentException("limit is negative:" + limit); + if (limit == 0) + return 0; + final int mask = this.mask; + final long capacity = mask + 1; + long producerLimit = lvProducerLimit(); + long pIndex; + int actualLimit; + do { + pIndex = lvProducerIndex(); + long available = producerLimit - pIndex; + if (available <= 0) { + final long cIndex = lvConsumerIndex(); + producerLimit = cIndex + capacity; + available = producerLimit - pIndex; + if (available <= 0) { + // FULL :( + return 0; + } else { + // update producer limit to the next index that we must recheck the consumer index + soProducerLimit(producerLimit); + } + } + actualLimit = Math.min((int) available, limit); + } while (!casProducerIndex(pIndex, pIndex + actualLimit)); + // right, now we claimed a few slots and can fill them with goodness + final AtomicReferenceArray buffer = this.buffer; + for (int i = 0; i < actualLimit; i++) { + // Won CAS, move on to storing + final int offset = calcCircularRefElementOffset(pIndex + i, mask); + soRefElement(buffer, offset, s.get()); + } + return actualLimit; + } + + @Override + public int drain(Consumer c) { + return drain(c, capacity()); + } + + @Override + public int fill(Supplier s) { + return MessagePassingQueueUtil.fillBounded(this, s); + } + + @Override + public void drain(Consumer c, WaitStrategy w, ExitCondition exit) { + MessagePassingQueueUtil.drain(this, c, w, exit); + } + + @Override + public void fill(Supplier s, WaitStrategy wait, ExitCondition exit) { + MessagePassingQueueUtil.fill(this, s, wait, exit); + } + + /** + * @deprecated This was renamed to failFastOffer please migrate + */ + @Deprecated + public int weakOffer(E e) { + return this.failFastOffer(e); + } +} diff --git a/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/MpscChunkedAtomicUnpaddedArrayQueue.java b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/MpscChunkedAtomicUnpaddedArrayQueue.java new file mode 100644 index 00000000..48bc5837 --- /dev/null +++ b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/MpscChunkedAtomicUnpaddedArrayQueue.java @@ -0,0 +1,85 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jctools.queues.atomic.unpadded; + +import org.jctools.util.Pow2; +import org.jctools.util.RangeUtil; +import static java.lang.Math.max; +import static java.lang.Math.min; +import static org.jctools.util.Pow2.roundToPowerOfTwo; +import java.util.concurrent.atomic.*; +import org.jctools.queues.*; +import static org.jctools.queues.atomic.unpadded.AtomicQueueUtil.*; + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is MpscChunkedArrayQueue.java. + */ +abstract class MpscChunkedAtomicUnpaddedArrayQueueColdProducerFields extends BaseMpscLinkedAtomicUnpaddedArrayQueue { + + protected final long maxQueueCapacity; + + MpscChunkedAtomicUnpaddedArrayQueueColdProducerFields(int initialCapacity, int maxCapacity) { + super(initialCapacity); + RangeUtil.checkGreaterThanOrEqual(maxCapacity, 4, "maxCapacity"); + RangeUtil.checkLessThan(roundToPowerOfTwo(initialCapacity), roundToPowerOfTwo(maxCapacity), "initialCapacity"); + maxQueueCapacity = ((long) Pow2.roundToPowerOfTwo(maxCapacity)) << 1; + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is MpscChunkedArrayQueue.java. + * + * An MPSC array queue which starts at initialCapacity and grows to maxCapacity in linked chunks + * of the initial size. The queue grows only when the current chunk is full and elements are not copied on + * resize, instead a link to the new chunk is stored in the old chunk for the consumer to follow. + */ +public class MpscChunkedAtomicUnpaddedArrayQueue extends MpscChunkedAtomicUnpaddedArrayQueueColdProducerFields { + + public MpscChunkedAtomicUnpaddedArrayQueue(int maxCapacity) { + super(max(2, min(1024, roundToPowerOfTwo(maxCapacity / 8))), maxCapacity); + } + + /** + * @param initialCapacity the queue initial capacity. If chunk size is fixed this will be the chunk size. + * Must be 2 or more. + * @param maxCapacity the maximum capacity will be rounded up to the closest power of 2 and will be the + * upper limit of number of elements in this queue. Must be 4 or more and round up to a larger + * power of 2 than initialCapacity. + */ + public MpscChunkedAtomicUnpaddedArrayQueue(int initialCapacity, int maxCapacity) { + super(initialCapacity, maxCapacity); + } + + @Override + protected long availableInQueue(long pIndex, long cIndex) { + return maxQueueCapacity - (pIndex - cIndex); + } + + @Override + public int capacity() { + return (int) (maxQueueCapacity / 2); + } + + @Override + protected int getNextBufferSize(AtomicReferenceArray buffer) { + return length(buffer); + } + + @Override + protected long getCurrentBufferCapacity(long mask) { + return mask; + } +} diff --git a/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/MpscGrowableAtomicUnpaddedArrayQueue.java b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/MpscGrowableAtomicUnpaddedArrayQueue.java new file mode 100644 index 00000000..3371b7d9 --- /dev/null +++ b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/MpscGrowableAtomicUnpaddedArrayQueue.java @@ -0,0 +1,60 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jctools.queues.atomic.unpadded; + +import org.jctools.util.Pow2; +import org.jctools.util.RangeUtil; +import java.util.concurrent.atomic.*; +import org.jctools.queues.*; +import static org.jctools.queues.atomic.unpadded.AtomicQueueUtil.*; + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is MpscGrowableArrayQueue.java. + * + * An MPSC array queue which starts at initialCapacity and grows to maxCapacity in linked chunks, + * doubling theirs size every time until the full blown backing array is used. + * The queue grows only when the current chunk is full and elements are not copied on + * resize, instead a link to the new chunk is stored in the old chunk for the consumer to follow. + */ +public class MpscGrowableAtomicUnpaddedArrayQueue extends MpscChunkedAtomicUnpaddedArrayQueue { + + public MpscGrowableAtomicUnpaddedArrayQueue(int maxCapacity) { + super(Math.max(2, Pow2.roundToPowerOfTwo(maxCapacity / 8)), maxCapacity); + } + + /** + * @param initialCapacity the queue initial capacity. If chunk size is fixed this will be the chunk size. + * Must be 2 or more. + * @param maxCapacity the maximum capacity will be rounded up to the closest power of 2 and will be the + * upper limit of number of elements in this queue. Must be 4 or more and round up to a larger + * power of 2 than initialCapacity. + */ + public MpscGrowableAtomicUnpaddedArrayQueue(int initialCapacity, int maxCapacity) { + super(initialCapacity, maxCapacity); + } + + @Override + protected int getNextBufferSize(AtomicReferenceArray buffer) { + final long maxSize = maxQueueCapacity / 2; + RangeUtil.checkLessThanOrEqual(length(buffer), maxSize, "buffer.length"); + final int newSize = 2 * (length(buffer) - 1); + return newSize + 1; + } + + @Override + protected long getCurrentBufferCapacity(long mask) { + return (mask + 2 == maxQueueCapacity) ? maxQueueCapacity : mask; + } +} diff --git a/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/MpscLinkedAtomicUnpaddedQueue.java b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/MpscLinkedAtomicUnpaddedQueue.java new file mode 100644 index 00000000..91c1682c --- /dev/null +++ b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/MpscLinkedAtomicUnpaddedQueue.java @@ -0,0 +1,158 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jctools.queues.atomic.unpadded; + +import java.util.Queue; +import java.util.concurrent.atomic.*; +import org.jctools.queues.*; +import static org.jctools.queues.atomic.unpadded.AtomicQueueUtil.*; + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is MpscLinkedQueue.java. + * + * This is a Java port of the MPSC algorithm as presented + * on + * 1024 Cores by D. Vyukov. The original has been adapted to Java and it's quirks with regards to memory + * model and layout: + *

    + *
  1. Use inheritance to ensure no false sharing occurs between producer/consumer node reference fields. + *
  2. Use XCHG functionality to the best of the JDK ability (see differences in JDK7/8 impls). + *
  3. Conform to {@link java.util.Queue} contract on poll. The original semantics are available via relaxedPoll. + *
+ * The queue is initialized with a stub node which is set to both the producer and consumer node references. + * From this point follow the notes on offer/poll. + */ +public class MpscLinkedAtomicUnpaddedQueue extends BaseLinkedAtomicUnpaddedQueue { + + public MpscLinkedAtomicUnpaddedQueue() { + LinkedQueueAtomicNode node = newNode(); + spConsumerNode(node); + xchgProducerNode(node); + } + + /** + * {@inheritDoc}
+ *

+ * IMPLEMENTATION NOTES:
+ * Offer is allowed from multiple threads.
+ * Offer allocates a new node and: + *

    + *
  1. Swaps it atomically with current producer node (only one producer 'wins') + *
  2. Sets the new node as the node following from the swapped producer node + *
+ * This works because each producer is guaranteed to 'plant' a new node and link the old node. No 2 + * producers can get the same producer node as part of XCHG guarantee. + * + * @see MessagePassingQueue#offer(Object) + * @see java.util.Queue#offer(java.lang.Object) + */ + @Override + public boolean offer(final E e) { + if (null == e) { + throw new NullPointerException(); + } + final LinkedQueueAtomicNode nextNode = newNode(e); + final LinkedQueueAtomicNode prevProducerNode = xchgProducerNode(nextNode); + // Should a producer thread get interrupted here the chain WILL be broken until that thread is resumed + // and completes the store in prev.next. This is a "bubble". + prevProducerNode.soNext(nextNode); + return true; + } + + /** + * {@inheritDoc} + *

+ * This method is only safe to call from the (single) consumer thread, and is subject to best effort when racing + * with producers. This method is potentially blocking when "bubble"s in the queue are visible. + */ + @Override + public boolean remove(Object o) { + if (null == o) { + // Null elements are not permitted, so null will never be removed. + return false; + } + final LinkedQueueAtomicNode originalConsumerNode = lpConsumerNode(); + LinkedQueueAtomicNode prevConsumerNode = originalConsumerNode; + LinkedQueueAtomicNode currConsumerNode = getNextConsumerNode(originalConsumerNode); + while (currConsumerNode != null) { + if (o.equals(currConsumerNode.lpValue())) { + LinkedQueueAtomicNode nextNode = getNextConsumerNode(currConsumerNode); + // e.g.: consumerNode -> node0 -> node1(o==v) -> node2 ... => consumerNode -> node0 -> node2 + if (nextNode != null) { + // We are removing an interior node. + prevConsumerNode.soNext(nextNode); + } else // This case reflects: prevConsumerNode != originalConsumerNode && nextNode == null + // At rest, this would be the producerNode, but we must contend with racing. Changes to subclassed + // queues need to consider remove() when implementing offer(). + { + // producerNode is currConsumerNode, try to atomically update the reference to move it to the + // previous node. + prevConsumerNode.soNext(null); + if (!casProducerNode(currConsumerNode, prevConsumerNode)) { + // If the producer(s) have offered more items we need to remove the currConsumerNode link. + nextNode = spinWaitForNextNode(currConsumerNode); + prevConsumerNode.soNext(nextNode); + } + } + // Avoid GC nepotism because we are discarding the current node. + currConsumerNode.soNext(null); + currConsumerNode.spValue(null); + return true; + } + prevConsumerNode = currConsumerNode; + currConsumerNode = getNextConsumerNode(currConsumerNode); + } + return false; + } + + @Override + public int fill(Supplier s) { + return MessagePassingQueueUtil.fillUnbounded(this, s); + } + + @Override + public int fill(Supplier s, int limit) { + if (null == s) + throw new IllegalArgumentException("supplier is null"); + if (limit < 0) + throw new IllegalArgumentException("limit is negative:" + limit); + if (limit == 0) + return 0; + LinkedQueueAtomicNode tail = newNode(s.get()); + final LinkedQueueAtomicNode head = tail; + for (int i = 1; i < limit; i++) { + final LinkedQueueAtomicNode temp = newNode(s.get()); + // spNext: xchgProducerNode ensures correct construction + tail.spNext(temp); + tail = temp; + } + final LinkedQueueAtomicNode oldPNode = xchgProducerNode(tail); + oldPNode.soNext(head); + return limit; + } + + @Override + public void fill(Supplier s, WaitStrategy wait, ExitCondition exit) { + MessagePassingQueueUtil.fill(this, s, wait, exit); + } + + private LinkedQueueAtomicNode getNextConsumerNode(LinkedQueueAtomicNode currConsumerNode) { + LinkedQueueAtomicNode nextNode = currConsumerNode.lvNext(); + if (nextNode == null && currConsumerNode != lvProducerNode()) { + nextNode = spinWaitForNextNode(currConsumerNode); + } + return nextNode; + } +} diff --git a/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/MpscUnboundedAtomicUnpaddedArrayQueue.java b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/MpscUnboundedAtomicUnpaddedArrayQueue.java new file mode 100644 index 00000000..2b3467aa --- /dev/null +++ b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/MpscUnboundedAtomicUnpaddedArrayQueue.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jctools.queues.atomic.unpadded; + +import java.util.concurrent.atomic.*; +import org.jctools.queues.*; +import static org.jctools.queues.atomic.unpadded.AtomicQueueUtil.*; + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is MpscUnboundedArrayQueue.java. + * + * An MPSC array queue which starts at initialCapacity and grows indefinitely in linked chunks of the initial size. + * The queue grows only when the current chunk is full and elements are not copied on + * resize, instead a link to the new chunk is stored in the old chunk for the consumer to follow. + */ +public class MpscUnboundedAtomicUnpaddedArrayQueue extends BaseMpscLinkedAtomicUnpaddedArrayQueue { + + public MpscUnboundedAtomicUnpaddedArrayQueue(int chunkSize) { + super(chunkSize); + } + + @Override + protected long availableInQueue(long pIndex, long cIndex) { + return Integer.MAX_VALUE; + } + + @Override + public int capacity() { + return MessagePassingQueue.UNBOUNDED_CAPACITY; + } + + @Override + public int drain(Consumer c) { + return drain(c, 4096); + } + + @Override + public int fill(Supplier s) { + return MessagePassingQueueUtil.fillUnbounded(this, s); + } + + @Override + protected int getNextBufferSize(AtomicReferenceArray buffer) { + return length(buffer); + } + + @Override + protected long getCurrentBufferCapacity(long mask) { + return mask; + } +} diff --git a/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SequencedAtomicReferenceArrayQueue.java b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SequencedAtomicReferenceArrayQueue.java new file mode 100644 index 00000000..c9a2a514 --- /dev/null +++ b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SequencedAtomicReferenceArrayQueue.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jctools.queues.atomic.unpadded; + +import java.util.concurrent.atomic.AtomicLongArray; + +abstract class SequencedAtomicReferenceArrayQueue extends + AtomicReferenceArrayQueue +{ + protected final AtomicLongArray sequenceBuffer; + + public SequencedAtomicReferenceArrayQueue(int capacity) + { + super(capacity); + int actualCapacity = this.mask + 1; + // pad data on either end with some empty slots. + sequenceBuffer = new AtomicLongArray(actualCapacity); + for (int i = 0; i < actualCapacity; i++) + { + soSequence(sequenceBuffer, i, i); + } + } + + protected final long calcSequenceOffset(long index) + { + return calcSequenceOffset(index, mask); + } + + protected static int calcSequenceOffset(long index, int mask) + { + return (int) index & mask; + } + + protected final void soSequence(AtomicLongArray buffer, int offset, long e) + { + buffer.lazySet(offset, e); + } + + protected final long lvSequence(AtomicLongArray buffer, int offset) + { + return buffer.get(offset); + } + +} diff --git a/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SpmcAtomicUnpaddedArrayQueue.java b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SpmcAtomicUnpaddedArrayQueue.java new file mode 100644 index 00000000..2f002427 --- /dev/null +++ b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SpmcAtomicUnpaddedArrayQueue.java @@ -0,0 +1,351 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jctools.queues.atomic.unpadded; + +import java.util.concurrent.atomic.*; +import org.jctools.queues.*; +import static org.jctools.queues.atomic.unpadded.AtomicQueueUtil.*; + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is SpmcArrayQueue.java. + */ +abstract class SpmcAtomicUnpaddedArrayQueueL1Pad extends AtomicReferenceArrayQueue { + + SpmcAtomicUnpaddedArrayQueueL1Pad(int capacity) { + super(capacity); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is SpmcArrayQueue.java. + */ +abstract class SpmcAtomicUnpaddedArrayQueueProducerIndexField extends SpmcAtomicUnpaddedArrayQueueL1Pad { + + private static final AtomicLongFieldUpdater P_INDEX_UPDATER = AtomicLongFieldUpdater.newUpdater(SpmcAtomicUnpaddedArrayQueueProducerIndexField.class, "producerIndex"); + + private volatile long producerIndex; + + SpmcAtomicUnpaddedArrayQueueProducerIndexField(int capacity) { + super(capacity); + } + + @Override + public final long lvProducerIndex() { + return producerIndex; + } + + final long lpProducerIndex() { + return producerIndex; + } + + final void soProducerIndex(long newValue) { + P_INDEX_UPDATER.lazySet(this, newValue); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is SpmcArrayQueue.java. + */ +abstract class SpmcAtomicUnpaddedArrayQueueL2Pad extends SpmcAtomicUnpaddedArrayQueueProducerIndexField { + + SpmcAtomicUnpaddedArrayQueueL2Pad(int capacity) { + super(capacity); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is SpmcArrayQueue.java. + */ +abstract class SpmcAtomicUnpaddedArrayQueueConsumerIndexField extends SpmcAtomicUnpaddedArrayQueueL2Pad { + + private static final AtomicLongFieldUpdater C_INDEX_UPDATER = AtomicLongFieldUpdater.newUpdater(SpmcAtomicUnpaddedArrayQueueConsumerIndexField.class, "consumerIndex"); + + private volatile long consumerIndex; + + SpmcAtomicUnpaddedArrayQueueConsumerIndexField(int capacity) { + super(capacity); + } + + @Override + public final long lvConsumerIndex() { + return consumerIndex; + } + + final boolean casConsumerIndex(long expect, long newValue) { + return C_INDEX_UPDATER.compareAndSet(this, expect, newValue); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is SpmcArrayQueue.java. + */ +abstract class SpmcAtomicUnpaddedArrayQueueMidPad extends SpmcAtomicUnpaddedArrayQueueConsumerIndexField { + + SpmcAtomicUnpaddedArrayQueueMidPad(int capacity) { + super(capacity); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is SpmcArrayQueue.java. + */ +abstract class SpmcAtomicUnpaddedArrayQueueProducerIndexCacheField extends SpmcAtomicUnpaddedArrayQueueMidPad { + + // This is separated from the consumerIndex which will be highly contended in the hope that this value spends most + // of it's time in a cache line that is Shared(and rarely invalidated) + private volatile long producerIndexCache; + + SpmcAtomicUnpaddedArrayQueueProducerIndexCacheField(int capacity) { + super(capacity); + } + + protected final long lvProducerIndexCache() { + return producerIndexCache; + } + + protected final void svProducerIndexCache(long newValue) { + producerIndexCache = newValue; + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is SpmcArrayQueue.java. + */ +abstract class SpmcAtomicUnpaddedArrayQueueL3Pad extends SpmcAtomicUnpaddedArrayQueueProducerIndexCacheField { + + SpmcAtomicUnpaddedArrayQueueL3Pad(int capacity) { + super(capacity); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is SpmcArrayQueue.java. + */ +public class SpmcAtomicUnpaddedArrayQueue extends SpmcAtomicUnpaddedArrayQueueL3Pad { + + public SpmcAtomicUnpaddedArrayQueue(final int capacity) { + super(capacity); + } + + @Override + public boolean offer(final E e) { + if (null == e) { + throw new NullPointerException(); + } + final AtomicReferenceArray buffer = this.buffer; + final int mask = this.mask; + final long currProducerIndex = lvProducerIndex(); + final int offset = calcCircularRefElementOffset(currProducerIndex, mask); + if (null != lvRefElement(buffer, offset)) { + long size = currProducerIndex - lvConsumerIndex(); + if (size > mask) { + return false; + } else { + // Bubble: This can happen because `poll` moves index before placing element. + // spin wait for slot to clear, buggers wait freedom + while (null != lvRefElement(buffer, offset)) { + // BURN + } + } + } + soRefElement(buffer, offset, e); + // single producer, so store ordered is valid. It is also required to correctly publish the element + // and for the consumers to pick up the tail value. + soProducerIndex(currProducerIndex + 1); + return true; + } + + @Override + public E poll() { + long currentConsumerIndex; + long currProducerIndexCache = lvProducerIndexCache(); + do { + currentConsumerIndex = lvConsumerIndex(); + if (currentConsumerIndex >= currProducerIndexCache) { + long currProducerIndex = lvProducerIndex(); + if (currentConsumerIndex >= currProducerIndex) { + return null; + } else { + currProducerIndexCache = currProducerIndex; + svProducerIndexCache(currProducerIndex); + } + } + } while (!casConsumerIndex(currentConsumerIndex, currentConsumerIndex + 1)); + // consumers are gated on latest visible tail, and so can't see a null value in the queue or overtake + // and wrap to hit same location. + return removeElement(buffer, currentConsumerIndex, mask); + } + + private E removeElement(final AtomicReferenceArray buffer, long index, final int mask) { + final int offset = calcCircularRefElementOffset(index, mask); + // load plain, element happens before it's index becomes visible + final E e = lpRefElement(buffer, offset); + // store ordered, make sure nulling out is visible. Producer is waiting for this value. + soRefElement(buffer, offset, null); + return e; + } + + @Override + public E peek() { + final AtomicReferenceArray buffer = this.buffer; + final int mask = this.mask; + long currProducerIndexCache = lvProducerIndexCache(); + long currentConsumerIndex; + long nextConsumerIndex = lvConsumerIndex(); + E e; + do { + currentConsumerIndex = nextConsumerIndex; + if (currentConsumerIndex >= currProducerIndexCache) { + long currProducerIndex = lvProducerIndex(); + if (currentConsumerIndex >= currProducerIndex) { + return null; + } else { + currProducerIndexCache = currProducerIndex; + svProducerIndexCache(currProducerIndex); + } + } + e = lvRefElement(buffer, calcCircularRefElementOffset(currentConsumerIndex, mask)); + // sandwich the element load between 2 consumer index loads + nextConsumerIndex = lvConsumerIndex(); + } while (null == e || nextConsumerIndex != currentConsumerIndex); + return e; + } + + @Override + public boolean relaxedOffer(E e) { + if (null == e) { + throw new NullPointerException("Null is not a valid element"); + } + final AtomicReferenceArray buffer = this.buffer; + final int mask = this.mask; + final long producerIndex = lpProducerIndex(); + final int offset = calcCircularRefElementOffset(producerIndex, mask); + if (null != lvRefElement(buffer, offset)) { + return false; + } + soRefElement(buffer, offset, e); + // single producer, so store ordered is valid. It is also required to correctly publish the element + // and for the consumers to pick up the tail value. + soProducerIndex(producerIndex + 1); + return true; + } + + @Override + public E relaxedPoll() { + return poll(); + } + + @Override + public E relaxedPeek() { + final AtomicReferenceArray buffer = this.buffer; + final int mask = this.mask; + long currentConsumerIndex; + long nextConsumerIndex = lvConsumerIndex(); + E e; + do { + currentConsumerIndex = nextConsumerIndex; + e = lvRefElement(buffer, calcCircularRefElementOffset(currentConsumerIndex, mask)); + // sandwich the element load between 2 consumer index loads + nextConsumerIndex = lvConsumerIndex(); + } while (nextConsumerIndex != currentConsumerIndex); + return e; + } + + @Override + public int drain(final Consumer c, final int limit) { + if (null == c) + throw new IllegalArgumentException("c is null"); + if (limit < 0) + throw new IllegalArgumentException("limit is negative: " + limit); + if (limit == 0) + return 0; + final AtomicReferenceArray buffer = this.buffer; + final int mask = this.mask; + long currProducerIndexCache = lvProducerIndexCache(); + int adjustedLimit = 0; + long currentConsumerIndex; + do { + currentConsumerIndex = lvConsumerIndex(); + // is there any space in the queue? + if (currentConsumerIndex >= currProducerIndexCache) { + long currProducerIndex = lvProducerIndex(); + if (currentConsumerIndex >= currProducerIndex) { + return 0; + } else { + currProducerIndexCache = currProducerIndex; + svProducerIndexCache(currProducerIndex); + } + } + // try and claim up to 'limit' elements in one go + int remaining = (int) (currProducerIndexCache - currentConsumerIndex); + adjustedLimit = Math.min(remaining, limit); + } while (!casConsumerIndex(currentConsumerIndex, currentConsumerIndex + adjustedLimit)); + for (int i = 0; i < adjustedLimit; i++) { + c.accept(removeElement(buffer, currentConsumerIndex + i, mask)); + } + return adjustedLimit; + } + + @Override + public int fill(final Supplier s, final int limit) { + if (null == s) + throw new IllegalArgumentException("supplier is null"); + if (limit < 0) + throw new IllegalArgumentException("limit is negative:" + limit); + if (limit == 0) + return 0; + final AtomicReferenceArray buffer = this.buffer; + final int mask = this.mask; + long producerIndex = this.lpProducerIndex(); + for (int i = 0; i < limit; i++) { + final int offset = calcCircularRefElementOffset(producerIndex, mask); + if (null != lvRefElement(buffer, offset)) { + return i; + } + producerIndex++; + soRefElement(buffer, offset, s.get()); + // ordered store -> atomic and ordered for size() + soProducerIndex(producerIndex); + } + return limit; + } + + @Override + public int drain(final Consumer c) { + return MessagePassingQueueUtil.drain(this, c); + } + + @Override + public int fill(final Supplier s) { + return fill(s, capacity()); + } + + @Override + public void drain(final Consumer c, final WaitStrategy w, final ExitCondition exit) { + MessagePassingQueueUtil.drain(this, c, w, exit); + } + + @Override + public void fill(final Supplier s, final WaitStrategy w, final ExitCondition e) { + MessagePassingQueueUtil.fill(this, s, w, e); + } +} diff --git a/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SpscAtomicUnpaddedArrayQueue.java b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SpscAtomicUnpaddedArrayQueue.java new file mode 100644 index 00000000..adb7bd2d --- /dev/null +++ b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SpscAtomicUnpaddedArrayQueue.java @@ -0,0 +1,372 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jctools.queues.atomic.unpadded; + +import org.jctools.util.SpscLookAheadUtil; +import java.util.concurrent.atomic.*; +import org.jctools.queues.*; +import static org.jctools.queues.atomic.unpadded.AtomicQueueUtil.*; + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is SpscArrayQueue.java. + */ +abstract class SpscAtomicUnpaddedArrayQueueColdField extends AtomicReferenceArrayQueue { + + final int lookAheadStep; + + SpscAtomicUnpaddedArrayQueueColdField(int capacity) { + super(capacity); + int actualCapacity = capacity(); + lookAheadStep = SpscLookAheadUtil.computeLookAheadStep(actualCapacity); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is SpscArrayQueue.java. + */ +abstract class SpscAtomicUnpaddedArrayQueueL1Pad extends SpscAtomicUnpaddedArrayQueueColdField { + + SpscAtomicUnpaddedArrayQueueL1Pad(int capacity) { + super(capacity); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is SpscArrayQueue.java. + */ +abstract class SpscAtomicUnpaddedArrayQueueProducerIndexFields extends SpscAtomicUnpaddedArrayQueueL1Pad { + + private static final AtomicLongFieldUpdater P_INDEX_UPDATER = AtomicLongFieldUpdater.newUpdater(SpscAtomicUnpaddedArrayQueueProducerIndexFields.class, "producerIndex"); + + private volatile long producerIndex; + + protected long producerLimit; + + SpscAtomicUnpaddedArrayQueueProducerIndexFields(int capacity) { + super(capacity); + } + + @Override + public final long lvProducerIndex() { + return producerIndex; + } + + final long lpProducerIndex() { + return producerIndex; + } + + final void soProducerIndex(final long newValue) { + P_INDEX_UPDATER.lazySet(this, newValue); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is SpscArrayQueue.java. + */ +abstract class SpscAtomicUnpaddedArrayQueueL2Pad extends SpscAtomicUnpaddedArrayQueueProducerIndexFields { + + SpscAtomicUnpaddedArrayQueueL2Pad(int capacity) { + super(capacity); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is SpscArrayQueue.java. + */ +abstract class SpscAtomicUnpaddedArrayQueueConsumerIndexField extends SpscAtomicUnpaddedArrayQueueL2Pad { + + private static final AtomicLongFieldUpdater C_INDEX_UPDATER = AtomicLongFieldUpdater.newUpdater(SpscAtomicUnpaddedArrayQueueConsumerIndexField.class, "consumerIndex"); + + private volatile long consumerIndex; + + SpscAtomicUnpaddedArrayQueueConsumerIndexField(int capacity) { + super(capacity); + } + + public final long lvConsumerIndex() { + return consumerIndex; + } + + final long lpConsumerIndex() { + return consumerIndex; + } + + final void soConsumerIndex(final long newValue) { + C_INDEX_UPDATER.lazySet(this, newValue); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is SpscArrayQueue.java. + */ +abstract class SpscAtomicUnpaddedArrayQueueL3Pad extends SpscAtomicUnpaddedArrayQueueConsumerIndexField { + + SpscAtomicUnpaddedArrayQueueL3Pad(int capacity) { + super(capacity); + } +} + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator + * which can found in the jctools-build module. The original source file is SpscArrayQueue.java. + * + * A Single-Producer-Single-Consumer queue backed by a pre-allocated buffer. + *

+ * This implementation is a mashup of the Fast Flow + * algorithm with an optimization of the offer method taken from the BQueue algorithm (a variation on Fast + * Flow), and adjusted to comply with Queue.offer semantics with regards to capacity.
+ * For convenience the relevant papers are available in the `resources` folder:
+ * + * 2010 - Pisa - SPSC Queues on Shared Cache Multi-Core Systems.pdf
+ * 2012 - Junchang- BQueue- Efficient and Practical Queuing.pdf
+ * + * This implementation is wait free. + */ +public class SpscAtomicUnpaddedArrayQueue extends SpscAtomicUnpaddedArrayQueueL3Pad { + + public SpscAtomicUnpaddedArrayQueue(final int capacity) { + super(Math.max(capacity, 4)); + } + + /** + * {@inheritDoc} + *

+ * This implementation is correct for single producer thread use only. + */ + @Override + public boolean offer(final E e) { + if (null == e) { + throw new NullPointerException(); + } + // local load of field to avoid repeated loads after volatile reads + final AtomicReferenceArray buffer = this.buffer; + final int mask = this.mask; + final long producerIndex = this.lpProducerIndex(); + if (producerIndex >= producerLimit && !offerSlowPath(buffer, mask, producerIndex)) { + return false; + } + final int offset = calcCircularRefElementOffset(producerIndex, mask); + soRefElement(buffer, offset, e); + // ordered store -> atomic and ordered for size() + soProducerIndex(producerIndex + 1); + return true; + } + + private boolean offerSlowPath(final AtomicReferenceArray buffer, final int mask, final long producerIndex) { + final int lookAheadStep = this.lookAheadStep; + if (null == lvRefElement(buffer, calcCircularRefElementOffset(producerIndex + lookAheadStep, mask))) { + producerLimit = producerIndex + lookAheadStep; + } else { + final int offset = calcCircularRefElementOffset(producerIndex, mask); + if (null != lvRefElement(buffer, offset)) { + return false; + } + } + return true; + } + + /** + * {@inheritDoc} + *

+ * This implementation is correct for single consumer thread use only. + */ + @Override + public E poll() { + final long consumerIndex = this.lpConsumerIndex(); + final int offset = calcCircularRefElementOffset(consumerIndex, mask); + // local load of field to avoid repeated loads after volatile reads + final AtomicReferenceArray buffer = this.buffer; + final E e = lvRefElement(buffer, offset); + if (null == e) { + return null; + } + soRefElement(buffer, offset, null); + // ordered store -> atomic and ordered for size() + soConsumerIndex(consumerIndex + 1); + return e; + } + + /** + * {@inheritDoc} + *

+ * This implementation is correct for single consumer thread use only. + */ + @Override + public E peek() { + return lvRefElement(buffer, calcCircularRefElementOffset(lpConsumerIndex(), mask)); + } + + @Override + public boolean relaxedOffer(final E message) { + return offer(message); + } + + @Override + public E relaxedPoll() { + return poll(); + } + + @Override + public E relaxedPeek() { + return peek(); + } + + @Override + public int drain(final Consumer c) { + return drain(c, capacity()); + } + + @Override + public int fill(final Supplier s) { + return fill(s, capacity()); + } + + @Override + public int drain(final Consumer c, final int limit) { + if (null == c) + throw new IllegalArgumentException("c is null"); + if (limit < 0) + throw new IllegalArgumentException("limit is negative: " + limit); + if (limit == 0) + return 0; + final AtomicReferenceArray buffer = this.buffer; + final int mask = this.mask; + final long consumerIndex = this.lpConsumerIndex(); + for (int i = 0; i < limit; i++) { + final long index = consumerIndex + i; + final int offset = calcCircularRefElementOffset(index, mask); + final E e = lvRefElement(buffer, offset); + if (null == e) { + return i; + } + soRefElement(buffer, offset, null); + // ordered store -> atomic and ordered for size() + soConsumerIndex(index + 1); + c.accept(e); + } + return limit; + } + + @Override + public int fill(final Supplier s, final int limit) { + if (null == s) + throw new IllegalArgumentException("supplier is null"); + if (limit < 0) + throw new IllegalArgumentException("limit is negative:" + limit); + if (limit == 0) + return 0; + final AtomicReferenceArray buffer = this.buffer; + final int mask = this.mask; + final int lookAheadStep = this.lookAheadStep; + final long producerIndex = this.lpProducerIndex(); + for (int i = 0; i < limit; i++) { + final long index = producerIndex + i; + final int lookAheadElementOffset = calcCircularRefElementOffset(index + lookAheadStep, mask); + if (null == lvRefElement(buffer, lookAheadElementOffset)) { + int lookAheadLimit = Math.min(lookAheadStep, limit - i); + for (int j = 0; j < lookAheadLimit; j++) { + final int offset = calcCircularRefElementOffset(index + j, mask); + soRefElement(buffer, offset, s.get()); + // ordered store -> atomic and ordered for size() + soProducerIndex(index + j + 1); + } + i += lookAheadLimit - 1; + } else { + final int offset = calcCircularRefElementOffset(index, mask); + if (null != lvRefElement(buffer, offset)) { + return i; + } + soRefElement(buffer, offset, s.get()); + // ordered store -> atomic and ordered for size() + soProducerIndex(index + 1); + } + } + return limit; + } + + @Override + public void drain(final Consumer c, final WaitStrategy w, final ExitCondition exit) { + if (null == c) + throw new IllegalArgumentException("c is null"); + if (null == w) + throw new IllegalArgumentException("wait is null"); + if (null == exit) + throw new IllegalArgumentException("exit condition is null"); + final AtomicReferenceArray buffer = this.buffer; + final int mask = this.mask; + long consumerIndex = this.lpConsumerIndex(); + int counter = 0; + while (exit.keepRunning()) { + for (int i = 0; i < 4096; i++) { + final int offset = calcCircularRefElementOffset(consumerIndex, mask); + final E e = lvRefElement(buffer, offset); + if (null == e) { + counter = w.idle(counter); + continue; + } + consumerIndex++; + counter = 0; + soRefElement(buffer, offset, null); + // ordered store -> atomic and ordered for size() + soConsumerIndex(consumerIndex); + c.accept(e); + } + } + } + + @Override + public void fill(final Supplier s, final WaitStrategy w, final ExitCondition e) { + if (null == w) + throw new IllegalArgumentException("waiter is null"); + if (null == e) + throw new IllegalArgumentException("exit condition is null"); + if (null == s) + throw new IllegalArgumentException("supplier is null"); + final AtomicReferenceArray buffer = this.buffer; + final int mask = this.mask; + final int lookAheadStep = this.lookAheadStep; + long producerIndex = this.lpProducerIndex(); + int counter = 0; + while (e.keepRunning()) { + final int lookAheadElementOffset = calcCircularRefElementOffset(producerIndex + lookAheadStep, mask); + if (null == lvRefElement(buffer, lookAheadElementOffset)) { + for (int j = 0; j < lookAheadStep; j++) { + final int offset = calcCircularRefElementOffset(producerIndex, mask); + producerIndex++; + soRefElement(buffer, offset, s.get()); + // ordered store -> atomic and ordered for size() + soProducerIndex(producerIndex); + } + } else { + final int offset = calcCircularRefElementOffset(producerIndex, mask); + if (null != lvRefElement(buffer, offset)) { + counter = w.idle(counter); + continue; + } + producerIndex++; + counter = 0; + soRefElement(buffer, offset, s.get()); + // ordered store -> atomic and ordered for size() + soProducerIndex(producerIndex); + } + } + } +} diff --git a/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SpscChunkedAtomicUnpaddedArrayQueue.java b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SpscChunkedAtomicUnpaddedArrayQueue.java new file mode 100644 index 00000000..a6e6b071 --- /dev/null +++ b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SpscChunkedAtomicUnpaddedArrayQueue.java @@ -0,0 +1,104 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jctools.queues.atomic.unpadded; + +import org.jctools.util.Pow2; +import org.jctools.util.RangeUtil; +import java.util.concurrent.atomic.*; +import org.jctools.queues.*; +import static org.jctools.queues.atomic.unpadded.AtomicQueueUtil.*; + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is SpscChunkedArrayQueue.java. + * + * An SPSC array queue which starts at initialCapacity and grows to maxCapacity in linked chunks + * of the initial size. The queue grows only when the current chunk is full and elements are not copied on + * resize, instead a link to the new chunk is stored in the old chunk for the consumer to follow.
+ * + * @param + */ +public class SpscChunkedAtomicUnpaddedArrayQueue extends BaseSpscLinkedAtomicUnpaddedArrayQueue { + + private final int maxQueueCapacity; + + private long producerQueueLimit; + + public SpscChunkedAtomicUnpaddedArrayQueue(int capacity) { + this(Math.max(8, Pow2.roundToPowerOfTwo(capacity / 8)), capacity); + } + + public SpscChunkedAtomicUnpaddedArrayQueue(int chunkSize, int capacity) { + RangeUtil.checkGreaterThanOrEqual(capacity, 16, "capacity"); + // minimal chunk size of eight makes sure minimal lookahead step is 2 + RangeUtil.checkGreaterThanOrEqual(chunkSize, 8, "chunkSize"); + maxQueueCapacity = Pow2.roundToPowerOfTwo(capacity); + int chunkCapacity = Pow2.roundToPowerOfTwo(chunkSize); + RangeUtil.checkLessThan(chunkCapacity, maxQueueCapacity, "chunkCapacity"); + long mask = chunkCapacity - 1; + // need extra element to point at next array + AtomicReferenceArray buffer = allocateRefArray(chunkCapacity + 1); + producerBuffer = buffer; + producerMask = mask; + consumerBuffer = buffer; + consumerMask = mask; + // we know it's all empty to start with + producerBufferLimit = mask - 1; + producerQueueLimit = maxQueueCapacity; + } + + @Override + final boolean offerColdPath(AtomicReferenceArray buffer, long mask, long pIndex, int offset, E v, Supplier s) { + // use a fixed lookahead step based on buffer capacity + final long lookAheadStep = (mask + 1) / 4; + long pBufferLimit = pIndex + lookAheadStep; + long pQueueLimit = producerQueueLimit; + if (pIndex >= pQueueLimit) { + // we tested against a potentially out of date queue limit, refresh it + final long cIndex = lvConsumerIndex(); + producerQueueLimit = pQueueLimit = cIndex + maxQueueCapacity; + // if we're full we're full + if (pIndex >= pQueueLimit) { + return false; + } + } + // if buffer limit is after queue limit we use queue limit. We need to handle overflow so + // cannot use Math.min + if (pBufferLimit - pQueueLimit > 0) { + pBufferLimit = pQueueLimit; + } + // go around the buffer or add a new buffer + if (// there's sufficient room in buffer/queue to use pBufferLimit + pBufferLimit > pIndex + 1 && null == lvRefElement(buffer, calcCircularRefElementOffset(pBufferLimit, mask))) { + // joy, there's plenty of room + producerBufferLimit = pBufferLimit - 1; + writeToQueue(buffer, v == null ? s.get() : v, pIndex, offset); + } else if (null == lvRefElement(buffer, calcCircularRefElementOffset(pIndex + 1, mask))) { + // buffer is not full + writeToQueue(buffer, v == null ? s.get() : v, pIndex, offset); + } else { + // we got one slot left to write into, and we are not full. Need to link new buffer. + // allocate new buffer of same length + final AtomicReferenceArray newBuffer = allocateRefArray((int) (mask + 2)); + producerBuffer = newBuffer; + linkOldToNew(pIndex, buffer, offset, newBuffer, offset, v == null ? s.get() : v); + } + return true; + } + + @Override + public int capacity() { + return maxQueueCapacity; + } +} diff --git a/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SpscGrowableAtomicUnpaddedArrayQueue.java b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SpscGrowableAtomicUnpaddedArrayQueue.java new file mode 100644 index 00000000..4e12933d --- /dev/null +++ b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SpscGrowableAtomicUnpaddedArrayQueue.java @@ -0,0 +1,147 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jctools.queues.atomic.unpadded; + +import org.jctools.util.Pow2; +import org.jctools.util.RangeUtil; +import org.jctools.util.SpscLookAheadUtil; +import java.util.concurrent.atomic.*; +import org.jctools.queues.*; +import static org.jctools.queues.atomic.unpadded.AtomicQueueUtil.*; + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is SpscGrowableArrayQueue.java. + * + * An SPSC array queue which starts at initialCapacity and grows to maxCapacity in linked chunks, + * doubling theirs size every time until the full blown backing array is used. + * The queue grows only when the current chunk is full and elements are not copied on + * resize, instead a link to the new chunk is stored in the old chunk for the consumer to follow.
+ * + * @param + */ +public class SpscGrowableAtomicUnpaddedArrayQueue extends BaseSpscLinkedAtomicUnpaddedArrayQueue { + + private final int maxQueueCapacity; + + private long lookAheadStep; + + public SpscGrowableAtomicUnpaddedArrayQueue(final int capacity) { + this(Math.max(8, Pow2.roundToPowerOfTwo(capacity / 8)), capacity); + } + + public SpscGrowableAtomicUnpaddedArrayQueue(final int chunkSize, final int capacity) { + RangeUtil.checkGreaterThanOrEqual(capacity, 16, "capacity"); + // minimal chunk size of eight makes sure minimal lookahead step is 2 + RangeUtil.checkGreaterThanOrEqual(chunkSize, 8, "chunkSize"); + maxQueueCapacity = Pow2.roundToPowerOfTwo(capacity); + int chunkCapacity = Pow2.roundToPowerOfTwo(chunkSize); + RangeUtil.checkLessThan(chunkCapacity, maxQueueCapacity, "chunkCapacity"); + long mask = chunkCapacity - 1; + // need extra element to point at next array + AtomicReferenceArray buffer = allocateRefArray(chunkCapacity + 1); + producerBuffer = buffer; + producerMask = mask; + consumerBuffer = buffer; + consumerMask = mask; + // we know it's all empty to start with + producerBufferLimit = mask - 1; + adjustLookAheadStep(chunkCapacity); + } + + @Override + final boolean offerColdPath(final AtomicReferenceArray buffer, final long mask, final long index, final int offset, final E v, final Supplier s) { + final long lookAheadStep = this.lookAheadStep; + // normal case, go around the buffer or resize if full (unless we hit max capacity) + if (lookAheadStep > 0) { + int lookAheadElementOffset = calcCircularRefElementOffset(index + lookAheadStep, mask); + // Try and look ahead a number of elements so we don't have to do this all the time + if (null == lvRefElement(buffer, lookAheadElementOffset)) { + // joy, there's plenty of room + producerBufferLimit = index + lookAheadStep - 1; + writeToQueue(buffer, v == null ? s.get() : v, index, offset); + return true; + } + // we're at max capacity, can use up last element + final int maxCapacity = maxQueueCapacity; + if (mask + 1 == maxCapacity) { + if (null == lvRefElement(buffer, offset)) { + writeToQueue(buffer, v == null ? s.get() : v, index, offset); + return true; + } + // we're full and can't grow + return false; + } + // not at max capacity, so must allow extra slot for next buffer pointer + if (null == lvRefElement(buffer, calcCircularRefElementOffset(index + 1, mask))) { + // buffer is not full + writeToQueue(buffer, v == null ? s.get() : v, index, offset); + } else { + // allocate new buffer of same length + final AtomicReferenceArray newBuffer = allocateRefArray((int) (2 * (mask + 1) + 1)); + producerBuffer = newBuffer; + producerMask = length(newBuffer) - 2; + final int offsetInNew = calcCircularRefElementOffset(index, producerMask); + linkOldToNew(index, buffer, offset, newBuffer, offsetInNew, v == null ? s.get() : v); + int newCapacity = (int) (producerMask + 1); + if (newCapacity == maxCapacity) { + long currConsumerIndex = lvConsumerIndex(); + // use lookAheadStep to store the consumer distance from final buffer + this.lookAheadStep = -(index - currConsumerIndex); + producerBufferLimit = currConsumerIndex + maxCapacity; + } else { + producerBufferLimit = index + producerMask - 1; + adjustLookAheadStep(newCapacity); + } + } + return true; + } else // the step is negative (or zero) in the period between allocating the max sized buffer and the + // consumer starting on it + { + final long prevElementsInOtherBuffers = -lookAheadStep; + // until the consumer starts using the current buffer we need to check consumer index to + // verify size + long currConsumerIndex = lvConsumerIndex(); + int size = (int) (index - currConsumerIndex); + // we're on max capacity or we wouldn't be here + int maxCapacity = (int) mask + 1; + if (size == maxCapacity) { + // consumer index has not changed since adjusting the lookAhead index, we're full + return false; + } + // if consumerIndex progressed enough so that current size indicates it is on same buffer + long firstIndexInCurrentBuffer = producerBufferLimit - maxCapacity + prevElementsInOtherBuffers; + if (currConsumerIndex >= firstIndexInCurrentBuffer) { + // job done, we've now settled into our final state + adjustLookAheadStep(maxCapacity); + } else // consumer is still on some other buffer + { + // how many elements out of buffer? + this.lookAheadStep = (int) (currConsumerIndex - firstIndexInCurrentBuffer); + } + producerBufferLimit = currConsumerIndex + maxCapacity; + writeToQueue(buffer, v == null ? s.get() : v, index, offset); + return true; + } + } + + private void adjustLookAheadStep(int capacity) { + lookAheadStep = SpscLookAheadUtil.computeLookAheadStep(capacity); + } + + @Override + public int capacity() { + return maxQueueCapacity; + } +} diff --git a/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SpscLinkedAtomicUnpaddedQueue.java b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SpscLinkedAtomicUnpaddedQueue.java new file mode 100644 index 00000000..751d1087 --- /dev/null +++ b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SpscLinkedAtomicUnpaddedQueue.java @@ -0,0 +1,110 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jctools.queues.atomic.unpadded; + +import java.util.concurrent.atomic.*; +import org.jctools.queues.*; +import static org.jctools.queues.atomic.unpadded.AtomicQueueUtil.*; + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is SpscLinkedQueue.java. + * + * This is a weakened version of the MPSC algorithm as presented + * on + * 1024 Cores by D. Vyukov. The original has been adapted to Java and it's quirks with regards to memory + * model and layout: + *

    + *
  1. Use inheritance to ensure no false sharing occurs between producer/consumer node reference fields. + *
  2. As this is an SPSC we have no need for XCHG, an ordered store is enough. + *
+ * The queue is initialized with a stub node which is set to both the producer and consumer node references. + * From this point follow the notes on offer/poll. + * + * @param + * @author nitsanw + */ +public class SpscLinkedAtomicUnpaddedQueue extends BaseLinkedAtomicUnpaddedQueue { + + public SpscLinkedAtomicUnpaddedQueue() { + LinkedQueueAtomicNode node = newNode(); + spProducerNode(node); + spConsumerNode(node); + // this ensures correct construction: StoreStore + node.soNext(null); + } + + /** + * {@inheritDoc}
+ *

+ * IMPLEMENTATION NOTES:
+ * Offer is allowed from a SINGLE thread.
+ * Offer allocates a new node (holding the offered value) and: + *

    + *
  1. Sets the new node as the producerNode + *
  2. Sets that node as the lastProducerNode.next + *
+ * From this follows that producerNode.next is always null and for all other nodes node.next is not null. + * + * @see MessagePassingQueue#offer(Object) + * @see java.util.Queue#offer(java.lang.Object) + */ + @Override + public boolean offer(final E e) { + if (null == e) { + throw new NullPointerException(); + } + final LinkedQueueAtomicNode nextNode = newNode(e); + LinkedQueueAtomicNode oldNode = lpProducerNode(); + soProducerNode(nextNode); + // Should a producer thread get interrupted here the chain WILL be broken until that thread is resumed + // and completes the store in prev.next. This is a "bubble". + // Inverting the order here will break the `isEmpty` invariant, and will require matching adjustments elsewhere. + oldNode.soNext(nextNode); + return true; + } + + @Override + public int fill(Supplier s) { + return MessagePassingQueueUtil.fillUnbounded(this, s); + } + + @Override + public int fill(Supplier s, int limit) { + if (null == s) + throw new IllegalArgumentException("supplier is null"); + if (limit < 0) + throw new IllegalArgumentException("limit is negative:" + limit); + if (limit == 0) + return 0; + LinkedQueueAtomicNode tail = newNode(s.get()); + final LinkedQueueAtomicNode head = tail; + for (int i = 1; i < limit; i++) { + final LinkedQueueAtomicNode temp = newNode(s.get()); + // spNext : soProducerNode ensures correct construction + tail.spNext(temp); + tail = temp; + } + final LinkedQueueAtomicNode oldPNode = lpProducerNode(); + soProducerNode(tail); + // same bubble as offer, and for the same reasons. + oldPNode.soNext(head); + return limit; + } + + @Override + public void fill(Supplier s, WaitStrategy wait, ExitCondition exit) { + MessagePassingQueueUtil.fill(this, s, wait, exit); + } +} diff --git a/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SpscUnboundedAtomicUnpaddedArrayQueue.java b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SpscUnboundedAtomicUnpaddedArrayQueue.java new file mode 100644 index 00000000..23e5e1ab --- /dev/null +++ b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/SpscUnboundedAtomicUnpaddedArrayQueue.java @@ -0,0 +1,78 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.jctools.queues.atomic.unpadded; + +import org.jctools.util.Pow2; +import java.util.concurrent.atomic.*; +import org.jctools.queues.*; +import static org.jctools.queues.atomic.unpadded.AtomicQueueUtil.*; + +/** + * NOTE: This class was automatically generated by org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator + * which can found in the jctools-build module. The original source file is SpscUnboundedArrayQueue.java. + * + * An SPSC array queue which starts at initialCapacity and grows indefinitely in linked chunks of the initial size. + * The queue grows only when the current chunk is full and elements are not copied on + * resize, instead a link to the new chunk is stored in the old chunk for the consumer to follow.
+ * + * @param + */ +public class SpscUnboundedAtomicUnpaddedArrayQueue extends BaseSpscLinkedAtomicUnpaddedArrayQueue { + + public SpscUnboundedAtomicUnpaddedArrayQueue(int chunkSize) { + int chunkCapacity = Math.max(Pow2.roundToPowerOfTwo(chunkSize), 16); + long mask = chunkCapacity - 1; + AtomicReferenceArray buffer = allocateRefArray(chunkCapacity + 1); + producerBuffer = buffer; + producerMask = mask; + consumerBuffer = buffer; + consumerMask = mask; + // we know it's all empty to start with + producerBufferLimit = mask - 1; + } + + @Override + final boolean offerColdPath(AtomicReferenceArray buffer, long mask, long pIndex, int offset, E v, Supplier s) { + // use a fixed lookahead step based on buffer capacity + final long lookAheadStep = (mask + 1) / 4; + long pBufferLimit = pIndex + lookAheadStep; + // go around the buffer or add a new buffer + if (null == lvRefElement(buffer, calcCircularRefElementOffset(pBufferLimit, mask))) { + // joy, there's plenty of room + producerBufferLimit = pBufferLimit - 1; + writeToQueue(buffer, v == null ? s.get() : v, pIndex, offset); + } else if (null == lvRefElement(buffer, calcCircularRefElementOffset(pIndex + 1, mask))) { + // buffer is not full + writeToQueue(buffer, v == null ? s.get() : v, pIndex, offset); + } else { + // we got one slot left to write into, and we are not full. Need to link new buffer. + // allocate new buffer of same length + final AtomicReferenceArray newBuffer = allocateRefArray((int) (mask + 2)); + producerBuffer = newBuffer; + producerBufferLimit = pIndex + mask - 1; + linkOldToNew(pIndex, buffer, offset, newBuffer, offset, v == null ? s.get() : v); + } + return true; + } + + @Override + public int fill(Supplier s) { + return fill(s, (int) this.producerMask); + } + + @Override + public int capacity() { + return UNBOUNDED_CAPACITY; + } +} diff --git a/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/package-info.java b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/package-info.java new file mode 100644 index 00000000..01a04cbf --- /dev/null +++ b/jctools-core/src/main/java/org/jctools/queues/atomic/unpadded/package-info.java @@ -0,0 +1,18 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@Export +package org.jctools.queues.atomic.unpadded; + +import org.osgi.annotation.bundle.Export; + diff --git a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestMpmcArray.java b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestMpmcArray.java index 3cdd7203..25f249ef 100644 --- a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestMpmcArray.java +++ b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestMpmcArray.java @@ -28,6 +28,8 @@ public static Collection parameters() list.add(makeAtomic(0, 0, SIZE, Ordering.FIFO)); list.add(makeUnpadded(0, 0, 2, Ordering.FIFO)); list.add(makeUnpadded(0, 0, SIZE, Ordering.FIFO)); + list.add(makeAtomicUnpadded(0, 0, 2, Ordering.FIFO)); + list.add(makeAtomicUnpadded(0, 0, SIZE, Ordering.FIFO)); return list; } } diff --git a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestMpscArray.java b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestMpscArray.java index ddf33e30..e776ed30 100644 --- a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestMpscArray.java +++ b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestMpscArray.java @@ -28,6 +28,8 @@ public static Collection parameters() list.add(makeAtomic(0, 1, SIZE, Ordering.FIFO));// MPSC size SIZE list.add(makeUnpadded(0, 1, 1, Ordering.FIFO));// MPSC size 1 list.add(makeUnpadded(0, 1, SIZE, Ordering.FIFO));// MPSC size SIZE + list.add(makeAtomicUnpadded(0, 1, 1, Ordering.FIFO));// MPSC size 1 + list.add(makeAtomicUnpadded(0, 1, SIZE, Ordering.FIFO));// MPSC size SIZE return list; } } diff --git a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestMpscGrowable.java b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestMpscGrowable.java index d96f730f..425e884a 100644 --- a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestMpscGrowable.java +++ b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestMpscGrowable.java @@ -4,6 +4,7 @@ import org.jctools.queues.spec.ConcurrentQueueSpec; import org.jctools.queues.spec.Ordering; import org.jctools.queues.unpadded.MpscGrowableUnpaddedArrayQueue; +import org.jctools.queues.atomic.unpadded.MpscGrowableAtomicUnpaddedArrayQueue; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -30,6 +31,8 @@ public static Collection parameters() list.add(makeParams(0, 1, SIZE, Ordering.FIFO, new MpscGrowableAtomicArrayQueue<>(8, SIZE)));// MPSC size SIZE list.add(makeParams(0, 1, 4, Ordering.FIFO, new MpscGrowableUnpaddedArrayQueue<>(2, 4)));// MPSC size 1 list.add(makeParams(0, 1, SIZE, Ordering.FIFO, new MpscGrowableUnpaddedArrayQueue<>(8, SIZE)));// MPSC size SIZE + list.add(makeParams(0, 1, 4, Ordering.FIFO, new MpscGrowableAtomicUnpaddedArrayQueue<>(2, 4)));// MPSC size 1 + list.add(makeParams(0, 1, SIZE, Ordering.FIFO, new MpscGrowableAtomicUnpaddedArrayQueue<>(8, SIZE)));// MPSC size SIZE return list; } } diff --git a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestMpscLinked.java b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestMpscLinked.java index 05cf8c94..111eddbb 100644 --- a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestMpscLinked.java +++ b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestMpscLinked.java @@ -25,6 +25,7 @@ public static Collection parameters() list.add(makeMpq(0, 1, 0, Ordering.FIFO));// unbounded MPSC list.add(makeAtomic(0, 1, 0, Ordering.FIFO));// unbounded MPSC list.add(makeUnpadded(0, 1, 0, Ordering.FIFO));// unbounded MPSC + list.add(makeAtomicUnpadded(0, 1, 0, Ordering.FIFO));// unbounded MPSC return list; } } diff --git a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestMpscUnbounded.java b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestMpscUnbounded.java index d52d6f95..54600fe2 100644 --- a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestMpscUnbounded.java +++ b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestMpscUnbounded.java @@ -4,6 +4,7 @@ import org.jctools.queues.spec.ConcurrentQueueSpec; import org.jctools.queues.spec.Ordering; import org.jctools.queues.unpadded.MpscUnboundedUnpaddedArrayQueue; +import org.jctools.queues.atomic.unpadded.MpscUnboundedAtomicUnpaddedArrayQueue; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -30,6 +31,8 @@ public static Collection parameters() list.add(makeParams(0, 1, 0, Ordering.FIFO, new MpscUnboundedAtomicArrayQueue<>(64))); list.add(makeParams(0, 1, 0, Ordering.FIFO, new MpscUnboundedUnpaddedArrayQueue<>(2))); list.add(makeParams(0, 1, 0, Ordering.FIFO, new MpscUnboundedUnpaddedArrayQueue<>(64))); + list.add(makeParams(0, 1, 0, Ordering.FIFO, new MpscUnboundedAtomicUnpaddedArrayQueue<>(2))); + list.add(makeParams(0, 1, 0, Ordering.FIFO, new MpscUnboundedAtomicUnpaddedArrayQueue<>(64))); return list; } } diff --git a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpmcArray.java b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpmcArray.java index 98ae2227..28265e09 100644 --- a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpmcArray.java +++ b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpmcArray.java @@ -28,6 +28,8 @@ public static Collection parameters() list.add(makeAtomic(1, 0, SIZE, Ordering.FIFO));// SPMC size SIZE list.add(makeUnpadded(1, 0, 1, Ordering.FIFO));// SPMC size 1 list.add(makeUnpadded(1, 0, SIZE, Ordering.FIFO));// SPMC size SIZE + list.add(makeAtomicUnpadded(1, 0, 1, Ordering.FIFO));// SPMC size 1 + list.add(makeAtomicUnpadded(1, 0, SIZE, Ordering.FIFO));// SPMC size SIZE return list; } } diff --git a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpscArray.java b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpscArray.java index 6e2255a6..33311efc 100644 --- a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpscArray.java +++ b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpscArray.java @@ -28,6 +28,8 @@ public static Collection parameters() list.add(makeAtomic(1, 1, SIZE, Ordering.FIFO));// SPSC size SIZE list.add(makeUnpadded(1, 1, 4, Ordering.FIFO));// SPSC size 4 list.add(makeUnpadded(1, 1, SIZE, Ordering.FIFO));// SPSC size SIZE + list.add(makeAtomicUnpadded(1, 1, 4, Ordering.FIFO));// SPSC size 4 + list.add(makeAtomicUnpadded(1, 1, SIZE, Ordering.FIFO));// SPSC size SIZE return list; } } diff --git a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpscChunked.java b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpscChunked.java index 564943f3..907679ed 100644 --- a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpscChunked.java +++ b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpscChunked.java @@ -4,6 +4,7 @@ import org.jctools.queues.spec.ConcurrentQueueSpec; import org.jctools.queues.spec.Ordering; import org.jctools.queues.unpadded.SpscChunkedUnpaddedArrayQueue; +import org.jctools.queues.atomic.unpadded.SpscChunkedAtomicUnpaddedArrayQueue; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -31,6 +32,8 @@ public static Collection parameters() list.add(makeParams(1, 1, SIZE, Ordering.FIFO, new SpscChunkedAtomicArrayQueue<>(8, SIZE)));// MPSC size SIZE list.add(makeParams(1, 1, 16, Ordering.FIFO, new SpscChunkedUnpaddedArrayQueue<>(8, 16)));// MPSC size 1 list.add(makeParams(1, 1, SIZE, Ordering.FIFO, new SpscChunkedUnpaddedArrayQueue<>(8, SIZE)));// MPSC size SIZE + list.add(makeParams(1, 1, 16, Ordering.FIFO, new SpscChunkedAtomicUnpaddedArrayQueue<>(8, 16)));// MPSC size 1 + list.add(makeParams(1, 1, SIZE, Ordering.FIFO, new SpscChunkedAtomicUnpaddedArrayQueue<>(8, SIZE)));// MPSC size SIZE return list; } diff --git a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpscGrowable.java b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpscGrowable.java index 5f7ddd32..24a87cc8 100644 --- a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpscGrowable.java +++ b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpscGrowable.java @@ -4,6 +4,7 @@ import org.jctools.queues.spec.ConcurrentQueueSpec; import org.jctools.queues.spec.Ordering; import org.jctools.queues.unpadded.SpscGrowableUnpaddedArrayQueue; +import org.jctools.queues.atomic.unpadded.SpscGrowableAtomicUnpaddedArrayQueue; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -31,6 +32,8 @@ public static Collection parameters() list.add(makeParams(1, 1, SIZE, Ordering.FIFO, new SpscGrowableAtomicArrayQueue<>(8, SIZE))); list.add(makeParams(1, 1, 16, Ordering.FIFO, new SpscGrowableUnpaddedArrayQueue<>(8, 16))); list.add(makeParams(1, 1, SIZE, Ordering.FIFO, new SpscGrowableUnpaddedArrayQueue<>(8, SIZE))); + list.add(makeParams(1, 1, 16, Ordering.FIFO, new SpscGrowableAtomicUnpaddedArrayQueue<>(8, 16))); + list.add(makeParams(1, 1, SIZE, Ordering.FIFO, new SpscGrowableAtomicUnpaddedArrayQueue<>(8, SIZE))); return list; } } diff --git a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpscLinked.java b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpscLinked.java index 8f17d0a2..01ef7bd3 100644 --- a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpscLinked.java +++ b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpscLinked.java @@ -25,6 +25,7 @@ public static Collection parameters() list.add(makeMpq(1, 1, 0, Ordering.FIFO));// unbounded SPSC list.add(makeAtomic(1, 1, 0, Ordering.FIFO));// unbounded SPSC list.add(makeUnpadded(1, 1, 0, Ordering.FIFO));// unbounded SPSC + list.add(makeAtomicUnpadded(1, 1, 0, Ordering.FIFO));// unbounded SPSC return list; } } diff --git a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpscUnbounded.java b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpscUnbounded.java index 979b3908..1c75695d 100644 --- a/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpscUnbounded.java +++ b/jctools-core/src/test/java/org/jctools/queues/MpqSanityTestSpscUnbounded.java @@ -4,6 +4,7 @@ import org.jctools.queues.spec.ConcurrentQueueSpec; import org.jctools.queues.spec.Ordering; import org.jctools.queues.unpadded.SpscUnboundedUnpaddedArrayQueue; +import org.jctools.queues.atomic.unpadded.SpscUnboundedAtomicUnpaddedArrayQueue; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -31,6 +32,8 @@ public static Collection parameters() list.add(makeParams(1, 1, 0, Ordering.FIFO, new SpscUnboundedAtomicArrayQueue<>(64))); list.add(makeParams(1, 1, 0, Ordering.FIFO, new SpscUnboundedUnpaddedArrayQueue<>(2))); list.add(makeParams(1, 1, 0, Ordering.FIFO, new SpscUnboundedUnpaddedArrayQueue<>(64))); + list.add(makeParams(1, 1, 0, Ordering.FIFO, new SpscUnboundedAtomicUnpaddedArrayQueue<>(2))); + list.add(makeParams(1, 1, 0, Ordering.FIFO, new SpscUnboundedAtomicUnpaddedArrayQueue<>(64))); return list; } } diff --git a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestMpmcArray.java b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestMpmcArray.java index 59068be0..08817832 100644 --- a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestMpmcArray.java +++ b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestMpmcArray.java @@ -2,7 +2,6 @@ import org.jctools.queues.spec.ConcurrentQueueSpec; import org.jctools.queues.spec.Ordering; -import org.jctools.util.TestUtil.*; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -34,6 +33,8 @@ public static Collection parameters() list.add(makeAtomic(0, 0, SIZE, Ordering.FIFO)); list.add(makeUnpadded(0, 0, 2, Ordering.FIFO)); list.add(makeUnpadded(0, 0, SIZE, Ordering.FIFO)); + list.add(makeAtomicUnpadded(0, 0, 2, Ordering.FIFO)); + list.add(makeAtomicUnpadded(0, 0, SIZE, Ordering.FIFO)); return list; } diff --git a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestMpscArray.java b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestMpscArray.java index 22ba92e2..1c923258 100644 --- a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestMpscArray.java +++ b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestMpscArray.java @@ -2,7 +2,6 @@ import org.jctools.queues.spec.ConcurrentQueueSpec; import org.jctools.queues.spec.Ordering; -import org.jctools.util.TestUtil.*; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -38,6 +37,9 @@ public static Collection parameters() list.add(makeUnpadded(0, 1, 1, Ordering.FIFO)); list.add(makeUnpadded(0, 1, 2, Ordering.FIFO)); list.add(makeUnpadded(0, 1, SIZE, Ordering.FIFO)); + list.add(makeAtomicUnpadded(0, 1, 1, Ordering.FIFO)); + list.add(makeAtomicUnpadded(0, 1, 2, Ordering.FIFO)); + list.add(makeAtomicUnpadded(0, 1, SIZE, Ordering.FIFO)); return list; } diff --git a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestMpscChunked.java b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestMpscChunked.java index 5593156a..edc4b48a 100644 --- a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestMpscChunked.java +++ b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestMpscChunked.java @@ -4,6 +4,7 @@ import org.jctools.queues.spec.ConcurrentQueueSpec; import org.jctools.queues.spec.Ordering; import org.jctools.queues.unpadded.MpscChunkedUnpaddedArrayQueue; +import org.jctools.queues.atomic.unpadded.MpscChunkedAtomicUnpaddedArrayQueue; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -34,6 +35,9 @@ public static Collection parameters() list.add(makeParams(0, 1, 4, Ordering.FIFO, new MpscChunkedUnpaddedArrayQueue<>(2, 4)));// MPSC size 1 list.add(makeParams(0, 1, SIZE, Ordering.FIFO, new MpscChunkedUnpaddedArrayQueue<>(8, SIZE)));// MPSC size SIZE list.add(makeParams(0, 1, 4096, Ordering.FIFO, new MpscChunkedUnpaddedArrayQueue<>(32, 4096)));// Netty recycler defaults + list.add(makeParams(0, 1, 4, Ordering.FIFO, new MpscChunkedAtomicUnpaddedArrayQueue<>(2, 4)));// MPSC size 1 + list.add(makeParams(0, 1, SIZE, Ordering.FIFO, new MpscChunkedAtomicUnpaddedArrayQueue<>(8, SIZE)));// MPSC size SIZE + list.add(makeParams(0, 1, 4096, Ordering.FIFO, new MpscChunkedAtomicUnpaddedArrayQueue<>(32, 4096)));// Netty recycler defaults return list; } } diff --git a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestMpscLinked.java b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestMpscLinked.java index 01f6c28e..61b10c67 100644 --- a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestMpscLinked.java +++ b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestMpscLinked.java @@ -26,6 +26,7 @@ public static Collection parameters() list.add(makeMpq(0, 1, 0, Ordering.FIFO)); list.add(makeAtomic(0, 1, 0, Ordering.FIFO)); list.add(makeUnpadded(0, 1, 0, Ordering.FIFO)); + list.add(makeAtomicUnpadded(0, 1, 0, Ordering.FIFO)); return list; } } diff --git a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestMpscUnboundedArray.java b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestMpscUnboundedArray.java index 04b20a5b..892ceb67 100644 --- a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestMpscUnboundedArray.java +++ b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestMpscUnboundedArray.java @@ -4,6 +4,7 @@ import org.jctools.queues.spec.ConcurrentQueueSpec; import org.jctools.queues.spec.Ordering; import org.jctools.queues.unpadded.MpscUnboundedUnpaddedArrayQueue; +import org.jctools.queues.atomic.unpadded.MpscUnboundedAtomicUnpaddedArrayQueue; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -31,6 +32,8 @@ public static Collection parameters() list.add(makeParams(0, 1, 0, Ordering.FIFO, new MpscUnboundedAtomicArrayQueue<>(64)));// MPSC size SIZE list.add(makeParams(0, 1, 0, Ordering.FIFO, new MpscUnboundedUnpaddedArrayQueue<>(2)));// MPSC size 1 list.add(makeParams(0, 1, 0, Ordering.FIFO, new MpscUnboundedUnpaddedArrayQueue<>(64)));// MPSC size SIZE + list.add(makeParams(0, 1, 0, Ordering.FIFO, new MpscUnboundedAtomicUnpaddedArrayQueue<>(2)));// MPSC size 1 + list.add(makeParams(0, 1, 0, Ordering.FIFO, new MpscUnboundedAtomicUnpaddedArrayQueue<>(64)));// MPSC size SIZE return list; } } diff --git a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpmcArray.java b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpmcArray.java index b79617e8..c34d4394 100644 --- a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpmcArray.java +++ b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpmcArray.java @@ -30,6 +30,8 @@ public static Collection parameters() list.add(makeAtomic(1, 0, SIZE, Ordering.FIFO)); list.add(makeUnpadded(1, 0, 1, Ordering.FIFO)); list.add(makeUnpadded(1, 0, SIZE, Ordering.FIFO)); + list.add(makeAtomicUnpadded(1, 0, 1, Ordering.FIFO)); + list.add(makeAtomicUnpadded(1, 0, SIZE, Ordering.FIFO)); return list; } diff --git a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpscArray.java b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpscArray.java index 48b94aec..404d92d8 100644 --- a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpscArray.java +++ b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpscArray.java @@ -29,6 +29,8 @@ public static Collection parameters() list.add(makeAtomic(1, 1, SIZE, Ordering.FIFO)); list.add(makeUnpadded(1, 1, 4, Ordering.FIFO)); list.add(makeUnpadded(1, 1, SIZE, Ordering.FIFO)); + list.add(makeAtomicUnpadded(1, 1, 4, Ordering.FIFO)); + list.add(makeAtomicUnpadded(1, 1, SIZE, Ordering.FIFO)); return list; } } diff --git a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpscChunked.java b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpscChunked.java index 0cc80119..9976e810 100644 --- a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpscChunked.java +++ b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpscChunked.java @@ -4,6 +4,7 @@ import org.jctools.queues.spec.ConcurrentQueueSpec; import org.jctools.queues.spec.Ordering; import org.jctools.queues.unpadded.SpscChunkedUnpaddedArrayQueue; +import org.jctools.queues.atomic.unpadded.SpscChunkedAtomicUnpaddedArrayQueue; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -31,6 +32,8 @@ public static Collection parameters() list.add(makeParams(1, 1, SIZE, Ordering.FIFO, new SpscChunkedAtomicArrayQueue<>(8, SIZE)));// MPSC size SIZE list.add(makeParams(1, 1, 16, Ordering.FIFO, new SpscChunkedUnpaddedArrayQueue<>(8, 16)));// MPSC size 1 list.add(makeParams(1, 1, SIZE, Ordering.FIFO, new SpscChunkedUnpaddedArrayQueue<>(8, SIZE)));// MPSC size SIZE + list.add(makeParams(1, 1, 16, Ordering.FIFO, new SpscChunkedAtomicUnpaddedArrayQueue<>(8, 16)));// MPSC size 1 + list.add(makeParams(1, 1, SIZE, Ordering.FIFO, new SpscChunkedAtomicUnpaddedArrayQueue<>(8, SIZE)));// MPSC size SIZE return list; } } diff --git a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpscGrowable.java b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpscGrowable.java index 94202444..ea362292 100644 --- a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpscGrowable.java +++ b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpscGrowable.java @@ -4,6 +4,7 @@ import org.jctools.queues.spec.ConcurrentQueueSpec; import org.jctools.queues.spec.Ordering; import org.jctools.queues.unpadded.SpscGrowableUnpaddedArrayQueue; +import org.jctools.queues.atomic.unpadded.SpscGrowableAtomicUnpaddedArrayQueue; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -35,6 +36,8 @@ public static Collection parameters() list.add(makeParams(1, 1, SIZE, Ordering.FIFO, new SpscGrowableAtomicArrayQueue<>(8, SIZE))); list.add(makeParams(1, 1, 16, Ordering.FIFO, new SpscGrowableUnpaddedArrayQueue<>(8, 16))); list.add(makeParams(1, 1, SIZE, Ordering.FIFO, new SpscGrowableUnpaddedArrayQueue<>(8, SIZE))); + list.add(makeParams(1, 1, 16, Ordering.FIFO, new SpscGrowableAtomicUnpaddedArrayQueue<>(8, 16))); + list.add(makeParams(1, 1, SIZE, Ordering.FIFO, new SpscGrowableAtomicUnpaddedArrayQueue<>(8, SIZE))); return list; } diff --git a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpscLinked.java b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpscLinked.java index 4929e31d..54b8f46c 100644 --- a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpscLinked.java +++ b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpscLinked.java @@ -26,6 +26,7 @@ public static Collection parameters() list.add(makeMpq(1, 1, 0, Ordering.FIFO)); list.add(makeAtomic(1, 1, 0, Ordering.FIFO)); list.add(makeUnpadded(1, 1, 0, Ordering.FIFO)); + list.add(makeAtomicUnpadded(1, 1, 0, Ordering.FIFO)); return list; } } diff --git a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpscUnbounded.java b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpscUnbounded.java index fc5260b2..169e6434 100644 --- a/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpscUnbounded.java +++ b/jctools-core/src/test/java/org/jctools/queues/QueueSanityTestSpscUnbounded.java @@ -4,6 +4,7 @@ import org.jctools.queues.spec.ConcurrentQueueSpec; import org.jctools.queues.spec.Ordering; import org.jctools.queues.unpadded.SpscUnboundedUnpaddedArrayQueue; +import org.jctools.queues.atomic.unpadded.SpscUnboundedAtomicUnpaddedArrayQueue; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -32,6 +33,9 @@ public static Collection parameters() list.add(makeParams(1, 1, 0, Ordering.FIFO, new SpscUnboundedAtomicArrayQueue<>(64))); list.add(makeParams(1, 1, 0, Ordering.FIFO, new SpscUnboundedUnpaddedArrayQueue<>(2))); list.add(makeParams(1, 1, 0, Ordering.FIFO, new SpscUnboundedUnpaddedArrayQueue<>(64))); + list.add(makeParams(1, 1, 0, Ordering.FIFO, new SpscUnboundedAtomicUnpaddedArrayQueue<>(2))); + list.add(makeParams(1, 1, 0, Ordering.FIFO, new SpscUnboundedAtomicUnpaddedArrayQueue<>(64))); + return list; } } diff --git a/jctools-core/src/test/java/org/jctools/util/TestUtil.java b/jctools-core/src/test/java/org/jctools/util/TestUtil.java index 9ddb7310..a74142d8 100644 --- a/jctools-core/src/test/java/org/jctools/util/TestUtil.java +++ b/jctools-core/src/test/java/org/jctools/util/TestUtil.java @@ -14,6 +14,7 @@ import static org.jctools.util.AtomicQueueFactory.newAtomicQueue; import static org.jctools.util.QueueFactory.newQueue; +import static org.jctools.util.UnpaddedQueueFactory.newAtomicUnpaddedQueue; import static org.jctools.util.UnpaddedQueueFactory.newUnpaddedQueue; public class TestUtil { @@ -81,6 +82,12 @@ public static Object[] makeUnpadded(int producers, int consumers, int capacity, return new Object[] {spec, newUnpaddedQueue(spec)}; } + public static Object[] makeAtomicUnpadded(int producers, int consumers, int capacity, Ordering ordering) + { + ConcurrentQueueSpec spec = makeSpec(producers, consumers, capacity, ordering); + return new Object[] {spec, newAtomicUnpaddedQueue(spec)}; + } + static ConcurrentQueueSpec makeSpec(int producers, int consumers, int capacity, Ordering ordering) { return new ConcurrentQueueSpec(producers, consumers, capacity, ordering, Preference.NONE); diff --git a/jctools-core/src/test/java/org/jctools/util/UnpaddedQueueFactory.java b/jctools-core/src/test/java/org/jctools/util/UnpaddedQueueFactory.java index f664b8d0..55220026 100644 --- a/jctools-core/src/test/java/org/jctools/util/UnpaddedQueueFactory.java +++ b/jctools-core/src/test/java/org/jctools/util/UnpaddedQueueFactory.java @@ -13,6 +13,7 @@ */ package org.jctools.util; +import org.jctools.queues.atomic.unpadded.*; import org.jctools.queues.spec.ConcurrentQueueSpec; import org.jctools.queues.unpadded.*; @@ -30,6 +31,49 @@ @Deprecated//(since = "4.0.0") public class UnpaddedQueueFactory { + + public static Queue newAtomicUnpaddedQueue(ConcurrentQueueSpec qs) + { + if (qs.isBounded()) + { + // SPSC + if (qs.isSpsc()) + { + return new SpscAtomicUnpaddedArrayQueue<>(qs.capacity); + } + // MPSC + else if (qs.isMpsc()) + { + return new MpscAtomicUnpaddedArrayQueue<>(qs.capacity); + } + // SPMC + else if (qs.isSpmc()) + { + return new SpmcAtomicUnpaddedArrayQueue<>(qs.capacity); + } + // MPMC + else + { + return new MpmcAtomicUnpaddedArrayQueue<>(qs.capacity); + } + } + else + { + // SPSC + if (qs.isSpsc()) + { + return new SpscLinkedAtomicUnpaddedQueue<>(); + } + // MPSC + else if (qs.isMpsc()) + { + return new MpscLinkedAtomicUnpaddedQueue<>(); + } + } + return new ConcurrentLinkedQueue(); + } + + public static Queue newUnpaddedQueue(ConcurrentQueueSpec qs) { if (qs.isBounded())