Skip to content

Commit

Permalink
Adding Atomic Unpadded queues
Browse files Browse the repository at this point in the history
  • Loading branch information
franz1981 committed Feb 7, 2024
1 parent 0dae71c commit e1d445e
Show file tree
Hide file tree
Showing 48 changed files with 4,473 additions and 21 deletions.
Expand Up @@ -15,6 +15,8 @@
import com.github.javaparser.ast.type.PrimitiveType;
import com.github.javaparser.ast.type.Type;

import java.util.Arrays;

/**
* This generator takes in an JCTools 'ArrayQueue' Java source file and patches {@link sun.misc.Unsafe} accesses into
* atomic {@link java.util.concurrent.atomic.AtomicLongFieldUpdater}. It outputs a Java source file with these patches.
Expand All @@ -25,11 +27,11 @@
public final class JavaParsingAtomicArrayQueueGenerator extends JavaParsingAtomicQueueGenerator {

public static void main(String[] args) throws Exception {
main(JavaParsingAtomicArrayQueueGenerator.class, args);
main(Boolean.parseBoolean(args[0]), JavaParsingAtomicArrayQueueGenerator.class, Arrays.copyOfRange(args, 1, args.length));
}

JavaParsingAtomicArrayQueueGenerator(String sourceFileName) {
super(sourceFileName);
JavaParsingAtomicArrayQueueGenerator(boolean unpadded ,String sourceFileName) {
super(unpadded, sourceFileName);
}

@Override
Expand Down Expand Up @@ -70,6 +72,17 @@ public void visit(ClassOrInterfaceDeclaration node, Void arg) {
+ JavaParsingAtomicArrayQueueGenerator.class.getName(),
"which can found in the jctools-build module. The original source file is " + sourceFileName + ".")
+ node.getJavadocComment().orElse(new JavadocComment("")).getContent());

if (unpadded) {
// remove padding fields
for (FieldDeclaration field : node.getFields())
{
String fieldName = field.getVariables().get(0).getNameAsString();
if (fieldName.startsWith("b0") || fieldName.startsWith("b1")) {
node.remove(field);
}
}
}
}

String fieldUpdaterFieldName(String fieldName) {
Expand Down
Expand Up @@ -13,6 +13,8 @@
import com.github.javaparser.ast.type.PrimitiveType;
import com.github.javaparser.ast.type.Type;

import java.util.Arrays;

/**
* This generator takes in an JCTools 'LinkedQueue' Java source file and patches {@link sun.misc.Unsafe} accesses into
* atomic {@link java.util.concurrent.atomic.AtomicLongFieldUpdater}. It outputs a Java source file with these patches.
Expand All @@ -22,13 +24,14 @@
*/
public final class JavaParsingAtomicLinkedQueueGenerator extends JavaParsingAtomicQueueGenerator {
private static final String MPSC_LINKED_ATOMIC_QUEUE_NAME = "MpscLinkedAtomicQueue";
private static final String MPSC_LINKED_UNPADDED_ATOMIC_QUEUE_NAME = "MpscLinkedAtomicUnpaddedQueue";

public static void main(String[] args) throws Exception {
main(JavaParsingAtomicLinkedQueueGenerator.class, args);
main(Boolean.parseBoolean(args[0]), JavaParsingAtomicLinkedQueueGenerator.class, Arrays.copyOfRange(args, 1, args.length));
}

JavaParsingAtomicLinkedQueueGenerator(String sourceFileName) {
super(sourceFileName);
JavaParsingAtomicLinkedQueueGenerator(boolean unpadded, String sourceFileName) {
super(unpadded, sourceFileName);
}

@Override
Expand All @@ -40,7 +43,8 @@ public void visit(ConstructorDeclaration n, Void arg) {
if (nameAsString.equals("WeakIterator"))
return;
n.setName(translateQueueName(nameAsString));
if (MPSC_LINKED_ATOMIC_QUEUE_NAME.equals(nameAsString)) {
String atomicQueueName = unpadded ? MPSC_LINKED_UNPADDED_ATOMIC_QUEUE_NAME : MPSC_LINKED_ATOMIC_QUEUE_NAME;
if (atomicQueueName.equals(nameAsString)) {
// Special case for MPSC because the Unsafe variant has a static factory method and a protected constructor.
n.setModifier(Keyword.PROTECTED, false);
n.setModifier(Keyword.PUBLIC, true);
Expand All @@ -56,7 +60,8 @@ public void visit(ClassOrInterfaceDeclaration node, Void arg) {
String nameAsString = node.getNameAsString();
if (nameAsString.contains("Queue"))
node.setName(translateQueueName(nameAsString));
if (MPSC_LINKED_ATOMIC_QUEUE_NAME.equals(nameAsString)) {
String atomicQueueName = unpadded ? MPSC_LINKED_UNPADDED_ATOMIC_QUEUE_NAME : MPSC_LINKED_ATOMIC_QUEUE_NAME;
if (atomicQueueName.equals(nameAsString)) {
/*
* Special case for MPSC
*/
Expand All @@ -80,6 +85,16 @@ public void visit(ClassOrInterfaceDeclaration node, Void arg) {
+ JavaParsingAtomicLinkedQueueGenerator.class.getName(),
"which can found in the jctools-build module. The original source file is " + sourceFileName + ".")
+ node.getJavadocComment().orElse(new JavadocComment("")).getContent());

if (unpadded) {
// remove padding fields
for (FieldDeclaration field : node.getFields()) {
String fieldName = field.getVariables().get(0).getNameAsString();
if (fieldName.startsWith("b0") || fieldName.startsWith("b1")) {
node.remove(field);
}
}
}
}

@Override
Expand Down
Expand Up @@ -59,11 +59,12 @@ abstract class JavaParsingAtomicQueueGenerator extends VoidVisitorAdapter<Void>

protected static final String INDENT_LEVEL = " ";
protected final String sourceFileName;
protected final boolean unpadded;

static void main(Class<? extends JavaParsingAtomicQueueGenerator> generatorClass, String[] args) throws Exception {
static void main(boolean unpadded, Class<? extends JavaParsingAtomicQueueGenerator> generatorClass, String[] args) throws Exception {

if (args.length < 2) {
throw new IllegalArgumentException("Usage: outputDirectory inputSourceFiles");
throw new IllegalArgumentException("Usage: unpadded outputDirectory inputSourceFiles");
}

File outputDirectory = new File(args[0]);
Expand All @@ -72,11 +73,15 @@ static void main(Class<? extends JavaParsingAtomicQueueGenerator> generatorClass
File file = new File(args[i]);
System.out.println("Processing " + file);
CompilationUnit cu = new JavaParser().parse(file).getResult().get();
JavaParsingAtomicQueueGenerator generator = buildGenerator(generatorClass, file.getName());
JavaParsingAtomicQueueGenerator generator = buildGenerator(generatorClass, unpadded, file.getName());
generator.visit(cu, null);

generator.organiseImports(cu);

if (unpadded) {
cleanupPaddingComments(cu);
}

String outputFileName = generator.translateQueueName(file.getName().replace(".java", "")) + ".java";

try (FileWriter writer = new FileWriter(new File(outputDirectory, outputFileName))) {
Expand All @@ -87,7 +92,21 @@ static void main(Class<? extends JavaParsingAtomicQueueGenerator> generatorClass
}
}

JavaParsingAtomicQueueGenerator(String sourceFileName) {


private static void cleanupPaddingComments(CompilationUnit cu)
{
for (Comment comment : cu.getAllContainedComments())
{
String content = comment.getContent();
if (content.contains("byte b") || content.contains("drop 8b") || content.contains("drop 16b") ) {
comment.remove();
}
}
}

JavaParsingAtomicQueueGenerator(boolean unpadded, String sourceFileName) {
this.unpadded = unpadded;
this.sourceFileName = sourceFileName;
}

Expand All @@ -98,7 +117,7 @@ static void main(Class<? extends JavaParsingAtomicQueueGenerator> generatorClass
public void visit(PackageDeclaration n, Void arg) {
super.visit(n, arg);
// Change the package of the output
n.setName("org.jctools.queues.atomic");
n.setName(unpadded? "org.jctools.queues.atomic.unpadded" : "org.jctools.queues.atomic");
}

@Override
Expand Down Expand Up @@ -151,11 +170,11 @@ protected void removeStaticFieldsAndInitialisers(ClassOrInterfaceDeclaration nod

protected String translateQueueName(String qName) {
if (qName.contains("LinkedQueue") || qName.contains("LinkedArrayQueue")) {
return qName.replace("Linked", "LinkedAtomic");
return qName.replace("Linked", unpadded ? "LinkedAtomicUnpadded" : "LinkedAtomic");
}

if (qName.contains("ArrayQueue")) {
return qName.replace("ArrayQueue", "AtomicArrayQueue");
return qName.replace("ArrayQueue", unpadded ? "AtomicUnpaddedArrayQueue" : "AtomicArrayQueue");
}

throw new IllegalArgumentException("Unexpected queue name: " + qName);
Expand Down Expand Up @@ -219,7 +238,7 @@ protected void replaceParentClassesForAtomics(ClassOrInterfaceDeclaration n) {
// ignore the JDK parent
break;
case "BaseLinkedQueue":
parent.setName("BaseLinkedAtomicQueue");
parent.setName(unpadded? "BaseLinkedAtomicUnpaddedQueue" : "BaseLinkedAtomicQueue");
break;
case "ConcurrentCircularArrayQueue":
parent.setName("AtomicReferenceArrayQueue");
Expand Down Expand Up @@ -258,7 +277,7 @@ protected void organiseImports(CompilationUnit cu) {
cu.addImport(new ImportDeclaration("java.util.concurrent.atomic", false, true));

cu.addImport(new ImportDeclaration("org.jctools.queues", false, true));
cu.addImport(staticImportDeclaration("org.jctools.queues.atomic.AtomicQueueUtil"));
cu.addImport(staticImportDeclaration(unpadded? "org.jctools.queues.atomic.unpadded.AtomicQueueUtil" : "org.jctools.queues.atomic.AtomicQueueUtil"));
}

protected String capitalise(String s) {
Expand Down Expand Up @@ -388,8 +407,8 @@ protected boolean isRefType(Type in, String className) {
return false;
}

private static <T> T buildGenerator(Class<? extends T> generatorClass, String fileName) throws Exception {
return generatorClass.getDeclaredConstructor(String.class).newInstance(fileName);
private static <T> T buildGenerator(Class<? extends T> generatorClass, boolean unpadded, String fileName) throws Exception {
return generatorClass.getDeclaredConstructor(boolean.class, String.class).newInstance(unpadded, fileName);
}

ImportDeclaration staticImportDeclaration(String name) {
Expand Down
59 changes: 59 additions & 0 deletions jctools-core/pom.xml
Expand Up @@ -133,6 +133,8 @@
<includeProjectDependencies>false</includeProjectDependencies>
<mainClass>org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator</mainClass>
<arguments>
<!-- unpadded -->
<argument>false</argument>
<!-- Output dest -->
<argument>${basedir}/src/main/java/org/jctools/queues/atomic</argument>
<!-- List of input files -->
Expand All @@ -154,6 +156,8 @@
<includeProjectDependencies>false</includeProjectDependencies>
<mainClass>org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator</mainClass>
<arguments>
<!-- unpadded -->
<argument>false</argument>
<!-- Output dest -->
<argument>${basedir}/src/main/java/org/jctools/queues/atomic</argument>
<!-- List of input files -->
Expand All @@ -173,6 +177,61 @@
</arguments>
</configuration>
</execution>
<execution>
<id>generate-atomic-unpadded-array-queues</id>
<goals>
<goal>java</goal>
</goals>
<phase>generate-sources</phase>
<configuration>
<includePluginDependencies>true</includePluginDependencies>
<includeProjectDependencies>false</includeProjectDependencies>
<mainClass>org.jctools.queues.atomic.JavaParsingAtomicArrayQueueGenerator</mainClass>
<arguments>
<!-- unpadded -->
<argument>true</argument>
<!-- Output dest -->
<argument>${basedir}/src/main/java/org/jctools/queues/atomic/unpadded</argument>
<!-- List of input files -->
<argument>${basedir}/src/main/java/org/jctools/queues/SpscArrayQueue.java</argument>
<argument>${basedir}/src/main/java/org/jctools/queues/SpmcArrayQueue.java</argument>
<argument>${basedir}/src/main/java/org/jctools/queues/MpscArrayQueue.java</argument>
<argument>${basedir}/src/main/java/org/jctools/queues/MpmcArrayQueue.java</argument>
</arguments>
</configuration>
</execution>
<execution>
<id>generate-atomic-unpadded-linked-queues</id>
<goals>
<goal>java</goal>
</goals>
<phase>generate-sources</phase>
<configuration>
<includePluginDependencies>true</includePluginDependencies>
<includeProjectDependencies>false</includeProjectDependencies>
<mainClass>org.jctools.queues.atomic.JavaParsingAtomicLinkedQueueGenerator</mainClass>
<arguments>
<!-- unpadded -->
<argument>true</argument>
<!-- Output dest -->
<argument>${basedir}/src/main/java/org/jctools/queues/atomic/unpadded</argument>
<!-- List of input files -->
<argument>${basedir}/src/main/java/org/jctools/queues/BaseLinkedQueue.java</argument>
<argument>${basedir}/src/main/java/org/jctools/queues/SpscLinkedQueue.java</argument>
<argument>${basedir}/src/main/java/org/jctools/queues/MpscLinkedQueue.java</argument>

<argument>${basedir}/src/main/java/org/jctools/queues/BaseSpscLinkedArrayQueue.java</argument>
<argument>${basedir}/src/main/java/org/jctools/queues/SpscChunkedArrayQueue.java</argument>
<argument>${basedir}/src/main/java/org/jctools/queues/SpscUnboundedArrayQueue.java</argument>
<argument>${basedir}/src/main/java/org/jctools/queues/SpscGrowableArrayQueue.java</argument>

<argument>${basedir}/src/main/java/org/jctools/queues/BaseMpscLinkedArrayQueue.java</argument>
<argument>${basedir}/src/main/java/org/jctools/queues/MpscChunkedArrayQueue.java</argument>
<argument>${basedir}/src/main/java/org/jctools/queues/MpscUnboundedArrayQueue.java</argument>
<argument>${basedir}/src/main/java/org/jctools/queues/MpscGrowableArrayQueue.java</argument>
</arguments>
</configuration>
</execution>
<execution>
<id>generate-unpadded-queues</id>
<goals>
Expand Down
@@ -0,0 +1,101 @@
package org.jctools.queues.atomic.unpadded;

import java.util.concurrent.atomic.AtomicLongArray;
import java.util.concurrent.atomic.AtomicReferenceArray;

final class AtomicQueueUtil
{
static <E> E lvRefElement(AtomicReferenceArray<E> buffer, int offset)
{
return buffer.get(offset);
}

static <E> E lpRefElement(AtomicReferenceArray<E> buffer, int offset)
{
return buffer.get(offset); // no weaker form available
}

static <E> void spRefElement(AtomicReferenceArray<E> buffer, int offset, E value)
{
buffer.lazySet(offset, value); // no weaker form available
}

static void soRefElement(AtomicReferenceArray buffer, int offset, Object value)
{
buffer.lazySet(offset, value);
}

static <E> void svRefElement(AtomicReferenceArray<E> buffer, int offset, E value)
{
buffer.set(offset, value);
}

static int calcRefElementOffset(long index)
{
return (int) index;
}

static int calcCircularRefElementOffset(long index, long mask)
{
return (int) (index & mask);
}

static <E> AtomicReferenceArray<E> allocateRefArray(int capacity)
{
return new AtomicReferenceArray<E>(capacity);
}

static void spLongElement(AtomicLongArray buffer, int offset, long e)
{
buffer.lazySet(offset, e);
}

static void soLongElement(AtomicLongArray buffer, int offset, long e)
{
buffer.lazySet(offset, e);
}

static long lpLongElement(AtomicLongArray buffer, int offset)
{
return buffer.get(offset);
}

static long lvLongElement(AtomicLongArray buffer, int offset)
{
return buffer.get(offset);
}

static int calcLongElementOffset(long index)
{
return (int) index;
}

static int calcCircularLongElementOffset(long index, int mask)
{
return (int) (index & mask);
}

static AtomicLongArray allocateLongArray(int capacity)
{
return new AtomicLongArray(capacity);
}

static int length(AtomicReferenceArray<?> buf)
{
return buf.length();
}

/**
* This method assumes index is actually (index << 1) because lower bit is used for resize hence the >> 1
*/
static int modifiedCalcCircularRefElementOffset(long index, long mask)
{
return (int) (index & mask) >> 1;
}

static int nextArrayOffset(AtomicReferenceArray<?> curr)
{
return length(curr) - 1;
}

}

0 comments on commit e1d445e

Please sign in to comment.