diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DumpStore.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DumpStore.java index f90346965f205..138c5ba46ba88 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DumpStore.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/store/DumpStore.java @@ -294,6 +294,7 @@ private boolean dumpRecord( STORE store, int size, StoreChannel fileChannel, Byt out.print( record ); buffer.clear(); fileChannel.read( buffer, id * size ); + buffer.flip(); dumpHex( record, buffer, id, size ); } @@ -320,7 +321,7 @@ else if ( allZero( buffer ) ) private boolean allZero( ByteBuffer buffer ) { - for ( int i = 0; i < buffer.position(); i++ ) + for ( int i = 0; i < buffer.limit(); i++ ) { if ( buffer.get( i ) != 0 ) { diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/util/HexPrinter.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/util/HexPrinter.java index bc778508c2bb5..d99f19052b93b 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/util/HexPrinter.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/util/HexPrinter.java @@ -214,14 +214,14 @@ public HexPrinter append( ByteBuffer bytes, int offset, int length ) } /** - * Append the bytes in the byte buffer, starting from position 0, ending at the buffer's current position, - * into print stream + * Append the bytes in the byte buffer, from its current position to its limit into print stream. This operation + * will not move the buffers current position. * @param bytes * @return */ public HexPrinter append( ByteBuffer bytes ) { - return append( bytes, 0, bytes.position() ); + return append( bytes, bytes.position(), bytes.remaining() ); } /** @@ -307,7 +307,7 @@ public static String hex( ByteBuffer bytes, int offset, int length ) */ public static String hex( ByteBuffer bytes ) { - return hex( bytes, 0, bytes.position() ); + return hex( bytes, bytes.position(), bytes.limit() ); } /** diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/DumpStoreTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/DumpStoreTest.java index 75a1026b87e57..be78b67544904 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/store/DumpStoreTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/store/DumpStoreTest.java @@ -19,12 +19,12 @@ */ package org.neo4j.kernel.impl.store; +import org.junit.Test; + import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.nio.ByteBuffer; -import org.junit.Test; - import org.neo4j.kernel.impl.store.record.AbstractBaseRecord; import static org.junit.Assert.assertEquals; @@ -44,6 +44,8 @@ public void dumpStoreShouldPrintBufferWithContent() throws Exception { buffer.put( i ); } + buffer.flip(); + AbstractBaseRecord record = mock( AbstractBaseRecord.class ); // When diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/util/HexPrinterTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/util/HexPrinterTest.java index 154fd132da392..bca97d77fc5b9 100644 --- a/community/kernel/src/test/java/org/neo4j/kernel/impl/util/HexPrinterTest.java +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/util/HexPrinterTest.java @@ -19,14 +19,13 @@ */ package org.neo4j.kernel.impl.util; +import org.junit.Test; + import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.nio.ByteBuffer; -import org.junit.Test; - import static java.lang.String.format; - import static org.junit.Assert.assertEquals; public class HexPrinterTest @@ -165,12 +164,18 @@ public void shouldPrintPartOfByteBuffer() throws Exception @Test public void shouldOnlyPrintBytesWrittenToBuffer() throws Exception { + // Given ByteBuffer bytes = ByteBuffer.allocate( 1024 ); for ( byte value = 0; value < 10; value++ ) { bytes.put( value ); } + bytes.flip(); + + // When String hexString = HexPrinter.hex( bytes ); + + // Then assertEquals( format( "00 01 02 03 04 05 06 07 08 09" ), hexString ); } } diff --git a/community/ndp/kernelextension/pom.xml b/community/ndp/kernelextension/pom.xml index 1622ca4cd43b1..afa2dd5fe2ec6 100644 --- a/community/ndp/kernelextension/pom.xml +++ b/community/ndp/kernelextension/pom.xml @@ -29,17 +29,17 @@ GNU General Public License, Version 3 http://www.gnu.org/licenses/gpl-3.0-standalone.html The software ("Software") developed and owned by Network Engine for -Objects in Lund AB (referred to in this notice as "Neo Technology") is -licensed under the GNU GENERAL PUBLIC LICENSE Version 3 to all third -parties and that license is included below. + Objects in Lund AB (referred to in this notice as "Neo Technology") is + licensed under the GNU GENERAL PUBLIC LICENSE Version 3 to all third + parties and that license is included below. -However, if you have executed an End User Software License and Services -Agreement or an OEM Software License and Support Services Agreement, or -another commercial license agreement with Neo Technology or one of its -affiliates (each, a "Commercial Agreement"), the terms of the license in -such Commercial Agreement will supersede the GNU GENERAL PUBLIC LICENSE -Version 3 and you may use the Software solely pursuant to the terms of -the relevant Commercial Agreement. + However, if you have executed an End User Software License and Services + Agreement or an OEM Software License and Support Services Agreement, or + another commercial license agreement with Neo Technology or one of its + affiliates (each, a "Commercial Agreement"), the terms of the license in + such Commercial Agreement will supersede the GNU GENERAL PUBLIC LICENSE + Version 3 and you may use the Software solely pursuant to the terms of + the relevant Commercial Agreement. @@ -113,8 +113,8 @@ the relevant Commercial Agreement. maven-antrun-plugin - - + + generate-version none diff --git a/community/ndp/kernelextension/src/main/java/org/neo4j/ext/NDPKernelExtension.java b/community/ndp/kernelextension/src/main/java/org/neo4j/ext/NDPKernelExtension.java index fdd1cb1477c46..e86fc4eeb1559 100644 --- a/community/ndp/kernelextension/src/main/java/org/neo4j/ext/NDPKernelExtension.java +++ b/community/ndp/kernelextension/src/main/java/org/neo4j/ext/NDPKernelExtension.java @@ -47,19 +47,21 @@ public class NDPKernelExtension extends KernelExtensionFactory ndp_enabled = setting("experimental.ndp.enabled", BOOLEAN, + @Description("Max time that sessions can be idle, after this interval a session will get closed.") + public static final Setting ndp_enabled = setting( "experimental.ndp.enabled", BOOLEAN, "false" ); - @Description( "Host and port for the Neo4j Data Protocol http transport" ) + @Description("Host and port for the Neo4j Data Protocol http transport") public static final Setting ndp_address = - setting("dbms.ndp.address", HOSTNAME_PORT, "localhost:7687" ); + setting( "dbms.ndp.address", HOSTNAME_PORT, "localhost:7687" ); } public interface Dependencies { LogService logService(); + Config config(); + GraphDatabaseService db(); } @@ -78,7 +80,7 @@ public Lifecycle newKernelExtension( Dependencies dependencies ) throws Throwabl final HostnamePort address = config.get( Settings.ndp_address ); final LifeSupport life = new LifeSupport(); - if(config.get( Settings.ndp_enabled )) + if ( config.get( Settings.ndp_enabled ) ) { final Sessions env = life.add( new StandardSessions( api, log ) ); diff --git a/community/ndp/kernelextension/src/test/java/org/neo4j/ext/NDPExtensionIT.java b/community/ndp/kernelextension/src/test/java/org/neo4j/ext/NDPExtensionIT.java index d866e02f87bcb..613aa64eb9f9b 100644 --- a/community/ndp/kernelextension/src/test/java/org/neo4j/ext/NDPExtensionIT.java +++ b/community/ndp/kernelextension/src/test/java/org/neo4j/ext/NDPExtensionIT.java @@ -51,7 +51,7 @@ public void shouldLaunchNDP() throws Throwable .newGraphDatabase(); // Then - assertEventuallyServerResponds("localhost", 7687); + assertEventuallyServerResponds( "localhost", 7687 ); } @Test @@ -65,25 +65,25 @@ public void shouldBeAbleToSpecifyHostAndPort() throws Throwable .newGraphDatabase(); // Then - assertEventuallyServerResponds("localhost", 8776); + assertEventuallyServerResponds( "localhost", 8776 ); } - private void assertEventuallyServerResponds(String host, int port) throws IOException, InterruptedException + private void assertEventuallyServerResponds( String host, int port ) throws IOException, InterruptedException { long timeout = System.currentTimeMillis() + 1000 * 30; - for(;;) + for (; ; ) { - if ( serverResponds(host, port) ) + if ( serverResponds( host, port ) ) { return; } else { - Thread.sleep(100); + Thread.sleep( 100 ); } // Make sure process still is alive - if(System.currentTimeMillis() > timeout) + if ( System.currentTimeMillis() > timeout ) { throw new RuntimeException( "Waited for 30 seconds for server to respond to HTTP calls, " + "but no response, timing out to avoid blocking forever." ); @@ -95,7 +95,7 @@ private boolean serverResponds( String host, int port ) throws IOException, Inte { try { - try(Socket socket = new Socket()) + try ( Socket socket = new Socket() ) { // Ok, we can connect - can we perform the version handshake? socket.connect( new InetSocketAddress( host, port ) ); @@ -103,15 +103,15 @@ private boolean serverResponds( String host, int port ) throws IOException, Inte InputStream in = socket.getInputStream(); // Hard-coded handshake, a general "test client" would be useful further on. - out.write( new byte[]{ 0,0,0,1, 0,0,0,0, 0,0,0,0, 0,0,0,0 } ); + out.write( new byte[]{0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} ); byte[] accepted = new byte[4]; in.read( accepted ); - return Arrays.equals( accepted, new byte[]{0,0,0,1} ); + return Arrays.equals( accepted, new byte[]{0, 0, 0, 1} ); } } - catch(ConnectException e) + catch ( ConnectException e ) { return false; } @@ -120,7 +120,7 @@ private boolean serverResponds( String host, int port ) throws IOException, Inte @After public void cleanup() { - if(db != null) + if ( db != null ) { db.shutdown(); } diff --git a/community/ndp/messaging-v1/pom.xml b/community/ndp/messaging-v1/pom.xml index 275d66f37c987..3d2daceacbb46 100644 --- a/community/ndp/messaging-v1/pom.xml +++ b/community/ndp/messaging-v1/pom.xml @@ -49,19 +49,30 @@ org.hamcrest hamcrest-all - 1.3 test junit junit - 4.11 test org.mockito mockito-all - 1.10.19 + test + + + + + org.asciidoctor + asciidoctorj + 1.5.2 + test + + + org.jsoup + jsoup + 1.8.1 test @@ -70,7 +81,6 @@ maven-compiler-plugin - 2.3.2 1.7 1.7 @@ -79,7 +89,6 @@ org.apache.maven.plugins maven-jar-plugin - 2.2 @@ -98,6 +107,35 @@ + + + + org.asciidoctor + asciidoctor-maven-plugin + 1.5.2 + + + generate-compliance-manual + generate-resources + + process-asciidoc + + + html + + + + + ../v1-docs/src/docs/dev + ${basedir}/target/messaging-manual/html + html + + + coderay + + + + diff --git a/community/ndp/messaging-v1/src/main/java/org/neo4j/ndp/messaging/v1/PackStreamMessageFormatV1.java b/community/ndp/messaging-v1/src/main/java/org/neo4j/ndp/messaging/v1/PackStreamMessageFormatV1.java index a9f83681e279d..9ded0d3a80072 100644 --- a/community/ndp/messaging-v1/src/main/java/org/neo4j/ndp/messaging/v1/PackStreamMessageFormatV1.java +++ b/community/ndp/messaging-v1/src/main/java/org/neo4j/ndp/messaging/v1/PackStreamMessageFormatV1.java @@ -36,6 +36,7 @@ import org.neo4j.graphdb.PropertyContainer; import org.neo4j.graphdb.Relationship; import org.neo4j.helpers.collection.Iterables; +import org.neo4j.kernel.api.exceptions.Status; import org.neo4j.ndp.messaging.v1.infrastructure.ValueNode; import org.neo4j.ndp.messaging.v1.infrastructure.ValuePath; import org.neo4j.ndp.messaging.v1.infrastructure.ValueRelationship; @@ -114,8 +115,13 @@ public void run() */ public Writer( PackOutput output, Runnable onMessageComplete ) { + this( new PackStream.Packer( output ), onMessageComplete ); + } + + public Writer( PackStream.Packer packer, Runnable onMessageComplete ) + { + this.packer = packer; this.onMessageComplete = onMessageComplete; - packer = new PackStream.Packer( output ); } @Override @@ -480,8 +486,13 @@ private void unpackFailureMessage( MessageHandler outpu { Map map = unpackRawMap(); - String codeStr = (String) map.get( "code" ); - String msg = (String) map.get( "message" ); + String codeStr = map.containsKey( "code" ) ? + (String) map.get( "code" ) : + Status.General.UnknownFailure.name(); + + String msg = map.containsKey( "message" ) ? + (String) map.get( "message" ) : + ""; output.handleFailureMessage( new Neo4jError( codeFromString( codeStr ), msg ) ); } diff --git a/community/ndp/messaging-v1/src/test/java/org/neo4j/ndp/messaging/v1/MessageFormatTest.java b/community/ndp/messaging-v1/src/test/java/org/neo4j/ndp/messaging/v1/MessageFormatTest.java index c9f514c7d1b84..2bca160528d9e 100644 --- a/community/ndp/messaging-v1/src/test/java/org/neo4j/ndp/messaging/v1/MessageFormatTest.java +++ b/community/ndp/messaging-v1/src/test/java/org/neo4j/ndp/messaging/v1/MessageFormatTest.java @@ -19,13 +19,13 @@ */ package org.neo4j.ndp.messaging.v1; +import org.junit.Test; + import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; -import org.junit.Test; - import org.neo4j.graphdb.DynamicRelationshipType; import org.neo4j.kernel.api.exceptions.Status; import org.neo4j.kernel.impl.util.HexPrinter; @@ -44,10 +44,8 @@ import org.neo4j.ndp.runtime.internal.Neo4jError; import static java.util.Arrays.asList; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; - import static org.neo4j.graphdb.DynamicLabel.label; import static org.neo4j.helpers.collection.MapUtil.map; import static org.neo4j.stream.Records.record; diff --git a/community/ndp/messaging-v1/src/test/java/org/neo4j/ndp/messaging/v1/util/MessageMatchers.java b/community/ndp/messaging-v1/src/test/java/org/neo4j/ndp/messaging/v1/util/MessageMatchers.java index 1cd8a123de195..2bc2b14ebab2b 100644 --- a/community/ndp/messaging-v1/src/test/java/org/neo4j/ndp/messaging/v1/util/MessageMatchers.java +++ b/community/ndp/messaging-v1/src/test/java/org/neo4j/ndp/messaging/v1/util/MessageMatchers.java @@ -19,15 +19,15 @@ */ package org.neo4j.ndp.messaging.v1.util; +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeMatcher; + import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Map; -import org.hamcrest.Description; -import org.hamcrest.Matcher; -import org.hamcrest.TypeSafeMatcher; - import org.neo4j.kernel.impl.util.HexPrinter; import org.neo4j.ndp.messaging.v1.PackStreamMessageFormatV1; import org.neo4j.ndp.messaging.v1.RecordingByteChannel; diff --git a/community/ndp/packstream-v1/pom.xml b/community/ndp/packstream-v1/pom.xml index 352838a43548a..57d7838c63267 100644 --- a/community/ndp/packstream-v1/pom.xml +++ b/community/ndp/packstream-v1/pom.xml @@ -25,17 +25,17 @@ GNU General Public License, Version 3 http://www.gnu.org/licenses/gpl-3.0-standalone.html The software ("Software") developed and owned by Network Engine for -Objects in Lund AB (referred to in this notice as "Neo Technology") is -licensed under the GNU GENERAL PUBLIC LICENSE Version 3 to all third -parties and that license is included below. + Objects in Lund AB (referred to in this notice as "Neo Technology") is + licensed under the GNU GENERAL PUBLIC LICENSE Version 3 to all third + parties and that license is included below. -However, if you have executed an End User Software License and Services -Agreement or an OEM Software License and Support Services Agreement, or -another commercial license agreement with Neo Technology or one of its -affiliates (each, a "Commercial Agreement"), the terms of the license in -such Commercial Agreement will supersede the GNU GENERAL PUBLIC LICENSE -Version 3 and you may use the Software solely pursuant to the terms of -the relevant Commercial Agreement. + However, if you have executed an End User Software License and Services + Agreement or an OEM Software License and Support Services Agreement, or + another commercial license agreement with Neo Technology or one of its + affiliates (each, a "Commercial Agreement"), the terms of the license in + such Commercial Agreement will supersede the GNU GENERAL PUBLIC LICENSE + Version 3 and you may use the Software solely pursuant to the terms of + the relevant Commercial Agreement. @@ -66,7 +66,6 @@ the relevant Commercial Agreement. maven-compiler-plugin - 2.3.2 1.7 1.7 @@ -75,13 +74,23 @@ the relevant Commercial Agreement. maven-antrun-plugin - - + + generate-version none + + maven-jar-plugin + + + + test-jar + + + + diff --git a/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/BufferedChannelInput.java b/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/BufferedChannelInput.java index a4e3c2ec628e4..10846e6867640 100644 --- a/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/BufferedChannelInput.java +++ b/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/BufferedChannelInput.java @@ -37,15 +37,15 @@ public BufferedChannelInput( int bufferCapacity ) public BufferedChannelInput reset( ReadableByteChannel ch ) { this.channel = ch; - this.buffer.position(0); - this.buffer.limit(0); + this.buffer.position( 0 ); + this.buffer.limit( 0 ); return this; } @Override public PackInput ensure( int numBytes ) throws IOException { - if(!attempt( numBytes )) + if ( !attempt( numBytes ) ) { throw new PackStream.EndOfStream( "Unexpected end of stream while trying to read " + numBytes + " bytes." ); } @@ -55,19 +55,19 @@ public PackInput ensure( int numBytes ) throws IOException @Override public PackInput attemptUpTo( int numBytes ) throws IOException { - attempt( Math.min(numBytes, buffer.capacity()) ); + attempt( Math.min( numBytes, buffer.capacity() ) ); return this; } @Override public boolean attempt( int numBytes ) throws IOException { - if(remaining() >= numBytes) + if ( remaining() >= numBytes ) { return true; } - if(buffer.remaining() > 0) + if ( buffer.remaining() > 0 ) { // If there is data remaining in the buffer, shift that remaining data to the beginning of the buffer. buffer.compact(); @@ -81,7 +81,8 @@ public boolean attempt( int numBytes ) throws IOException do { count = channel.read( buffer ); - } while( count >= 0 && (buffer.position() < numBytes && buffer.remaining() != 0)); + } + while ( count >= 0 && (buffer.position() < numBytes && buffer.remaining() != 0) ); buffer.flip(); return buffer.remaining() >= numBytes; @@ -126,13 +127,13 @@ public double getDouble() @Override public PackInput get( byte[] into, int offset, int toRead ) { - buffer.get(into, offset, toRead); + buffer.get( into, offset, toRead ); return this; } @Override public byte peek() { - return buffer.get(buffer.position()); + return buffer.get( buffer.position() ); } } diff --git a/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/BufferedChannelOutput.java b/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/BufferedChannelOutput.java index 8de760cf468fe..9569a07a631aa 100644 --- a/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/BufferedChannelOutput.java +++ b/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/BufferedChannelOutput.java @@ -41,7 +41,7 @@ public BufferedChannelOutput( WritableByteChannel channel ) public BufferedChannelOutput( WritableByteChannel channel, int bufferSize ) { - this(bufferSize); + this( bufferSize ); reset( channel ); } @@ -64,7 +64,7 @@ public BufferedChannelOutput ensure( int size ) throws IOException public BufferedChannelOutput flush() throws IOException { buffer.flip(); - do { channel.write( buffer ); } while( buffer.remaining() > 0 ); + do { channel.write( buffer ); } while ( buffer.remaining() > 0 ); buffer.clear(); return this; } @@ -80,14 +80,14 @@ public PackOutput put( byte value ) public PackOutput put( byte[] data, int offset, int length ) throws IOException { int index = 0; - while(index < length) + while ( index < length ) { int amountToWrite = Math.min( buffer.remaining(), length - index ); - buffer.put( data, offset, amountToWrite ); + buffer.put( data, offset + index, amountToWrite ); index += amountToWrite; - if( buffer.remaining() == 0) + if ( buffer.remaining() == 0 ) { flush(); } diff --git a/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/PackInput.java b/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/PackInput.java index ef167c5879edf..a64ad07a3d5cf 100644 --- a/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/PackInput.java +++ b/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/PackInput.java @@ -25,6 +25,7 @@ public interface PackInput { /** * Ensure the specified number of bytes are available for reading + * * @throws org.neo4j.packstream.PackStream.EndOfStream if there are not enough bytes available */ PackInput ensure( int numBytes ) throws IOException; diff --git a/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/PackStream.java b/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/PackStream.java index 1a1c756433159..3367e8ed6bede 100644 --- a/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/PackStream.java +++ b/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/PackStream.java @@ -40,13 +40,14 @@ * PackStream is a messaging serialisation format heavily inspired by MessagePack. * The key differences are in the type system itself which (among other things) replaces extensions with structures. * The Packer and Unpacker implementations are also faster than their MessagePack counterparts. - * + *

* Note that several marker byte values are RESERVED for future use. - * Extra markers should not be added casually and such additions must be follow a strict process involving both client and server software. - * + * Extra markers should not be added casually and such additions must be follow a strict process involving both + * client and server software. + *

* The table below shows all allocated marker byte values. - * - * + *

+ *

* * * @@ -54,7 +55,8 @@ * * * - * + * * * * @@ -62,34 +64,48 @@ * * * - * - * - * + * + * + * * - * - * - * + * + * + * * - * - * - * + * + * + * * - * - * - * + * + * + * * - * - * - * + * + * + * * * * *
MarkerBinaryTypeDescription
00..7F0xxxxxxx+TINY_INTInteger 0 to 127
80..8F1000xxxxTINY_TEXT
A0..AF1010xxxxTINY_MAP
B0..BF1011xxxxTINY_STRUCT
C011000000NULL
C111000001FLOAT_6464-bit floating point number (double)
C111000001FLOAT_6464-bit floating point number + * (double)
C211000010FALSEBoolean false
C311000011TRUEBoolean true
C4..C7110001xxRESERVED
C911001001INT_816-bit signed integer
CA11001010INT_832-bit signed integer
CB11001011INT_864-bit signed integer
CC11001100BYTES_8Byte string (fewer than 28 bytes)
CD11001101BYTES_16Byte string (fewer than 216 bytes)
CE11001110BYTES_32Byte string (fewer than 232 bytes)
CC11001100BYTES_8Byte string (fewer than 28 + * bytes)
CD11001101BYTES_16Byte string (fewer than 216 + * bytes)
CE11001110BYTES_32Byte string (fewer than 232 + * bytes)
CF11001111RESERVED
D011010000TEXT_8UTF-8 encoded text string (fewer than 28 bytes)
D111010001TEXT_16UTF-8 encoded text string (fewer than 216 bytes)
D211010010TEXT_32UTF-8 encoded text string (fewer than 232 bytes)
D011010000TEXT_8UTF-8 encoded text string (fewer than + * 28 bytes)
D111010001TEXT_16UTF-8 encoded text string (fewer than + * 216 bytes)
D211010010TEXT_32UTF-8 encoded text string (fewer than + * 232 bytes)
D311010011RESERVED
D411010100LIST_8List (fewer than 28 items)
D511010101LIST_16List (fewer than 216 items)
D611010110LIST_32List (fewer than 232 items)
D411010100LIST_8List (fewer than 28 + * items)
D511010101LIST_16List (fewer than 216 + * items)
D611010110LIST_32List (fewer than 232 + * items)
D711010111RESERVED
D811011000MAP_8Map (fewer than 28 key:value pairs)
D911011001MAP_16Map (fewer than 216 key:value pairs)
DA11011010MAP_32Map (fewer than 232 key:value pairs)
D811011000MAP_8Map (fewer than 28 key:value + * pairs)
D911011001MAP_16Map (fewer than 216 key:value + * pairs)
DA11011010MAP_32Map (fewer than 232 key:value + * pairs)
DB11011011RESERVED
DC11011100STRUCT_8Structure (fewer than 28 fields)
DD11011101STRUCT_16Structure (fewer than 216 fields)
DE11011110STRUCT_32Structure (fewer than 232 fields)
DC11011100STRUCT_8Structure (fewer than 28 + * fields)
DD11011101STRUCT_16Structure (fewer than 216 + * fields)
DE11011110STRUCT_32Structure (fewer than 232 + * fields)
DF11011111RESERVED
E0..EF1110xxxxRESERVED
F0..FF1111xxxx-TINY_INTInteger -1 to -16
- * */ public class PackStream { - + public static final byte TINY_TEXT = (byte) 0x80; public static final byte TINY_LIST = (byte) 0x90; public static final byte TINY_MAP = (byte) 0xA0; @@ -143,11 +159,11 @@ public class PackStream public static final byte RESERVED_EE = (byte) 0xEE; public static final byte RESERVED_EF = (byte) 0xEF; - private static final long PLUS_2_TO_THE_31 = 2147483648L; - private static final long PLUS_2_TO_THE_15 = 32768L; - private static final long PLUS_2_TO_THE_7 = 128L; - private static final long MINUS_2_TO_THE_4 = -16L; - private static final long MINUS_2_TO_THE_7 = -128L; + private static final long PLUS_2_TO_THE_31 = 2147483648L; + private static final long PLUS_2_TO_THE_15 = 32768L; + private static final long PLUS_2_TO_THE_7 = 128L; + private static final long MINUS_2_TO_THE_4 = -16L; + private static final long MINUS_2_TO_THE_7 = -128L; private static final long MINUS_2_TO_THE_15 = -32768L; private static final long MINUS_2_TO_THE_31 = -2147483648L; @@ -155,7 +171,9 @@ public class PackStream private static final int DEFAULT_BUFFER_CAPACITY = 8192; - private PackStream() {} + private PackStream() + { + } public static class Packer { @@ -189,53 +207,53 @@ public void packRaw( byte[] data ) throws IOException public void packNull() throws IOException { out.ensure( 1 ) - .put( NULL ); + .put( NULL ); } public void pack( boolean value ) throws IOException { out.ensure( 1 ) - .put( value ? TRUE : FALSE ); + .put( value ? TRUE : FALSE ); } public void pack( long value ) throws IOException { - if ( value >= MINUS_2_TO_THE_4 && value < PLUS_2_TO_THE_7) + if ( value >= MINUS_2_TO_THE_4 && value < PLUS_2_TO_THE_7 ) { out.ensure( 1 ) - .put( (byte) value ); + .put( (byte) value ); } else if ( value >= MINUS_2_TO_THE_7 && value < MINUS_2_TO_THE_4 ) { out.ensure( 2 ) - .put( INT_8 ) - .put( (byte) value ); + .put( INT_8 ) + .put( (byte) value ); } else if ( value >= MINUS_2_TO_THE_15 && value < PLUS_2_TO_THE_15 ) { out.ensure( 3 ) - .put( INT_16 ) - .putShort( (short) value ); + .put( INT_16 ) + .putShort( (short) value ); } else if ( value >= MINUS_2_TO_THE_31 && value < PLUS_2_TO_THE_31 ) { out.ensure( 5 ) - .put( INT_32 ) - .putInt( (int) value ); + .put( INT_32 ) + .putInt( (int) value ); } else { out.ensure( 9 ) - .put( INT_64 ) - .putLong( value ); + .put( INT_64 ) + .putLong( value ); } } public void pack( double value ) throws IOException { out.ensure( 9 ) - .put( FLOAT_64 ) - .putDouble( value ); + .put( FLOAT_64 ) + .putDouble( value ); } public void pack( byte[] values ) throws IOException @@ -327,20 +345,20 @@ public void packBytesHeader( int size ) throws IOException if ( size <= Byte.MAX_VALUE ) { out.ensure( 2 ) - .put( BYTES_8 ) - .put( (byte) size ); + .put( BYTES_8 ) + .put( (byte) size ); } else if ( size <= Short.MAX_VALUE ) { out.ensure( 3 ) - .put( BYTES_16 ) - .putShort( (short) size ); + .put( BYTES_16 ) + .putShort( (short) size ); } else { out.ensure( 5 ) - .put( BYTES_32 ) - .putInt( size ); + .put( BYTES_32 ) + .putInt( size ); } } @@ -349,25 +367,25 @@ public void packTextHeader( int size ) throws IOException if ( size < 0x10 ) { out.ensure( 1 ) - .put( (byte) (TINY_TEXT | size) ); + .put( (byte) (TINY_TEXT | size) ); } else if ( size <= Byte.MAX_VALUE ) { out.ensure( 2 ) - .put( TEXT_8 ) - .put( (byte) size ); + .put( TEXT_8 ) + .put( (byte) size ); } else if ( size <= Short.MAX_VALUE ) { out.ensure( 3 ) - .put( TEXT_16 ) - .putShort( (short) size ); + .put( TEXT_16 ) + .putShort( (short) size ); } else { out.ensure( 5 ) - .put( TEXT_32 ) - .putInt( size ); + .put( TEXT_32 ) + .putInt( size ); } } @@ -376,25 +394,25 @@ public void packListHeader( int size ) throws IOException if ( size < 0x10 ) { out.ensure( 1 ) - .put( (byte) (TINY_LIST | size) ); + .put( (byte) (TINY_LIST | size) ); } else if ( size <= Byte.MAX_VALUE ) { out.ensure( 2 ) - .put( LIST_8 ) - .put( (byte) size ); + .put( LIST_8 ) + .put( (byte) size ); } else if ( size <= Short.MAX_VALUE ) { out.ensure( 3 ) - .put( LIST_16 ) - .putShort( (short) size ); + .put( LIST_16 ) + .putShort( (short) size ); } else { out.ensure( 5 ) - .put( LIST_32 ) - .putInt( size ); + .put( LIST_32 ) + .putInt( size ); } } @@ -403,25 +421,25 @@ public void packMapHeader( int size ) throws IOException if ( size < 0x10 ) { out.ensure( 1 ) - .put( (byte) (TINY_MAP | size) ); + .put( (byte) (TINY_MAP | size) ); } else if ( size <= Byte.MAX_VALUE ) { out.ensure( 2 ) - .put( MAP_8 ) - .put( (byte) size ); + .put( MAP_8 ) + .put( (byte) size ); } else if ( size <= Short.MAX_VALUE ) { out.ensure( 3 ) - .put( MAP_16 ) - .putShort( (short) size ); + .put( MAP_16 ) + .putShort( (short) size ); } else { out.ensure( 5 ) - .put( MAP_32 ) - .putInt( size ); + .put( MAP_32 ) + .putInt( size ); } } @@ -430,22 +448,22 @@ public void packStructHeader( int size, char signature ) throws IOException if ( size < 0x10 ) { out.ensure( 2 ) - .put( (byte) (TINY_STRUCT | size) ) - .put( (byte) signature ); + .put( (byte) (TINY_STRUCT | size) ) + .put( (byte) signature ); } else if ( size <= Byte.MAX_VALUE ) { out.ensure( 3 ) - .put( STRUCT_8 ) - .put( (byte) size ) - .put( (byte) signature ); + .put( STRUCT_8 ) + .put( (byte) size ) + .put( (byte) signature ); } else if ( size <= Short.MAX_VALUE ) { out.ensure( 4 ) - .put( STRUCT_16 ) - .putShort( (short) size ) - .put( (byte) signature ); + .put( STRUCT_16 ) + .putShort( (short) size ) + .put( (byte) signature ); } else { @@ -479,7 +497,7 @@ public Unpacker( PackInput in ) public Unpacker reset( ReadableByteChannel ch ) { - ((BufferedChannelInput)in).reset( ch ); + ((BufferedChannelInput) in).reset( ch ); return this; } @@ -488,6 +506,9 @@ public boolean hasNext() throws IOException return in.remaining() > 0 || in.attempt( 1 ); } + // TODO: This currently returns the number of fields in the struct. In 99% of cases we will look at the struct + // signature to determine how to read it, suggest we make that what we return here, + // and have the number of fields available through some alternate optional mechanism. public long unpackStructHeader() throws IOException { final byte markerByte = in.ensure( 1 ).get(); @@ -495,11 +516,14 @@ public long unpackStructHeader() throws IOException final byte markerLowNibble = (byte) (markerByte & 0x0F); if ( markerHighNibble == TINY_STRUCT ) { return markerLowNibble; } - switch(markerByte) + switch ( markerByte ) { - case STRUCT_8: return unpackUINT8(); - case STRUCT_16: return unpackUINT16(); - default: throw new Unexpected( "Expected a struct, but got: " + toHexString( markerByte )); + case STRUCT_8: + return unpackUINT8(); + case STRUCT_16: + return unpackUINT16(); + default: + throw new Unexpected( "Expected a struct, but got: " + toHexString( markerByte ) ); } } @@ -512,15 +536,19 @@ public long unpackListHeader() throws IOException { final byte markerByte = in.ensure( 1 ).get(); final byte markerHighNibble = (byte) (markerByte & 0xF0); - final byte markerLowNibble = (byte) (markerByte & 0x0F); + final byte markerLowNibble = (byte) (markerByte & 0x0F); if ( markerHighNibble == TINY_LIST ) { return markerLowNibble; } - switch(markerByte) + switch ( markerByte ) { - case LIST_8: return unpackUINT8(); - case LIST_16: return unpackUINT16(); - case LIST_32: return unpackUINT32(); - default: throw new Unexpected( "Expected a list, but got: " + toHexString( markerByte )); + case LIST_8: + return unpackUINT8(); + case LIST_16: + return unpackUINT16(); + case LIST_32: + return unpackUINT32(); + default: + throw new Unexpected( "Expected a list, but got: " + toHexString( markerByte ) ); } } @@ -531,42 +559,51 @@ public long unpackMapHeader() throws IOException final byte markerLowNibble = (byte) (markerByte & 0x0F); if ( markerHighNibble == TINY_MAP ) { return markerLowNibble; } - switch(markerByte) + switch ( markerByte ) { - case MAP_8: return unpackUINT8(); - case MAP_16: return unpackUINT16(); - case MAP_32: return unpackUINT32(); - default: throw new Unexpected( "Expected a map, but got: " + toHexString( markerByte )); + case MAP_8: + return unpackUINT8(); + case MAP_16: + return unpackUINT16(); + case MAP_32: + return unpackUINT32(); + default: + throw new Unexpected( "Expected a map, but got: " + toHexString( markerByte ) ); } } public long unpackLong() throws IOException { final byte markerByte = in.ensure( 1 ).get(); - if ( markerByte >= MINUS_2_TO_THE_4) { return markerByte; } - switch(markerByte) - { - case INT_8: return in.ensure( 1 ).get(); - case INT_16: return in.ensure( 2 ).getShort(); - case INT_32: return in.ensure( 4 ).getInt(); - case INT_64: return in.ensure( 8 ).getLong(); - default: throw new Unexpected( "Expected an integer, but got: " + toHexString( markerByte )); + if ( markerByte >= MINUS_2_TO_THE_4 ) { return markerByte; } + switch ( markerByte ) + { + case INT_8: + return in.ensure( 1 ).get(); + case INT_16: + return in.ensure( 2 ).getShort(); + case INT_32: + return in.ensure( 4 ).getInt(); + case INT_64: + return in.ensure( 8 ).getLong(); + default: + throw new Unexpected( "Expected an integer, but got: " + toHexString( markerByte ) ); } } public double unpackDouble() throws IOException { final byte markerByte = in.ensure( 1 ).get(); - if(markerByte == FLOAT_64) + if ( markerByte == FLOAT_64 ) { return in.ensure( 8 ).getDouble(); } - throw new Unexpected( "Expected a double, but got: " + toHexString( markerByte )); + throw new Unexpected( "Expected a double, but got: " + toHexString( markerByte ) ); } public String unpackString() throws IOException { - return new String(unpackUtf8(), UTF_8); + return new String( unpackUtf8(), UTF_8 ); } public byte[] unpackUtf8() throws IOException @@ -576,10 +613,12 @@ public byte[] unpackUtf8() throws IOException final byte markerLowNibble = (byte) (markerByte & 0x0F); if ( markerHighNibble == TINY_TEXT ) { return unpackBytes( markerLowNibble ); } - switch(markerByte) + switch ( markerByte ) { - case TEXT_8: return unpackBytes( unpackUINT8() ); - case TEXT_16: return unpackBytes( unpackUINT16() ); + case TEXT_8: + return unpackBytes( unpackUINT8() ); + case TEXT_16: + return unpackBytes( unpackUINT16() ); case TEXT_32: { long size = unpackUINT32(); @@ -592,18 +631,22 @@ public byte[] unpackUtf8() throws IOException throw new Overflow( "TEXT_32 too long for Java" ); } } - default: throw new Unexpected( "Expected a string, but got: " + toHexString( markerByte & 0xFF )); + default: + throw new Unexpected( "Expected a string, but got: " + toHexString( markerByte & 0xFF ) ); } } public boolean unpackBoolean() throws IOException { final byte markerByte = in.ensure( 1 ).get(); - switch(markerByte) + switch ( markerByte ) { - case TRUE: return true; - case FALSE: return false; - default: throw new Unexpected( "Expected a boolean, but got: " + toHexString( markerByte & 0xFF )); + case TRUE: + return true; + case FALSE: + return false; + default: + throw new Unexpected( "Expected a boolean, but got: " + toHexString( markerByte & 0xFF ) ); } } @@ -774,22 +817,25 @@ private long unpackUINT32() throws IOException private byte[] unpackBytes( int size ) throws IOException { - if ( size == 0 ) return EMPTY_BYTE_ARRAY; + if ( size == 0 ) + { + return EMPTY_BYTE_ARRAY; + } byte[] heapBuffer = new byte[size]; int index = 0; - while(index < size) + while ( index < size ) { int toRead = Math.min( in.remaining(), size - index ); in.get( heapBuffer, index, toRead ); index += toRead; - if(in.remaining() == 0 && index < size) + if ( in.remaining() == 0 && index < size ) { in.attemptUpTo( size - index ); - if(in.remaining() == 0) + if ( in.remaining() == 0 ) { throw new EndOfStream( "Expected " + (size - index) + " bytes available, " + - "but no more bytes accessible from underlying stream." ); + "but no more bytes accessible from underlying stream." ); } } } @@ -798,7 +844,10 @@ private byte[] unpackBytes( int size ) throws IOException private List unpackList( int size ) throws IOException { - if ( size == 0 ) return EMPTY_LIST_OF_VALUES; + if ( size == 0 ) + { + return EMPTY_LIST_OF_VALUES; + } List list = new ArrayList<>( size ); for ( int i = 0; i < size; i++ ) { @@ -809,7 +858,10 @@ private List unpackList( int size ) throws IOException private Map unpackMap( int size ) throws IOException { - if ( size == 0 ) return EMPTY_MAP_OF_VALUES; + if ( size == 0 ) + { + return EMPTY_MAP_OF_VALUES; + } Map map = new LinkedHashMap<>( size ); for ( int i = 0; i < size; i++ ) { @@ -828,15 +880,19 @@ public PackType peekNextType() throws IOException final byte markerByte = in.ensure( 1 ).peek(); final byte markerHighNibble = (byte) (markerByte & 0xF0); - switch(markerHighNibble) + switch ( markerHighNibble ) { - case TINY_TEXT: return PackType.TEXT; - case TINY_LIST: return PackType.LIST; - case TINY_MAP: return PackType.MAP; - case TINY_STRUCT: return PackType.STRUCT; + case TINY_TEXT: + return PackType.TEXT; + case TINY_LIST: + return PackType.LIST; + case TINY_MAP: + return PackType.MAP; + case TINY_STRUCT: + return PackType.STRUCT; } - switch(markerByte) + switch ( markerByte ) { case NULL: return PackType.NULL; diff --git a/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/PackType.java b/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/PackType.java index ababeae278205..db92cf284534c 100644 --- a/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/PackType.java +++ b/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/PackType.java @@ -21,5 +21,13 @@ public enum PackType { - NULL, BOOLEAN, INTEGER, FLOAT, BYTES, TEXT, LIST, MAP, STRUCT + NULL, + BOOLEAN, + INTEGER, + FLOAT, + BYTES, + TEXT, + LIST, + MAP, + STRUCT } diff --git a/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/PackValue.java b/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/PackValue.java index d811ef3ea5720..d4aef8b870a9d 100644 --- a/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/PackValue.java +++ b/community/ndp/packstream-v1/src/main/java/org/neo4j/packstream/PackValue.java @@ -30,25 +30,52 @@ public abstract class PackValue implements Iterable { public static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; public static final List EMPTY_LIST_OF_VALUES = Collections.EMPTY_LIST; - public static final Map EMPTY_MAP_OF_VALUES = Collections.EMPTY_MAP; - - public boolean isNull() { return false; } + public static final Map EMPTY_MAP_OF_VALUES = Collections.EMPTY_MAP; - public boolean isBoolean() { return false; } + public boolean isNull() + { + return false; + } - public boolean isInteger() { return false; } + public boolean isBoolean() + { + return false; + } + + public boolean isInteger() + { + return false; + } - public boolean isFloat() { return false; } + public boolean isFloat() + { + return false; + } - public boolean isBytes() { return false; } + public boolean isBytes() + { + return false; + } - public boolean isText() { return false; } + public boolean isText() + { + return false; + } - public boolean isList() { return false; } + public boolean isList() + { + return false; + } - public boolean isMap() { return false; } + public boolean isMap() + { + return false; + } - public boolean isStruct() { return false; } + public boolean isStruct() + { + return false; + } public abstract boolean booleanValue(); @@ -64,11 +91,13 @@ public abstract class PackValue implements Iterable public abstract String stringValue(); - public int size() { + public int size() + { return 1; } - public char signature() { + public char signature() + { return '\0'; } @@ -109,28 +138,52 @@ public static class NullValue extends PackValue public static final NullValue NULL_VALUE = new NullValue(); @Override - public boolean isNull() { return true; } + public boolean isNull() + { + return true; + } @Override - public boolean booleanValue() { return false; } + public boolean booleanValue() + { + return false; + } @Override - public int intValue() { return 0; } + public int intValue() + { + return 0; + } @Override - public long longValue() { return 0; } + public long longValue() + { + return 0; + } @Override - public float floatValue() { return 0; } + public float floatValue() + { + return 0; + } @Override - public double doubleValue() { return 0; } + public double doubleValue() + { + return 0; + } @Override - public byte[] byteArrayValue() { return new byte[0]; } + public byte[] byteArrayValue() + { + return new byte[0]; + } @Override - public String stringValue() { return null; } + public String stringValue() + { + return null; + } @Override public int size() @@ -148,31 +201,58 @@ public static class BooleanValue extends PackValue private final boolean value; - private BooleanValue( boolean value ) { this.value = value; } + private BooleanValue( boolean value ) + { + this.value = value; + } @Override - public boolean isBoolean() { return true; } + public boolean isBoolean() + { + return true; + } @Override - public boolean booleanValue() { return value; } + public boolean booleanValue() + { + return value; + } @Override - public int intValue() { return value ? 1 : 0; } + public int intValue() + { + return value ? 1 : 0; + } @Override - public long longValue() { return value ? 1 : 0; } + public long longValue() + { + return value ? 1 : 0; + } @Override - public float floatValue() { return value ? 1 : 0; } + public float floatValue() + { + return value ? 1 : 0; + } @Override - public double doubleValue() { return value ? 1 : 0; } + public double doubleValue() + { + return value ? 1 : 0; + } @Override - public byte[] byteArrayValue() { return null; } + public byte[] byteArrayValue() + { + return null; + } @Override - public String stringValue() { return value ? "true" : "false"; } + public String stringValue() + { + return value ? "true" : "false"; + } } @@ -181,6 +261,7 @@ public static class IntegerValue extends PackValue // Integer cache private static final IntegerValue[] positiveValues = new IntegerValue[256]; private static final IntegerValue[] negativeValues = new IntegerValue[256]; + static { // Keep hold of small values so instances can be reused @@ -217,34 +298,61 @@ else if ( value >= 0 ) } } } - + private final long value; - private IntegerValue( long value ) { this.value = value; } + private IntegerValue( long value ) + { + this.value = value; + } @Override - public boolean isInteger() { return true; } + public boolean isInteger() + { + return true; + } @Override - public boolean booleanValue() { return value != 0; } + public boolean booleanValue() + { + return value != 0; + } @Override - public int intValue() { return (int) value; } + public int intValue() + { + return (int) value; + } @Override - public long longValue() { return value; } + public long longValue() + { + return value; + } @Override - public float floatValue() { return (float) value; } + public float floatValue() + { + return (float) value; + } @Override - public double doubleValue() { return (double) value; } + public double doubleValue() + { + return (double) value; + } @Override - public byte[] byteArrayValue() { return new byte[0]; } + public byte[] byteArrayValue() + { + return new byte[0]; + } @Override - public String stringValue() { return Long.toString( value ); } + public String stringValue() + { + return Long.toString( value ); + } } @@ -253,38 +361,65 @@ public static class FloatValue extends PackValue private final double value; - public FloatValue( double value ) { this.value = value; } + public FloatValue( double value ) + { + this.value = value; + } @Override - public boolean isFloat() { return true; } + public boolean isFloat() + { + return true; + } @Override - public boolean booleanValue() { return value != 0; } + public boolean booleanValue() + { + return value != 0; + } @Override - public int intValue() { return (int) value; } + public int intValue() + { + return (int) value; + } @Override - public long longValue() { return (long) value; } + public long longValue() + { + return (long) value; + } @Override - public float floatValue() { return (float) value; } + public float floatValue() + { + return (float) value; + } @Override - public double doubleValue() { return value; } + public double doubleValue() + { + return value; + } @Override - public byte[] byteArrayValue() { return new byte[0]; } + public byte[] byteArrayValue() + { + return new byte[0]; + } @Override - public String stringValue() { return Double.toString( value ); } + public String stringValue() + { + return Double.toString( value ); + } } public static class BytesValue extends PackValue { private static final BytesValue EMPTY_BYTES = new BytesValue( EMPTY_BYTE_ARRAY ); - + public static BytesValue getInstance( byte[] value ) { if ( value.length == 0 ) @@ -296,34 +431,61 @@ public static BytesValue getInstance( byte[] value ) return new BytesValue( value ); } } - + private final byte[] value; - private BytesValue( byte[] value ) { this.value = value; } + private BytesValue( byte[] value ) + { + this.value = value; + } @Override - public boolean isBytes() { return true; } + public boolean isBytes() + { + return true; + } @Override - public boolean booleanValue() { return value.length > 0; } + public boolean booleanValue() + { + return value.length > 0; + } @Override - public int intValue() { throw cantConvertTo( "int" ); } + public int intValue() + { + throw cantConvertTo( "int" ); + } @Override - public long longValue() { throw cantConvertTo( "long" ); } + public long longValue() + { + throw cantConvertTo( "long" ); + } @Override - public float floatValue() { throw cantConvertTo( "float" ); } + public float floatValue() + { + throw cantConvertTo( "float" ); + } @Override - public double doubleValue() { throw cantConvertTo( "double" ); } + public double doubleValue() + { + throw cantConvertTo( "double" ); + } @Override - public byte[] byteArrayValue() { return value; } + public byte[] byteArrayValue() + { + return value; + } @Override - public String stringValue() { return new String( value ); } + public String stringValue() + { + return new String( value ); + } @Override public PackValue get( int index ) @@ -347,7 +509,7 @@ public int size() private UnsupportedOperationException cantConvertTo( String toTarget ) { - return new UnsupportedOperationException( "Bytes value cannot be converted to "+toTarget+"." ); + return new UnsupportedOperationException( "Bytes value cannot be converted to " + toTarget + "." ); } } @@ -370,35 +532,62 @@ public static TextValue getInstance( byte[] value ) private final byte[] utf8; private String decoded; - private TextValue( char value ) { this( Character.toString( value ).getBytes( UTF_8 ) ); } + private TextValue( char value ) + { + this( Character.toString( value ).getBytes( UTF_8 ) ); + } - private TextValue( byte[] utf8 ) { this.utf8 = utf8; } + private TextValue( byte[] utf8 ) + { + this.utf8 = utf8; + } @Override - public boolean isText() { return true; } + public boolean isText() + { + return true; + } @Override - public boolean booleanValue() { return stringValue().length() > 0; } + public boolean booleanValue() + { + return stringValue().length() > 0; + } @Override - public int intValue() { return Integer.parseInt( stringValue() ); } + public int intValue() + { + return Integer.parseInt( stringValue() ); + } @Override - public long longValue() { return Long.parseLong( stringValue() ); } + public long longValue() + { + return Long.parseLong( stringValue() ); + } @Override - public float floatValue() { return Float.parseFloat( stringValue() ); } + public float floatValue() + { + return Float.parseFloat( stringValue() ); + } @Override - public double doubleValue() { return Double.parseDouble( stringValue() ); } + public double doubleValue() + { + return Double.parseDouble( stringValue() ); + } @Override - public byte[] byteArrayValue() { return utf8; } + public byte[] byteArrayValue() + { + return utf8; + } @Override public String stringValue() { - if(decoded == null) + if ( decoded == null ) { decoded = new String( utf8, UTF_8 ); } @@ -447,37 +636,70 @@ public static ListValue getInstance( List values ) private final List values; - private ListValue( List values ) { this.values = values; } + private ListValue( List values ) + { + this.values = values; + } @Override - public boolean isList() { return true; } + public boolean isList() + { + return true; + } @Override - public boolean booleanValue() { return !values.isEmpty(); } + public boolean booleanValue() + { + return !values.isEmpty(); + } @Override - public int intValue() { return 0; } + public int intValue() + { + return 0; + } @Override - public long longValue() { return 0; } + public long longValue() + { + return 0; + } @Override - public float floatValue() { return 0; } + public float floatValue() + { + return 0; + } @Override - public double doubleValue() { return 0; } + public double doubleValue() + { + return 0; + } @Override - public byte[] byteArrayValue() { return null; } + public byte[] byteArrayValue() + { + return null; + } @Override - public String stringValue() { return null; } + public String stringValue() + { + return null; + } @Override - public List listValue() { return values; } + public List listValue() + { + return values; + } @Override - public Iterator iterator() { return values.iterator(); } + public Iterator iterator() + { + return values.iterator(); + } @Override public PackValue get( int index ) @@ -506,7 +728,7 @@ public static class MapValue extends PackValue private static final MapValue EMPTY_MAP = new MapValue( EMPTY_MAP_OF_VALUES ); - public static MapValue getInstance( Map values ) + public static MapValue getInstance( Map values ) { if ( values.isEmpty() ) { @@ -520,34 +742,64 @@ public static MapValue getInstance( Map values ) private final Map values; - private MapValue( Map values ) { this.values = values; } + private MapValue( Map values ) + { + this.values = values; + } @Override - public boolean isMap() { return true; } + public boolean isMap() + { + return true; + } @Override - public boolean booleanValue() { return !values.isEmpty(); } + public boolean booleanValue() + { + return !values.isEmpty(); + } @Override - public int intValue() { return 0; } + public int intValue() + { + return 0; + } @Override - public long longValue() { return 0; } + public long longValue() + { + return 0; + } @Override - public float floatValue() { return 0; } + public float floatValue() + { + return 0; + } @Override - public double doubleValue() { return 0; } + public double doubleValue() + { + return 0; + } @Override - public byte[] byteArrayValue() { return null; } + public byte[] byteArrayValue() + { + return null; + } @Override - public String stringValue() { return null; } + public String stringValue() + { + return null; + } @Override - public Map mapValue() { return values; } + public Map mapValue() + { + return values; + } @Override public PackValue get( String key ) @@ -588,34 +840,64 @@ private StructValue( char signature, List values ) } @Override - public boolean isStruct() { return true; } + public boolean isStruct() + { + return true; + } @Override - public boolean booleanValue() { return !values.isEmpty(); } + public boolean booleanValue() + { + return !values.isEmpty(); + } @Override - public int intValue() { return 0; } + public int intValue() + { + return 0; + } @Override - public long longValue() { return 0; } + public long longValue() + { + return 0; + } @Override - public float floatValue() { return 0; } + public float floatValue() + { + return 0; + } @Override - public double doubleValue() { return 0; } + public double doubleValue() + { + return 0; + } @Override - public byte[] byteArrayValue() { return null; } + public byte[] byteArrayValue() + { + return null; + } @Override - public String stringValue() { return null; } + public String stringValue() + { + return null; + } @Override - public List listValue() { return values; } + public List listValue() + { + return values; + } @Override - public Iterator iterator() { return values.iterator(); } + public Iterator iterator() + { + return values.iterator(); + } @Override public PackValue get( int index ) diff --git a/community/ndp/packstream-v1/src/test/java/org/neo4j/packstream/PackStreamTest.java b/community/ndp/packstream-v1/src/test/java/org/neo4j/packstream/PackStreamTest.java index 0e4089b7b86ff..64ee6d9ed2903 100644 --- a/community/ndp/packstream-v1/src/test/java/org/neo4j/packstream/PackStreamTest.java +++ b/community/ndp/packstream-v1/src/test/java/org/neo4j/packstream/PackStreamTest.java @@ -42,9 +42,9 @@ public class PackStreamTest { - public static Map asMap( Object... keysAndValues ) + public static Map asMap( Object... keysAndValues ) { - Map map = new LinkedHashMap<>( keysAndValues.length / 2 ); + Map map = new LinkedHashMap<>( keysAndValues.length / 2 ); String key = null; for ( Object keyOrValue : keysAndValues ) { @@ -79,7 +79,7 @@ public Machine( int bufferSize ) { this.output = new ByteArrayOutputStream(); this.writable = Channels.newChannel( this.output ); - this.packer = new PackStream.Packer( new BufferedChannelOutput( this.writable, bufferSize) ); + this.packer = new PackStream.Packer( new BufferedChannelOutput( this.writable, bufferSize ) ); } public void reset() @@ -651,7 +651,7 @@ public void testCanDoStreamingListUnpacking() throws Throwable // Given Machine machine = new Machine(); PackStream.Packer packer = machine.packer(); - packer.pack( asList(1,2,3,asList(4,5)) ); + packer.pack( asList( 1, 2, 3, asList( 4, 5 ) ) ); packer.flush(); // When I unpack this value @@ -687,7 +687,7 @@ public void testCanDoStreamingStructUnpacking() throws Throwable packer.pack( 1 ); packer.pack( 2 ); packer.pack( 3 ); - packer.pack( asList( 4,5 ) ); + packer.pack( asList( 4, 5 ) ); packer.flush(); // When I unpack this value @@ -805,8 +805,8 @@ public void handlesDataCrossingBufferBoundaries() throws Throwable long second = unpacker.unpackLong(); // Then - assertEquals(Long.MAX_VALUE, first); - assertEquals(Long.MAX_VALUE, second); + assertEquals( Long.MAX_VALUE, first ); + assertEquals( Long.MAX_VALUE, second ); } @Test @@ -817,8 +817,8 @@ public void testCanPeekOnNextType() throws Throwable assertPeekType( PackType.INTEGER, 123 ); assertPeekType( PackType.FLOAT, 123.123 ); assertPeekType( PackType.BOOLEAN, true ); - assertPeekType( PackType.LIST, asList( 1,2,3 ) ); - assertPeekType( PackType.MAP, asMap( "l",3 ) ); + assertPeekType( PackType.LIST, asList( 1, 2, 3 ) ); + assertPeekType( PackType.MAP, asMap( "l", 3 ) ); } void assertPeekType( PackType type, Object value ) throws IOException diff --git a/community/ndp/pom.xml b/community/ndp/pom.xml index 7809d8b2aa675..98070a32feff0 100644 --- a/community/ndp/pom.xml +++ b/community/ndp/pom.xml @@ -21,23 +21,23 @@ pom Neo4j - Data Protocol Project - + GNU General Public License, Version 3 http://www.gnu.org/licenses/gpl-3.0-standalone.html The software ("Software") developed and owned by Network Engine for -Objects in Lund AB (referred to in this notice as "Neo Technology") is -licensed under the GNU GENERAL PUBLIC LICENSE Version 3 to all third -parties and that license is included below. + Objects in Lund AB (referred to in this notice as "Neo Technology") is + licensed under the GNU GENERAL PUBLIC LICENSE Version 3 to all third + parties and that license is included below. -However, if you have executed an End User Software License and Services -Agreement or an OEM Software License and Support Services Agreement, or -another commercial license agreement with Neo Technology or one of its -affiliates (each, a "Commercial Agreement"), the terms of the license in -such Commercial Agreement will supersede the GNU GENERAL PUBLIC LICENSE -Version 3 and you may use the Software solely pursuant to the terms of -the relevant Commercial Agreement. + However, if you have executed an End User Software License and Services + Agreement or an OEM Software License and Support Services Agreement, or + another commercial license agreement with Neo Technology or one of its + affiliates (each, a "Commercial Agreement"), the terms of the license in + such Commercial Agreement will supersede the GNU GENERAL PUBLIC LICENSE + Version 3 and you may use the Software solely pursuant to the terms of + the relevant Commercial Agreement. @@ -49,5 +49,6 @@ the relevant Commercial Agreement. messaging-v1 kernelextension transport-socket + v1-docs diff --git a/community/ndp/runtime/pom.xml b/community/ndp/runtime/pom.xml index 4b1154ed32335..ecfddb3ccfc5b 100644 --- a/community/ndp/runtime/pom.xml +++ b/community/ndp/runtime/pom.xml @@ -29,17 +29,17 @@ GNU General Public License, Version 3 http://www.gnu.org/licenses/gpl-3.0-standalone.html The software ("Software") developed and owned by Network Engine for -Objects in Lund AB (referred to in this notice as "Neo Technology") is -licensed under the GNU GENERAL PUBLIC LICENSE Version 3 to all third -parties and that license is included below. + Objects in Lund AB (referred to in this notice as "Neo Technology") is + licensed under the GNU GENERAL PUBLIC LICENSE Version 3 to all third + parties and that license is included below. -However, if you have executed an End User Software License and Services -Agreement or an OEM Software License and Support Services Agreement, or -another commercial license agreement with Neo Technology or one of its -affiliates (each, a "Commercial Agreement"), the terms of the license in -such Commercial Agreement will supersede the GNU GENERAL PUBLIC LICENSE -Version 3 and you may use the Software solely pursuant to the terms of -the relevant Commercial Agreement. + However, if you have executed an End User Software License and Services + Agreement or an OEM Software License and Support Services Agreement, or + another commercial license agreement with Neo Technology or one of its + affiliates (each, a "Commercial Agreement"), the terms of the license in + such Commercial Agreement will supersede the GNU GENERAL PUBLIC LICENSE + Version 3 and you may use the Software solely pursuant to the terms of + the relevant Commercial Agreement. @@ -109,8 +109,8 @@ the relevant Commercial Agreement. maven-antrun-plugin - - + + generate-version none diff --git a/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/Session.java b/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/Session.java index 67b52814273b2..308c1e4d624bd 100644 --- a/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/Session.java +++ b/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/Session.java @@ -28,11 +28,12 @@ * A user session associated with a given {@link Sessions}. The majority of methods on this * interface are asynchronous, requesting that you provide a {@link Session.Callback} to be used to * publish the result. - * - * This arrangement reflects the asynchronous nature of sessions and enables several features. This includes out-of-band + *

+ * This arrangement reflects the asynchronous nature of sessions and enables several features. This includes + * out-of-band * messages such as warnings, forwarding statements over the network, and ignoring messages until errors have been * acknowledged by the client. - * + *

* While the operations are asynchronous, they are guaranteed to be executed in calling order. This allows you to call * several operations in sequence without waiting for the previous operation to complete. */ @@ -42,12 +43,15 @@ public interface Session extends AutoCloseable * Callback for handling the result of requests. For a given session, callbacks will be invoked serially, * in the order they were given. This means you may pass the same callback multiple times without waiting for a * reply, and are guaranteed that your callbacks will be called in order. + * * @param * @param */ - interface Callback + interface Callback { - public static final Callback NO_OP = new Adapter(){ }; + public static final Callback NO_OP = new Adapter() + { + }; /** Called zero or more times with results, if the operation invoked yields results. */ void result( V result, A attachment ) throws Exception; @@ -61,7 +65,7 @@ interface Callback /** Called when the state machine ignores an operation, because it is waiting for an error to be acknowledged */ void ignored( A attachment ); - public static abstract class Adapter implements Callback + public static abstract class Adapter implements Callback { @Override public void result( V result, A attachment ) throws Exception @@ -94,12 +98,12 @@ public void ignored( A attachment ) /** * Run a statement, yielding a result stream which can be retrieved through pulling it in a subsequent call. - * + *

* If there is a statement running already, all remaining items in its stream must be {@link #pullAll(Object, * Session.Callback) pulled} or {@link #discardAll(Object, Session.Callback) * discarded}. */ - void run( String statement, Map params, A attachment, Callback callback ); + void run( String statement, Map params, A attachment, Callback callback ); /** * Retrieve all remaining entries in the current result. This is a distinct operation from 'run' in order to @@ -119,8 +123,9 @@ public void ignored( A attachment ) * this method. The point of this is that we can do pipelining, sending multiple requests in one go and * optimistically assuming they will succeed. If any of them fail all subsequent requests are declined until the * client has acknowledged it has seen the error and has taken it into account for upcoming requests. - * - * Whenever an error has been acknowledged, the session will revert back to its intial state. Any ongoing statements + *

+ * Whenever an error has been acknowledged, the session will revert back to its intial state. Any ongoing + * statements * or transactions will have been rolled back and/or disposed of. */ void acknowledgeFailure( A attachment, Callback callback ); diff --git a/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/StatementMetadata.java b/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/StatementMetadata.java index 7e5e00cfba4c1..54e4564ad0a4c 100644 --- a/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/StatementMetadata.java +++ b/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/StatementMetadata.java @@ -20,7 +20,8 @@ package org.neo4j.ndp.runtime; /** - * Metadata that becomes available as soon as a statement is started, and is sent to the client before the result stream + * Metadata that becomes available as soon as a statement is started, and is sent to the client before the result + * stream * is sent. */ public interface StatementMetadata diff --git a/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/CypherAdapterStream.java b/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/CypherAdapterStream.java index 59436405596c6..e6304e83ac613 100644 --- a/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/CypherAdapterStream.java +++ b/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/CypherAdapterStream.java @@ -35,7 +35,7 @@ public CypherAdapterStream( Result delegate ) { this.delegate = delegate; this.fieldNames = delegate.columns().toArray( new String[delegate.columns().size()] ); - this.currentRecord = new CypherAdapterRecord(fieldNames); + this.currentRecord = new CypherAdapterRecord( fieldNames ); } @Override @@ -53,7 +53,7 @@ public String[] fieldNames() @Override public void visitAll( Visitor visitor ) throws Exception { - while(delegate.hasNext()) + while ( delegate.hasNext() ) { visitor.visit( currentRecord.reset( delegate.next() ) ); } @@ -96,7 +96,7 @@ public Record copy() }; } - public CypherAdapterRecord reset( Map cypherRecord ) + public CypherAdapterRecord reset( Map cypherRecord ) { for ( int i = 0; i < fields.length; i++ ) { diff --git a/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/CypherStatementRunner.java b/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/CypherStatementRunner.java index a9139fce70f2d..b906b2d2bd93f 100644 --- a/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/CypherStatementRunner.java +++ b/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/CypherStatementRunner.java @@ -37,20 +37,20 @@ public CypherStatementRunner( GraphDatabaseService db ) @Override public RecordStream run( final SessionState ctx, final String statement, - final Map params ) throws KernelException + final Map params ) throws KernelException { // Temporary until we move parsing to cypher, or run a parser up here - if(statement.equalsIgnoreCase( "begin" )) + if ( statement.equalsIgnoreCase( "begin" ) ) { ctx.beginTransaction(); return RecordStream.EMPTY; } - else if(statement.equalsIgnoreCase( "commit" )) + else if ( statement.equalsIgnoreCase( "commit" ) ) { ctx.commitTransaction(); return RecordStream.EMPTY; } - else if(statement.equalsIgnoreCase( "rollback" )) + else if ( statement.equalsIgnoreCase( "rollback" ) ) { ctx.rollbackTransaction(); return RecordStream.EMPTY; diff --git a/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/ErrorTranslator.java b/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/ErrorTranslator.java index 8b8766c68564f..b10e9a43cb084 100644 --- a/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/ErrorTranslator.java +++ b/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/ErrorTranslator.java @@ -33,24 +33,24 @@ public class ErrorTranslator private final Log log; - public ErrorTranslator(Log log) + public ErrorTranslator( Log log ) { this.log = log; } public Neo4jError translate( Throwable any ) { - if(any instanceof KernelException) + if ( any instanceof KernelException ) { - return new Neo4jError( ((KernelException)any).status(), any.getMessage() ); + return new Neo4jError( ((KernelException) any).status(), any.getMessage() ); } - else if( any instanceof CypherException ) + else if ( any instanceof CypherException ) { - return new Neo4jError(((CypherException) any).status(), any.getMessage()); + return new Neo4jError( ((CypherException) any).status(), any.getMessage() ); } else { - if( any.getCause() != null ) + if ( any.getCause() != null ) { return translate( any.getCause() ); } @@ -58,8 +58,8 @@ else if( any instanceof CypherException ) // Log unknown errors. log.warn( "Client triggered unknown error: " + any.getMessage(), any ); - return new Neo4jError( Status.General.UnknownFailure, "An unexpected failure occurred: '" - + any.getMessage() + "'." ); + return new Neo4jError( Status.General.UnknownFailure, "An unexpected failure occurred: '" + + any.getMessage() + "'." ); } } diff --git a/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/Neo4jError.java b/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/Neo4jError.java index 0e323fcc0a140..c470e56bfbc8f 100644 --- a/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/Neo4jError.java +++ b/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/Neo4jError.java @@ -30,7 +30,7 @@ public class Neo4jError private final Status status; private final String message; - public Neo4jError(Status status, String message) + public Neo4jError( Status status, String message ) { this.status = status; this.message = message; @@ -84,7 +84,7 @@ public String toString() public static Status codeFromString( String codeStr ) { String[] parts = codeStr.split( "\\." ); - if(parts.length != 4) + if ( parts.length != 4 ) { return Status.General.UnknownFailure; } @@ -93,22 +93,22 @@ public static Status codeFromString( String codeStr ) String error = parts[3]; // Note: the input string may contain arbitrary input data, using reflection would open network attack vector - switch(category) + switch ( category ) { case "Schema": - return Status.Schema.valueOf(error); + return Status.Schema.valueOf( error ); case "LegacyIndex": - return Status.LegacyIndex.valueOf(error); + return Status.LegacyIndex.valueOf( error ); case "General": - return Status.General.valueOf(error); + return Status.General.valueOf( error ); case "Statement": - return Status.Statement.valueOf(error); + return Status.Statement.valueOf( error ); case "Transaction": - return Status.Transaction.valueOf(error); + return Status.Transaction.valueOf( error ); case "Request": - return Status.Request.valueOf(error); + return Status.Request.valueOf( error ); case "Network": - return Status.Network.valueOf(error); + return Status.Network.valueOf( error ); default: return Status.General.UnknownFailure; } diff --git a/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/StatementRunner.java b/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/StatementRunner.java index 5126599adeb6b..7f7a438c7518d 100644 --- a/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/StatementRunner.java +++ b/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/StatementRunner.java @@ -22,8 +22,8 @@ import java.util.Map; import org.neo4j.kernel.api.exceptions.KernelException; -import org.neo4j.stream.RecordStream; import org.neo4j.ndp.runtime.internal.session.SessionState; +import org.neo4j.stream.RecordStream; /** * A runtime handler can handle a textual input language, yielding results. Query engines are not expected to be @@ -31,5 +31,5 @@ */ public interface StatementRunner { - RecordStream run( SessionState ctx, String statement, Map params ) throws KernelException; + RecordStream run( SessionState ctx, String statement, Map params ) throws KernelException; } diff --git a/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/session/SessionStateMachine.java b/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/session/SessionStateMachine.java index ed799a33043ee..9e5b8c0922f96 100644 --- a/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/session/SessionStateMachine.java +++ b/community/ndp/runtime/src/main/java/org/neo4j/ndp/runtime/internal/session/SessionStateMachine.java @@ -53,175 +53,174 @@ enum State * No open transaction, no open result. */ IDLE - { - @Override - public State beginTransaction( SessionStateMachine ctx ) - { - assert ctx.currentTransaction == null; - ctx.implicitTransaction = false; - ctx.currentTransaction = ctx.db.beginTx(); - return IN_TRANSACTION; - } - - @Override - public State runStatement( SessionStateMachine ctx, String statement, Map params ) - { - try { - ctx.currentResult = ctx.statementRunner.run( ctx, statement, params ); - ctx.result( ctx.currentStatementMetadata ); - return STREAM_OPEN; - } - catch( Throwable e ) - { - return error(ctx, e); - } - } + @Override + public State beginTransaction( SessionStateMachine ctx ) + { + assert ctx.currentTransaction == null; + ctx.implicitTransaction = false; + ctx.currentTransaction = ctx.db.beginTx(); + return IN_TRANSACTION; + } - @Override - public State beginImplicitTransaction( SessionStateMachine ctx ) - { - assert ctx.currentTransaction == null; - ctx.implicitTransaction = true; - ctx.currentTransaction = ctx.db.beginTx(); - return IN_TRANSACTION; - } + @Override + public State runStatement( SessionStateMachine ctx, String statement, Map params ) + { + try + { + ctx.currentResult = ctx.statementRunner.run( ctx, statement, params ); + ctx.result( ctx.currentStatementMetadata ); + return STREAM_OPEN; + } + catch ( Throwable e ) + { + return error( ctx, e ); + } + } + + @Override + public State beginImplicitTransaction( SessionStateMachine ctx ) + { + assert ctx.currentTransaction == null; + ctx.implicitTransaction = true; + ctx.currentTransaction = ctx.db.beginTx(); + return IN_TRANSACTION; + } - }, + }, /** * Open transaction, no open stream - * + *

* This is when the client has explicitly requested a transaction to be opened. */ IN_TRANSACTION - { - @Override - public State runStatement( SessionStateMachine ctx, String statement, Map params ) - { - return IDLE.runStatement( ctx, statement, params); - } - - @Override - public State commitTransaction( SessionStateMachine ctx ) - { - try { - ctx.currentTransaction.success(); - ctx.currentTransaction.close(); - } - catch(Throwable e) - { - return error(ctx, e); - } - finally - { - ctx.currentTransaction = null; - } - return IDLE; - } + @Override + public State runStatement( SessionStateMachine ctx, String statement, Map params ) + { + return IDLE.runStatement( ctx, statement, params ); + } - @Override - public State rollbackTransaction( SessionStateMachine ctx ) - { - try - { - Transaction tx = ctx.currentTransaction; - ctx.currentTransaction = null; + @Override + public State commitTransaction( SessionStateMachine ctx ) + { + try + { + ctx.currentTransaction.success(); + ctx.currentTransaction.close(); + } + catch ( Throwable e ) + { + return error( ctx, e ); + } + finally + { + ctx.currentTransaction = null; + } + return IDLE; + } - tx.failure(); - tx.close(); - return IDLE; - } - catch(Throwable e) - { - return error(ctx, e); - } - } - }, + @Override + public State rollbackTransaction( SessionStateMachine ctx ) + { + try + { + Transaction tx = ctx.currentTransaction; + ctx.currentTransaction = null; + + tx.failure(); + tx.close(); + return IDLE; + } + catch ( Throwable e ) + { + return error( ctx, e ); + } + } + }, /** * A result stream is ready for consumption, there may or may not be an open transaction. */ STREAM_OPEN - { - @Override - public State pullAll( SessionStateMachine ctx ) - { - try { - ctx.result( ctx.currentResult ); - return discardAll( ctx ); - } - catch(Throwable e) - { - return error(ctx, e); - } - } - - @Override - public State discardAll( SessionStateMachine ctx ) - { - try - { - ctx.currentResult.close(); - - if( !ctx.hasTransaction() ) + @Override + public State pullAll( SessionStateMachine ctx ) { - return IDLE; - } - else if ( ctx.implicitTransaction ) - { - return IN_TRANSACTION.commitTransaction( ctx ); + try + { + ctx.result( ctx.currentResult ); + return discardAll( ctx ); + } + catch ( Throwable e ) + { + return error( ctx, e ); + } } - else + + @Override + public State discardAll( SessionStateMachine ctx ) { - return IN_TRANSACTION; + try + { + ctx.currentResult.close(); + + if ( !ctx.hasTransaction() ) + { + return IDLE; + } + else if ( ctx.implicitTransaction ) + { + return IN_TRANSACTION.commitTransaction( ctx ); + } + else + { + return IN_TRANSACTION; + } + } + catch ( Throwable e ) + { + return error( ctx, e ); + } + finally + { + ctx.currentResult = null; + } } - } - catch(Throwable e) - { - return error(ctx, e); - } - finally - { - ctx.currentResult = null; - } - } - }, + }, /** An error has occurred, client must acknowledge it before anything else is allowed. */ ERROR - { - @Override - public State acknowledgeError( SessionStateMachine ctx ) - { - return IDLE; - } + { + @Override + public State acknowledgeError( SessionStateMachine ctx ) + { + return IDLE; + } - @Override - protected State onNoImplementation( SessionStateMachine ctx, String command ) - { - ctx.ignored(); - return ERROR; - } - }, + @Override + protected State onNoImplementation( SessionStateMachine ctx, String command ) + { + ctx.ignored(); + return ERROR; + } + }, /** The state machine is permanently stopped. */ STOPPED - { - @Override - public State halt( SessionStateMachine ctx ) - { - throw new IllegalStateException( "No operations allowed, session has been closed." ); - } - } - ; + { + @Override + public State halt( SessionStateMachine ctx ) + { + throw new IllegalStateException( "No operations allowed, session has been closed." ); + } + }; // Operations that a session can perform. Individual states override these if they want to support them. - public State runStatement( SessionStateMachine ctx, String statement, Map params ) + public State runStatement( SessionStateMachine ctx, String statement, Map params ) { return onNoImplementation( ctx, "running a statement" ); } @@ -264,18 +263,18 @@ public State acknowledgeError( SessionStateMachine ctx ) protected State onNoImplementation( SessionStateMachine ctx, String command ) { String msg = "'" + command + "' cannot be done when a session is in the '" + ctx.state.name() + "' state."; - return error(ctx, new Neo4jError( Status.Request.Invalid, msg )); + return error( ctx, new Neo4jError( Status.Request.Invalid, msg ) ); } public State halt( SessionStateMachine ctx ) { - if(ctx.currentTransaction != null) + if ( ctx.currentTransaction != null ) { try { ctx.currentTransaction.close(); } - catch( Throwable e) + catch ( Throwable e ) { ctx.error( ctx.errTrans.translate( e ) ); } @@ -342,7 +341,7 @@ public String[] fieldNames() /** Callback attachment */ private Object currentAttachment; - + private ThreadToStatementContextBridge txBridge; /** @@ -355,7 +354,8 @@ public String[] fieldNames() // Note: We shouldn't depend on GDB like this, I think. Better to define an SPI that we can shape into a spec // for exactly the kind of underlying support the state machine needs. - public SessionStateMachine( GraphDatabaseService db, ThreadToStatementContextBridge txBridge, StatementRunner engine, Log log ) + public SessionStateMachine( GraphDatabaseService db, ThreadToStatementContextBridge txBridge, + StatementRunner engine, Log log ) { this.db = db; this.txBridge = txBridge; @@ -372,13 +372,15 @@ public String key() } @Override - public void run( String statement, Map params, A attachment, Callback callback ) + public void run( String statement, Map params, A attachment, + Callback callback ) { before( attachment, callback ); try { state = state.runStatement( this, statement, params ); - } finally { after(); } + } + finally { after(); } } @Override @@ -388,7 +390,8 @@ public void pullAll( A attachment, Callback callback ) try { state = state.pullAll( this ); - } finally { after(); } + } + finally { after(); } } @Override @@ -398,7 +401,8 @@ public void discardAll( A attachment, Callback callback ) try { state = state.discardAll( this ); - } finally { after(); } + } + finally { after(); } } @Override @@ -408,7 +412,8 @@ public void acknowledgeFailure( A attachment, Callback callback ) try { state = state.acknowledgeError( this ); - } finally { after(); } + } + finally { after(); } } @Override @@ -418,7 +423,8 @@ public void close() try { state = state.halt( this ); - } finally { after(); } + } + finally { after(); } } @Override @@ -486,7 +492,7 @@ private void before( Object attachment, Callback cb ) } /** Signal to the currently attached client callback that the request has been processed */ - @SuppressWarnings( "unchecked" ) + @SuppressWarnings("unchecked") private void after() { try @@ -516,7 +522,7 @@ private void after() /** Forward an error to the currently attached callback */ private void error( Neo4jError err ) { - if(currentCallback != null) + if ( currentCallback != null ) { currentCallback.failure( err, currentAttachment ); } @@ -525,7 +531,7 @@ private void error( Neo4jError err ) /** Forward a result to the currently attached callback */ private void result( Object result ) throws Exception { - if(currentCallback != null) + if ( currentCallback != null ) { currentCallback.result( result, currentAttachment ); } @@ -537,9 +543,9 @@ private void result( Object result ) throws Exception */ private void ignored() { - if(currentCallback != null) + if ( currentCallback != null ) { - currentCallback.ignored(currentAttachment); + currentCallback.ignored( currentAttachment ); } } } diff --git a/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/integration/RecordingCallback.java b/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/integration/RecordingCallback.java index 4e5be4a06d699..1d04623d0dc11 100644 --- a/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/integration/RecordingCallback.java +++ b/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/integration/RecordingCallback.java @@ -40,7 +40,7 @@ public class RecordingCallback implements Session.Callback public Call next() throws InterruptedException { Call msg = calls.poll( 10, TimeUnit.SECONDS ); - if(msg == null) + if ( msg == null ) { throw new RuntimeException( "Waited 10 seconds for message, but no message arrived." ); } @@ -66,7 +66,7 @@ public void result( Object result, Object attachment ) throw new RuntimeException( e ); } } - else if(result instanceof StatementMetadata) + else if ( result instanceof StatementMetadata ) { results.add( new StatementSuccess( (StatementMetadata) result ) ); } @@ -87,9 +87,9 @@ public void completed( Object attachment ) { try { - if(ignored) + if ( ignored ) { - calls.add(new Ignored()); + calls.add( new Ignored() ); } else if ( errors.size() > 0 ) { @@ -107,7 +107,8 @@ else if ( results.size() == 1 ) { calls.add( new Success() ); } - } finally + } + finally { results.clear(); errors.clear(); @@ -176,7 +177,7 @@ public static class StatementSuccess extends Success { private final StatementMetadata meta; - public StatementSuccess(StatementMetadata meta) + public StatementSuccess( StatementMetadata meta ) { this.meta = meta; } diff --git a/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/integration/SessionIT.java b/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/integration/SessionIT.java index 1cf7b7c1b25de..c59bd8e67114b 100644 --- a/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/integration/SessionIT.java +++ b/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/integration/SessionIT.java @@ -34,15 +34,16 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; import static org.neo4j.ndp.runtime.Session.Callback.NO_OP; -import static org.neo4j.ndp.runtime.integration.SessionMatchers.streamContaining; import static org.neo4j.ndp.runtime.integration.SessionMatchers.failedWith; +import static org.neo4j.ndp.runtime.integration.SessionMatchers.streamContaining; import static org.neo4j.ndp.runtime.integration.SessionMatchers.success; import static org.neo4j.runtime.internal.runner.StreamMatchers.eqRecord; -@SuppressWarnings( "unchecked" ) +@SuppressWarnings("unchecked") public class SessionIT { - @Rule public TestSessions env = new TestSessions(); + @Rule + public TestSessions env = new TestSessions(); private final RecordingCallback responses = new RecordingCallback(); @Test @@ -62,7 +63,7 @@ public void shouldExecuteStatement() throws Throwable session.pullAll( null, responses ); // Then - assertThat(responses.next(), streamContaining( + assertThat( responses.next(), streamContaining( eqRecord( equalTo( "k" ) ) ) ); } @@ -73,14 +74,14 @@ public void shouldSucceedOn__run__pullAll__run() throws Throwable Session session = env.newSession(); // And Given that I've ran and pulled one stream - session.run( "RETURN 1", EMPTY_MAP, null, Session.Callback.NO_OP ); - session.pullAll( null, Session.Callback.NO_OP ); + session.run( "RETURN 1", EMPTY_MAP, null, Session.Callback.NO_OP ); + session.pullAll( null, Session.Callback.NO_OP ); // When I run a new statement session.run( "RETURN 1", EMPTY_MAP, null, responses ); // Then - assertThat( responses.next(), success()); + assertThat( responses.next(), success() ); } @Test @@ -90,14 +91,14 @@ public void shouldSucceedOn__run__discardAll__run() throws Throwable Session session = env.newSession(); // And Given that I've ran and pulled one stream - session.run( "RETURN 1", EMPTY_MAP, null, Session.Callback.NO_OP ); + session.run( "RETURN 1", EMPTY_MAP, null, Session.Callback.NO_OP ); session.discardAll( null, Session.Callback.NO_OP ); // When I run a new statement session.run( "RETURN 1", EMPTY_MAP, null, responses ); // Then - assertThat( responses.next(), success()); + assertThat( responses.next(), success() ); } @Test @@ -107,7 +108,7 @@ public void shouldSucceedOn__run_BEGIN__pullAll__run_COMMIT__pullALL__run_COMMIT Session session = env.newSession(); // And Given that I've ran and pulled one stream - session.run( "BEGIN", EMPTY_MAP, null, Session.Callback.NO_OP ); + session.run( "BEGIN", EMPTY_MAP, null, Session.Callback.NO_OP ); session.pullAll( null, Session.Callback.NO_OP ); session.run( "COMMIT", EMPTY_MAP, null, NO_OP ); session.pullAll( null, Session.Callback.NO_OP ); @@ -116,7 +117,7 @@ public void shouldSucceedOn__run_BEGIN__pullAll__run_COMMIT__pullALL__run_COMMIT session.run( "BEGIN", EMPTY_MAP, null, responses ); // Then - assertThat( responses.next(), success()); + assertThat( responses.next(), success() ); } @Test @@ -126,13 +127,13 @@ public void shouldFailOn__run__run() throws Throwable Session session = env.newSession(); // And Given that I've ran one statement - session.run( "RETURN 1", EMPTY_MAP, null, Session.Callback.NO_OP ); + session.run( "RETURN 1", EMPTY_MAP, null, Session.Callback.NO_OP ); // When I run a new statement, before consuming the stream session.run( "RETURN 1", EMPTY_MAP, null, responses ); // Then - assertThat( responses.next(), failedWith( Status.Request.Invalid )); + assertThat( responses.next(), failedWith( Status.Request.Invalid ) ); } @Test @@ -142,14 +143,14 @@ public void shouldFailOn__pullAll__pullAll() throws Throwable Session session = env.newSession(); // And Given that I've ran and pulled one stream - session.run( "RETURN 1", EMPTY_MAP, null, Session.Callback.NO_OP ); + session.run( "RETURN 1", EMPTY_MAP, null, Session.Callback.NO_OP ); session.pullAll( null, Session.Callback.NO_OP ); // When I attempt to pull more items from the stream session.pullAll( null, responses ); // Then - assertThat( responses.next(), failedWith( Status.Request.Invalid )); + assertThat( responses.next(), failedWith( Status.Request.Invalid ) ); } @Test @@ -159,14 +160,14 @@ public void shouldFailOn__pullAll__discardAll() throws Throwable Session session = env.newSession(); // And Given that I've ran and pulled one stream - session.run( "RETURN 1", EMPTY_MAP, null, Session.Callback.NO_OP ); + session.run( "RETURN 1", EMPTY_MAP, null, Session.Callback.NO_OP ); session.pullAll( null, Session.Callback.NO_OP ); // When I attempt to pull more items from the stream session.discardAll( null, responses ); // Then - assertThat( responses.next(), failedWith( Status.Request.Invalid )); + assertThat( responses.next(), failedWith( Status.Request.Invalid ) ); } @Test @@ -176,14 +177,14 @@ public void shouldFailOn__discardAll__discardAll() throws Throwable Session session = env.newSession(); // And Given that I've ran and pulled one stream - session.run( "RETURN 1", EMPTY_MAP, null, Session.Callback.NO_OP ); + session.run( "RETURN 1", EMPTY_MAP, null, Session.Callback.NO_OP ); session.discardAll( null, Session.Callback.NO_OP ); // When I attempt to pull more items from the stream session.discardAll( null, responses ); // Then - assertThat( responses.next(), failedWith( Status.Request.Invalid )); + assertThat( responses.next(), failedWith( Status.Request.Invalid ) ); } @Test @@ -193,14 +194,14 @@ public void shouldFailOn__discardAll__pullAll() throws Throwable Session session = env.newSession(); // And Given that I've ran and pulled one stream - session.run( "RETURN 1", EMPTY_MAP, null, Session.Callback.NO_OP ); + session.run( "RETURN 1", EMPTY_MAP, null, Session.Callback.NO_OP ); session.discardAll( null, Session.Callback.NO_OP ); // When I attempt to pull more items from the stream session.pullAll( null, responses ); // Then - assertThat( responses.next(), failedWith( Status.Request.Invalid )); + assertThat( responses.next(), failedWith( Status.Request.Invalid ) ); } @Test @@ -219,7 +220,7 @@ public void shouldHandleImplicitCommitFailure() throws Throwable assertThat( responses.next(), success() ); // But the stop should have failed, since it implicitly triggers commit and thus triggers a failure - assertThat(responses.next(), failedWith( Status.Transaction.ValidationFailed ) ); + assertThat( responses.next(), failedWith( Status.Transaction.ValidationFailed ) ); } @Test @@ -247,8 +248,8 @@ public void failure( Neo4jError err, Object attachment ) } ); // Then - assertThat( error.get(), equalTo(new Neo4jError( Status.General.UnknownFailure, - "An unexpected failure occurred: 'Ooopsies!'." )) ); + assertThat( error.get(), equalTo( new Neo4jError( Status.General.UnknownFailure, + "An unexpected failure occurred: 'Ooopsies!'." ) ) ); } @Test @@ -263,15 +264,15 @@ public void shouldBeAbleToCleanlyRunMultipleSessionsInSingleThread() throws Thro // When I issue a statement in a separate session Object[] stream = runAndPull( secondSession, "CREATE (a:Person) RETURN id(a)" ); - long id = (long) ((Record)stream[0]).fields()[0]; + long id = (long) ((Record) stream[0]).fields()[0]; // And when I roll back that first session transaction runAndPull( firstSession, "ROLLBACK" ); // Then the two should not have interfered with each other stream = runAndPull( secondSession, "MATCH (a:Person) WHERE id(a) = " + id + " RETURN COUNT(*)" ); - assertThat( ((Record)stream[0]).fields()[0], equalTo( (Object) 1L ) ); - + assertThat( ((Record) stream[0]).fields()[0], equalTo( (Object) 1L ) ); + } @Test @@ -290,7 +291,7 @@ public void shouldAllowNewTransactionAfterFailure() throws Throwable Object[] stream = runAndPull( session, "RETURN 1" ); // Then - assertThat( ((Record)stream[0]).fields()[0], equalTo((Object) 1L ) ); + assertThat( ((Record) stream[0]).fields()[0], equalTo( (Object) 1L ) ); } diff --git a/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/integration/SessionMatchers.java b/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/integration/SessionMatchers.java index d11688f7070d5..4a86aa02720eb 100644 --- a/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/integration/SessionMatchers.java +++ b/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/integration/SessionMatchers.java @@ -62,15 +62,15 @@ public static Matcher statementMetadata( final String[] @Override protected boolean matchesSafely( RecordingCallback.Call item ) { - if(!(item instanceof RecordingCallback.StatementSuccess)) + if ( !(item instanceof RecordingCallback.StatementSuccess) ) { return false; } StatementMetadata meta = ((RecordingCallback.StatementSuccess) item).meta(); - assertTrue( Arrays.toString(fieldNames) + " == " + Arrays.toString( meta.fieldNames() ), - Arrays.equals( fieldNames, meta.fieldNames() )); + assertTrue( Arrays.toString( fieldNames ) + " == " + Arrays.toString( meta.fieldNames() ), + Arrays.equals( fieldNames, meta.fieldNames() ) ); return true; } @@ -83,7 +83,7 @@ public void describeTo( Description description ) }; } - public static Matcher callsWere( final Matcher ... calls ) + public static Matcher callsWere( final Matcher... calls ) { return new TypeSafeMatcher() { @@ -97,7 +97,7 @@ protected boolean matchesSafely( RecordingCallback item ) assertThat( item.next(), calls[i] ); } } - catch(InterruptedException e) + catch ( InterruptedException e ) { throw new RuntimeException( e ); } @@ -107,7 +107,7 @@ protected boolean matchesSafely( RecordingCallback item ) @Override public void describeTo( Description description ) { - description.appendList( "Calls[",",","]", asList(calls) ); + description.appendList( "Calls[", ",", "]", asList( calls ) ); } }; } @@ -119,7 +119,7 @@ public static Matcher streamContaining( final Matcher @Override protected boolean matchesSafely( RecordingCallback.Call item ) { - if(!(item instanceof RecordingCallback.Result) ) + if ( !(item instanceof RecordingCallback.Result) ) { return false; } @@ -127,7 +127,7 @@ protected boolean matchesSafely( RecordingCallback.Call item ) Object[] actual = ((RecordingCallback.Result) item).records(); for ( int i = 0; i < values.length; i++ ) { - if(!values[i].matches( actual[i] )) + if ( !values[i].matches( actual[i] ) ) { return false; } @@ -140,7 +140,7 @@ protected boolean matchesSafely( RecordingCallback.Call item ) public void describeTo( Description description ) { description.appendText( "Stream{" ); - description.appendList( "values=[", "\n", "]" , asList(values)); + description.appendList( "values=[", "\n", "]", asList( values ) ); description.appendText( "}" ); } }; @@ -184,15 +184,15 @@ public void describeTo( Description description ) }; } - public static Matcher> mapMatcher( final Object... alternatingKeyValue ) + public static Matcher> mapMatcher( final Object... alternatingKeyValue ) { - final Map expected = MapUtil.map( alternatingKeyValue ); - return new TypeSafeMatcher>() + final Map expected = MapUtil.map( alternatingKeyValue ); + return new TypeSafeMatcher>() { @Override - protected boolean matchesSafely( Map item ) + protected boolean matchesSafely( Map item ) { - assertThat( item.entrySet(), equalTo(expected.entrySet()) ); + assertThat( item.entrySet(), equalTo( expected.entrySet() ) ); return true; } diff --git a/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/integration/TestSessions.java b/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/integration/TestSessions.java index d0aca337e29be..96eec513aaddc 100644 --- a/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/integration/TestSessions.java +++ b/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/integration/TestSessions.java @@ -29,8 +29,8 @@ import org.neo4j.kernel.GraphDatabaseAPI; import org.neo4j.kernel.lifecycle.LifeSupport; import org.neo4j.logging.NullLog; -import org.neo4j.ndp.runtime.Sessions; import org.neo4j.ndp.runtime.Session; +import org.neo4j.ndp.runtime.Sessions; import org.neo4j.ndp.runtime.internal.StandardSessions; import org.neo4j.test.TestGraphDatabaseFactory; @@ -50,7 +50,7 @@ public Statement apply( final Statement base, Description description ) public void evaluate() throws Throwable { gdb = new TestGraphDatabaseFactory().newImpermanentDatabase(); - actual = life.add(new StandardSessions( (GraphDatabaseAPI) gdb, NullLog.getInstance() )); + actual = life.add( new StandardSessions( (GraphDatabaseAPI) gdb, NullLog.getInstance() ) ); life.start(); try { @@ -64,7 +64,8 @@ public void evaluate() throws Throwable { session.close(); } - } catch(Throwable e) { e.printStackTrace(); } + } + catch ( Throwable e ) { e.printStackTrace(); } gdb.shutdown(); } @@ -75,7 +76,7 @@ public void evaluate() throws Throwable @Override public Session newSession() { - if(actual == null) + if ( actual == null ) { throw new IllegalStateException( "Cannot access test environment before test is running." ); } diff --git a/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/integration/TransactionIT.java b/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/integration/TransactionIT.java index 1c051015bb254..cd8c30f0dfb6f 100644 --- a/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/integration/TransactionIT.java +++ b/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/integration/TransactionIT.java @@ -33,7 +33,8 @@ public class TransactionIT @Rule public TestSessions env = new TestSessions(); - @Test @SuppressWarnings( "unchecked" ) + @Test + @SuppressWarnings("unchecked") public void shouldHandleBeginCommit() throws Throwable { // Given @@ -56,7 +57,8 @@ public void shouldHandleBeginCommit() throws Throwable assertThat( responses.next(), success() ); } - @Test @SuppressWarnings( "unchecked" ) + @Test + @SuppressWarnings("unchecked") public void shouldHandleBeginRollback() throws Throwable { // Given diff --git a/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/internal/CypherCursorAdapterTest.java b/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/internal/CypherCursorAdapterTest.java index 6955f582c33bf..c53980e5ed6eb 100644 --- a/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/internal/CypherCursorAdapterTest.java +++ b/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/internal/CypherCursorAdapterTest.java @@ -29,8 +29,8 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.neo4j.runtime.internal.runner.StreamMatchers.equalsStream; import static org.neo4j.runtime.internal.runner.StreamMatchers.eqRecord; +import static org.neo4j.runtime.internal.runner.StreamMatchers.equalsStream; public class CypherCursorAdapterTest { @@ -51,6 +51,6 @@ public void nextShouldNotEqualNotNext() assertThat( cursor, equalsStream( new String[]{"name"}, eqRecord( equalTo( "bob" ) ), - eqRecord( equalTo( "Steve Brook" ) )) ); + eqRecord( equalTo( "Steve Brook" ) ) ) ); } } \ No newline at end of file diff --git a/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/internal/CypherStatementRunnerTest.java b/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/internal/CypherStatementRunnerTest.java index bd95ecf46be22..218827ce9e389 100644 --- a/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/internal/CypherStatementRunnerTest.java +++ b/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/internal/CypherStatementRunnerTest.java @@ -38,7 +38,7 @@ public class CypherStatementRunnerTest private final SessionStateMachine ctx = mock( SessionStateMachine.class ); @Test - @SuppressWarnings( "unchecked" ) + @SuppressWarnings("unchecked") public void shouldCreateImplicitTxIfNoneExists() throws Exception { // Given diff --git a/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/internal/session/StateMachineErrorTest.java b/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/internal/session/StateMachineErrorTest.java index 07c71f09559f2..a99ee58d06ba2 100644 --- a/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/internal/session/StateMachineErrorTest.java +++ b/community/ndp/runtime/src/test/java/org/neo4j/ndp/runtime/internal/session/StateMachineErrorTest.java @@ -59,7 +59,7 @@ public class StateMachineErrorTest @Before public void setup() { - when(db.beginTx()).thenReturn( tx ); + when( db.beginTx() ).thenReturn( tx ); } @Test @@ -68,8 +68,8 @@ public void testSyntaxError() throws Throwable // Given RecordingCallback responses = new RecordingCallback(); - doThrow( new SyntaxException( "src/test" ) ).when( runner ).run( any(SessionState.class), - any(String.class), any(Map.class) ); + doThrow( new SyntaxException( "src/test" ) ).when( runner ).run( any( SessionState.class ), + any( String.class ), any( Map.class ) ); SessionStateMachine machine = new SessionStateMachine( db, txBridge, runner, NullLog.getInstance() ); @@ -77,8 +77,8 @@ public void testSyntaxError() throws Throwable machine.run( "this is nonsense", EMPTY_MAP, null, responses ); // Then - assertThat(responses.next(), failedWith( Status.Statement.InvalidSyntax ) ); - assertThat(machine.state(), equalTo( ERROR )); + assertThat( responses.next(), failedWith( Status.Statement.InvalidSyntax ) ); + assertThat( machine.state(), equalTo( ERROR ) ); } @Test @@ -93,7 +93,7 @@ public void result( Object result, Object attachment ) throw new RuntimeException( "Well, that didn't work out very well." ); } }; - when(runner.run( any(SessionState.class), any(String.class), any(Map.class) )) + when( runner.run( any( SessionState.class ), any( String.class ), any( Map.class ) ) ) .thenReturn( mock( RecordStream.class ) ); SessionStateMachine machine = new SessionStateMachine( db, txBridge, runner, NullLog.getInstance() ); @@ -105,8 +105,8 @@ public void result( Object result, Object attachment ) machine.pullAll( null, failingCallback ); // Then - assertThat(failingCallback.next(), failedWith( Status.General.UnknownFailure )); - assertThat(machine.state(), equalTo( ERROR )); + assertThat( failingCallback.next(), failedWith( Status.General.UnknownFailure ) ); + assertThat( machine.state(), equalTo( ERROR ) ); } @Test @@ -119,13 +119,13 @@ public void testRollbackError() throws Throwable machine.beginTransaction(); // And given that transaction will fail to roll back - doThrow(new TransactionFailureException( "This just isn't going well for us." )).when( tx ).close(); + doThrow( new TransactionFailureException( "This just isn't going well for us." ) ).when( tx ).close(); // When machine.rollbackTransaction(); // Then - assertThat(machine.state(), equalTo( ERROR )); + assertThat( machine.state(), equalTo( ERROR ) ); } @Test @@ -139,28 +139,28 @@ public void testCantDoAnythingIfInErrorState() throws Throwable machine.commitTransaction(); // No tx to be committed! // Then it should be in an error state - assertThat(machine.state(), equalTo(ERROR)); + assertThat( machine.state(), equalTo( ERROR ) ); // and no action other than acknowledging the error should be possible machine.beginTransaction(); - assertThat(machine.state(), equalTo(ERROR)); + assertThat( machine.state(), equalTo( ERROR ) ); machine.beginImplicitTransaction(); - assertThat(machine.state(), equalTo(ERROR)); + assertThat( machine.state(), equalTo( ERROR ) ); machine.commitTransaction(); - assertThat(machine.state(), equalTo(ERROR)); + assertThat( machine.state(), equalTo( ERROR ) ); machine.rollbackTransaction(); - assertThat(machine.state(), equalTo(ERROR)); + assertThat( machine.state(), equalTo( ERROR ) ); // this includes externally triggered actions machine.run( "src/test", EMPTY_MAP, null, messages ); - assertThat(machine.state(), equalTo(ERROR)); + assertThat( machine.state(), equalTo( ERROR ) ); assertThat( messages.next(), ignored() ); machine.pullAll( null, messages ); - assertThat(machine.state(), equalTo(ERROR)); + assertThat( machine.state(), equalTo( ERROR ) ); assertThat( messages.next(), ignored() ); // And nothing at all should have been done diff --git a/community/ndp/transport-socket/pom.xml b/community/ndp/transport-socket/pom.xml index 23cd071f13587..dadb570e4c2dc 100644 --- a/community/ndp/transport-socket/pom.xml +++ b/community/ndp/transport-socket/pom.xml @@ -139,7 +139,6 @@ - org.apache.maven.plugins maven-failsafe-plugin @@ -150,6 +149,16 @@ + + maven-jar-plugin + + + + test-jar + + + + maven-antrun-plugin diff --git a/community/ndp/transport-socket/src/main/java/org/neo4j/ndp/transport/socket/ChunkedInput.java b/community/ndp/transport-socket/src/main/java/org/neo4j/ndp/transport/socket/ChunkedInput.java index 263a1a291aa6d..d43c6c358755c 100644 --- a/community/ndp/transport-socket/src/main/java/org/neo4j/ndp/transport/socket/ChunkedInput.java +++ b/community/ndp/transport-socket/src/main/java/org/neo4j/ndp/transport/socket/ChunkedInput.java @@ -212,4 +212,5 @@ private void ensureChunkAvailable() } } } + } diff --git a/community/ndp/transport-socket/src/main/java/org/neo4j/ndp/transport/socket/ChunkedOutput.java b/community/ndp/transport-socket/src/main/java/org/neo4j/ndp/transport/socket/ChunkedOutput.java index 610a36d746fb3..279e9c5f18d22 100644 --- a/community/ndp/transport-socket/src/main/java/org/neo4j/ndp/transport/socket/ChunkedOutput.java +++ b/community/ndp/transport-socket/src/main/java/org/neo4j/ndp/transport/socket/ChunkedOutput.java @@ -29,12 +29,18 @@ public class ChunkedOutput implements PackOutput { public static final int CHUNK_HEADER_SIZE = 2; + public static final int MESSAGE_BOUNDARY = 0; + private final int bufferSize; + private final int maxChunkSize; + private ByteBuf buffer; private ChannelHandlerContext channel; private int currentChunkHeaderOffset; + /** Are currently in the middle of writing a chunk? */ private boolean chunkOpen = false; + private Runnable onMessageComplete = new Runnable() { @Override @@ -42,11 +48,16 @@ public void run() { try { - closeCurrentChunk(); + closeChunkIfOpen(); + + // Ensure there's space to write the message boundary + if ( buffer.writableBytes() < CHUNK_HEADER_SIZE ) + { + flush(); + } // Write message boundary - ensure( CHUNK_HEADER_SIZE ); - putShort( (short) 0 ); + buffer.writeShort( MESSAGE_BOUNDARY ); // Mark us as not currently in a chunk chunkOpen = false; @@ -68,11 +79,13 @@ public ChunkedOutput() public ChunkedOutput( int bufferSize ) { this.bufferSize = bufferSize; + this.maxChunkSize = bufferSize - CHUNK_HEADER_SIZE; } @Override public PackOutput ensure( int size ) throws IOException { + assert size <= maxChunkSize : size + " > " + maxChunkSize; if ( buffer == null ) { newBuffer(); @@ -95,14 +108,12 @@ else if ( buffer.writableBytes() < size ) @Override public PackOutput flush() throws IOException { - if ( chunkOpen ) + if ( buffer != null && buffer.readableBytes() > 0 ) { - closeCurrentChunk(); + closeChunkIfOpen(); + channel.writeAndFlush( buffer ); + newBuffer(); } - - channel.writeAndFlush( buffer ); - newBuffer(); - return this; } @@ -152,10 +163,11 @@ public PackOutput put( byte[] data, int offset, int length ) throws IOException int index = 0; while ( index < length ) { - int amountToWrite = Math.min( buffer.writableBytes(), length - index ); + int amountToWrite = Math.min( buffer == null ? maxChunkSize : buffer.writableBytes() - CHUNK_HEADER_SIZE, + length - index ); ensure( amountToWrite ); - buffer.writeBytes( data, offset, amountToWrite ); + buffer.writeBytes( data, offset + index, amountToWrite ); index += amountToWrite; if ( buffer.writableBytes() == 0 ) @@ -166,10 +178,14 @@ public PackOutput put( byte[] data, int offset, int length ) throws IOException return this; } - private void closeCurrentChunk() + private void closeChunkIfOpen() { - int chunkSize = buffer.writerIndex() - (currentChunkHeaderOffset + CHUNK_HEADER_SIZE); - buffer.setShort( currentChunkHeaderOffset, chunkSize ); + if ( chunkOpen ) + { + int chunkSize = buffer.writerIndex() - (currentChunkHeaderOffset + CHUNK_HEADER_SIZE); + buffer.setShort( currentChunkHeaderOffset, chunkSize ); + chunkOpen = false; + } } private void newBuffer() diff --git a/community/ndp/transport-socket/src/main/java/org/neo4j/ndp/transport/socket/SocketProtocolV1.java b/community/ndp/transport-socket/src/main/java/org/neo4j/ndp/transport/socket/SocketProtocolV1.java index 74f2f3bc7a437..8fc552f12f1ee 100644 --- a/community/ndp/transport-socket/src/main/java/org/neo4j/ndp/transport/socket/SocketProtocolV1.java +++ b/community/ndp/transport-socket/src/main/java/org/neo4j/ndp/transport/socket/SocketProtocolV1.java @@ -54,6 +54,12 @@ public class SocketProtocolV1 implements SocketProtocol private final Log log; private final AtomicInteger inFlight = new AtomicInteger( 0 ); + private final byte[] currentHeader = new byte[2]; + + private boolean isReadingSplitHeader = false; + private boolean isReadingSplitChunk = false; + private int splitChunkRemaining = 0; + public SocketProtocolV1( final Log log, Session session ) { this.log = log; @@ -84,25 +90,77 @@ public void handle( ChannelHandlerContext channelContext, ByteBuf data ) onBatchOfMessagesStarted(); while ( data.readableBytes() > 0 ) { - int chunkSize = data.readUnsignedShort(); - if ( chunkSize > 0 ) + // Chunks may split across network packages, so we get partial chunks. If that's currently the case, + // we gather those parts up as if they were individual chunks + if ( isReadingSplitChunk ) + { + collectChunk( data, splitChunkRemaining ); + } + else if ( data.readableBytes() == 1 ) + { + // Chunk header is split across network packets.. + currentHeader[0] = data.readByte(); + } + else { - if ( chunkSize <= data.readableBytes() ) + // Read next chunk header + int chunkSize; + if ( isReadingSplitHeader ) { - // Incoming buffer contains the whole chunk, forward it to our chunked input handling - input.addChunk( data.readSlice( chunkSize ) ); + currentHeader[1] = data.readByte(); + chunkSize = (currentHeader[0] << 8 | currentHeader[1]) & 0xffff; } else { - throw new UnsupportedOperationException( "Chunks split across packets not yet supported" ); // TODO + chunkSize = data.readUnsignedShort(); + } + + // Is there data in the chunk, or are we at a message boundary? + if ( chunkSize > 0 ) + { + // Save this chunk + collectChunk( data, chunkSize ); + } + else + { + // A full message collected, forward it for processing + processCollectedMessage( channelContext ); } } + } + onBatchOfMessagesDone(); + } + + private void collectChunk( ByteBuf data, int chunkSize ) + { + if ( chunkSize <= data.readableBytes() ) + { + // Incoming buffer contains the whole chunk, forward it to our chunked input handling + input.addChunk( data.readSlice( chunkSize ) ); + isReadingSplitChunk = false; + } + else + { + // Incoming buffer does not contain all the data we expect in the current chunk, gather up whats + // available, treat that as a chunk and remember how much we've got left to read from the next network + // packet. + if ( data.readableBytes() > 0 ) + { + input.addChunk( data ); + } + + if ( isReadingSplitChunk ) + { + // In the middle of reading split chunks, count down how much remains. + splitChunkRemaining -= chunkSize; + } else { - processChunkedMessage( channelContext ); + // First of a series of split chunk pieces, note the total length we need to read + isReadingSplitChunk = true; + splitChunkRemaining = chunkSize; } } - onBatchOfMessagesDone(); } @Override @@ -111,7 +169,7 @@ public int version() return VERSION; } - public void processChunkedMessage( final ChannelHandlerContext channelContext ) + public void processCollectedMessage( final ChannelHandlerContext channelContext ) { output.setTargetChannel( channelContext ); try diff --git a/community/ndp/transport-socket/src/main/java/org/neo4j/ndp/transport/socket/SocketTransportHandler.java b/community/ndp/transport-socket/src/main/java/org/neo4j/ndp/transport/socket/SocketTransportHandler.java index 401db47325329..0002ba0c9b6b0 100644 --- a/community/ndp/transport-socket/src/main/java/org/neo4j/ndp/transport/socket/SocketTransportHandler.java +++ b/community/ndp/transport-socket/src/main/java/org/neo4j/ndp/transport/socket/SocketTransportHandler.java @@ -68,6 +68,8 @@ public void channelRead( ChannelHandlerContext ctx, Object msg ) throws Exceptio } } + // TODO: Handle channelInactive (eg. close our session if we've opened one) + private void chooseProtocolVersion( ChannelHandlerContext ctx, ByteBuf buffer ) throws Exception { switch ( protocolChooser.handleVersionHandshakeChunk( buffer ) ) diff --git a/community/ndp/transport-socket/src/test/java/org/neo4j/ndp/transport/socket/ChunkedInputTest.java b/community/ndp/transport-socket/src/test/java/org/neo4j/ndp/transport/socket/ChunkedInputTest.java index 4ebd251eeceaf..ef2ef168b6ac3 100644 --- a/community/ndp/transport-socket/src/test/java/org/neo4j/ndp/transport/socket/ChunkedInputTest.java +++ b/community/ndp/transport-socket/src/test/java/org/neo4j/ndp/transport/socket/ChunkedInputTest.java @@ -81,4 +81,22 @@ public void shouldReadIntoMisalignedDestinationBuffer() throws Throwable // Then assertThat( bytes, equalTo( new byte[]{7, 0, 0} ) ); } + + @Test + public void shouldReadPartialChunk() throws Throwable + { + // Given + ChunkedInput ch = new ChunkedInput(); + + ch.addChunk( wrappedBuffer( new byte[]{1, 2} ) ); + ch.addChunk( wrappedBuffer( new byte[]{3} ) ); + ch.addChunk( wrappedBuffer( new byte[]{4, 5} ) ); + + // When + byte[] bytes = new byte[5]; + ch.get( bytes, 0, 5 ); + + // Then + assertThat( bytes, equalTo( new byte[]{1, 2, 3, 4, 5} ) ); + } } \ No newline at end of file diff --git a/community/ndp/transport-socket/src/test/java/org/neo4j/ndp/transport/socket/ChunkedOutputTest.java b/community/ndp/transport-socket/src/test/java/org/neo4j/ndp/transport/socket/ChunkedOutputTest.java index 001bdfc47e60c..e8e9c3b2b1d34 100644 --- a/community/ndp/transport-socket/src/test/java/org/neo4j/ndp/transport/socket/ChunkedOutputTest.java +++ b/community/ndp/transport-socket/src/test/java/org/neo4j/ndp/transport/socket/ChunkedOutputTest.java @@ -22,14 +22,13 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.UnpooledByteBufAllocator; import io.netty.channel.ChannelHandlerContext; - -import java.nio.ByteBuffer; - import org.junit.Before; import org.junit.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; +import java.nio.ByteBuffer; + import org.neo4j.kernel.impl.util.HexPrinter; import static org.hamcrest.CoreMatchers.equalTo; diff --git a/community/ndp/transport-socket/src/test/java/org/neo4j/ndp/transport/socket/integration/NDPConn.java b/community/ndp/transport-socket/src/test/java/org/neo4j/ndp/transport/socket/integration/NDPConn.java index 6eeb4258ee4d4..35c8fe562b5ab 100644 --- a/community/ndp/transport-socket/src/test/java/org/neo4j/ndp/transport/socket/integration/NDPConn.java +++ b/community/ndp/transport-socket/src/test/java/org/neo4j/ndp/transport/socket/integration/NDPConn.java @@ -24,10 +24,12 @@ import java.io.OutputStream; import java.net.InetSocketAddress; import java.net.Socket; +import java.nio.ByteBuffer; import org.neo4j.helpers.HostnamePort; +import org.neo4j.kernel.impl.util.HexPrinter; -public class NDPConn +public class NDPConn implements AutoCloseable { private Socket socket; private InputStream in; @@ -53,18 +55,45 @@ public byte[] recv( int length ) throws IOException, InterruptedException long timeout = System.currentTimeMillis() + 1000 * 30; byte[] bytes = new byte[length]; int left = length, read; + + waitUntilAvailable( bytes, timeout, left ); + while ( (read = in.read( bytes, length - left, left )) != -1 && left > 0 ) { - if ( System.currentTimeMillis() > timeout ) - { - throw new IOException( "Waited 30 seconds for " + left + " bytes, recieved " + (length - left) ); - } left -= read; if ( left > 0 ) { - Thread.sleep( 10 ); + waitUntilAvailable( bytes, timeout, left ); } } return bytes; } + + private void waitUntilAvailable( byte[] recieved, long timeout, int left ) throws IOException + { + while ( in.available() == 0 ) + { + if ( System.currentTimeMillis() > timeout ) + { + throw new IOException( "Waited 30 seconds for " + left + " bytes, " + + "recieved " + (recieved.length - left) + ":\n" + + HexPrinter.hex( + ByteBuffer.wrap( recieved ), 0, recieved.length - left ) ); + } + } + } + + public void disconnect() throws IOException + { + if ( socket != null && socket.isConnected() ) + { + socket.close(); + } + } + + @Override + public void close() throws Exception + { + disconnect(); + } } diff --git a/community/ndp/v1-docs/LICENSES.txt b/community/ndp/v1-docs/LICENSES.txt new file mode 100644 index 0000000000000..e62824d7096a5 --- /dev/null +++ b/community/ndp/v1-docs/LICENSES.txt @@ -0,0 +1,4 @@ +This file contains the full license text of the included third party +libraries. For an overview of the licenses see the NOTICE.txt file. + + diff --git a/community/ndp/v1-docs/NOTICE.txt b/community/ndp/v1-docs/NOTICE.txt new file mode 100644 index 0000000000000..b7383b9874625 --- /dev/null +++ b/community/ndp/v1-docs/NOTICE.txt @@ -0,0 +1,27 @@ +Neo4j +Copyright © 2002-2015 Network Engine for Objects in Lund AB (referred to +in this notice as "Neo Technology") + [http://neotechnology.com] + +This product includes software ("Software") developed by Neo Technology. + +The copyright in the bundled Neo4j graph database (including the +Software) is owned by Neo Technology. The Software developed and owned +by Neo Technology is licensed under the GNU GENERAL PUBLIC LICENSE +Version 3 (http://www.fsf.org/licensing/licenses/gpl-3.0.html) ("GPL") +to all third parties and that license, as required by the GPL, is +included in the LICENSE.txt file. + +However, if you have executed an End User Software License and Services +Agreement or an OEM Software License and Support Services Agreement, or +another commercial license agreement with Neo Technology or one of its +affiliates (each, a "Commercial Agreement"), the terms of the license in +such Commercial Agreement will supersede the GPL and you may use the +software solely pursuant to the terms of the relevant Commercial +Agreement. + +Full license texts are found in LICENSES.txt. + +Third-party licenses +-------------------- + diff --git a/community/ndp/v1-docs/pom.xml b/community/ndp/v1-docs/pom.xml new file mode 100644 index 0000000000000..698f72c1eeb7a --- /dev/null +++ b/community/ndp/v1-docs/pom.xml @@ -0,0 +1,175 @@ + + + 4.0.0 + + org.neo4j + parent + 2.3-SNAPSHOT + ../../.. + + + + GPL-3-header.txt + notice-gpl-prefix.txt + + + org.neo4j.ndp + neo4j-ndp-v1-docs + + jar + Neo4j - Data Protocol Documentation V1 + Documentation for the Data Protocol, Version 1 + + + + + org.neo4j.ndp + neo4j-ndp-transport-socket + ${project.version} + test + + + + org.neo4j.ndp + neo4j-ndp-transport-socket + ${project.version} + test-jar + test + + + + org.neo4j.ndp + neo4j-ndp-messaging-v1 + ${project.version} + test-jar + test + + + + org.neo4j + neo4j-kernel + ${project.version} + test-jar + test + + + + org.neo4j + neo4j-io + ${project.version} + test-jar + test + + + + org.neo4j.ndp + neo4j-ndp-runtime + ${project.version} + test + + + + org.neo4j + neo4j-kernel + ${project.version} + test + + + + + org.hamcrest + hamcrest-all + test + + + junit + junit + test + + + org.mockito + mockito-all + test + + + + + org.asciidoctor + asciidoctorj + 1.5.2 + test + + + org.jsoup + jsoup + 1.8.1 + test + + + + + + + maven-compiler-plugin + 2.3.2 + + 1.7 + 1.7 + + + + org.apache.maven.plugins + maven-jar-plugin + 2.2 + + + + test-jar + + + + + + + maven-antrun-plugin + + + generate-version + none + + + + + + + org.asciidoctor + asciidoctor-maven-plugin + 1.5.2 + + + generate-compliance-manual + generate-resources + + process-asciidoc + + + html + + + + + ${basedir}/src/docs/dev + ${basedir}/target/protocol-manual/html + html + + + coderay + + + + + + + \ No newline at end of file diff --git a/community/ndp/v1-docs/src/docs/dev/examples.asciidoc b/community/ndp/v1-docs/src/docs/dev/examples.asciidoc new file mode 100644 index 0000000000000..99a874ddf5c1e --- /dev/null +++ b/community/ndp/v1-docs/src/docs/dev/examples.asciidoc @@ -0,0 +1,174 @@ +[[ndp-examples]] +== Examples + +This section contains concrete examples showing how to perform tasks using the full protocol stack. + +=== Running a cypher query + +This illustrates running a simple cypher query without parameters, and retrieving the results. + +.Run query +[source,ndp_exchange] +---- +# Handshake +Client: +Client: 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 +Server: 00 00 00 01 + +Client: RUN "RETURN 1 AS num" {} + + 00 13 b2 10 8f 52 45 54 55 52 4e 20 31 20 41 53 + 20 6e 75 6d a0 00 00 + +Server: SUCCESS { fields: ['num'] } + + 00 0f b1 70 a1 86 66 69 65 6c 64 73 91 83 6e 75 + 6d 00 00 + +Client: PULL_ALL + + 00 02 B0 3F 00 00 + +Server: RECORD [1] + + 00 04 b1 71 91 01 00 00 + +Server: SUCCESS {} + + 00 03 b1 70 a0 00 00 +---- + +=== Pipelining + +This illustrates running two statements and retreiving their results, without waiting for the server to respond +in-between. + +Note that these two statements are executed in two individual transactions, implicitly created for each statement. + +.Pipelining +[source,ndp_exchange] +---- +# Handshake +Client: +Client: 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 +Server: 00 00 00 01 + +# Batch of messages +Client: RUN "RETURN 1 AS num" {} + + 00 13 b2 10 8f 52 45 54 55 52 4e 20 31 20 41 53 + 20 6e 75 6d a0 00 00 + +Client: PULL_ALL + + 00 02 B0 3F 00 00 + +Client: RUN "RETURN 1 AS num" {} + + 00 13 b2 10 8f 52 45 54 55 52 4e 20 31 20 41 53 + 20 6e 75 6d a0 00 00 + +Client: PULL_ALL + + 00 02 B0 3F 00 00 + +# Server responses +Server: SUCCESS { fields: ['num'] } + + 00 0f b1 70 a1 86 66 69 65 6c 64 73 91 83 6e 75 + 6d 00 00 + +Server: RECORD [1] + + 00 04 b1 71 91 01 00 00 + +Server: SUCCESS {} + + 00 03 b1 70 a0 00 00 + +Server: SUCCESS { fields: ['num'] } + + 00 0f b1 70 a1 86 66 69 65 6c 64 73 91 83 6e 75 + 6d 00 00 + +Server: RECORD [1] + + 00 04 b1 71 91 01 00 00 + +Server: SUCCESS {} + + 00 03 b1 70 a0 00 00 +---- + +=== Error handling + +This illustrates how the server behaves when a request fails, and the server ignores incoming messages until an +`ACK_FAILURE` message is recieved. + +.Error handling +[source,ndp_exchange] +---- +# Handshake +Client: +Client: 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 +Server: 00 00 00 01 + + +# Message with syntax error +Client: RUN "This will cause a syntax error" {} + + 00 23 b2 10 d0 1e 54 68 69 73 20 77 69 6c 6c 20 + 63 61 75 73 65 20 61 20 73 79 6e 74 61 78 20 65 + 72 72 6f 72 a0 00 00 + + +# Server responds with failure +Server: FAILURE {code:"Neo.ClientError.Statement.InvalidSyntax", + message:"Invalid input 'T': expected (line 1, column 1 (offset: 0)) + "This will cause a syntax error" + ^"} + + 00 a0 b1 7f a2 84 63 6f 64 65 d0 27 4e 65 6f 2e + 43 6c 69 65 6e 74 45 72 72 6f 72 2e 53 74 61 74 + 65 6d 65 6e 74 2e 49 6e 76 61 6c 69 64 53 79 6e + 74 61 78 87 6d 65 73 73 61 67 65 d0 65 49 6e 76 + 61 6c 69 64 20 69 6e 70 75 74 20 27 54 27 3a 20 + 65 78 70 65 63 74 65 64 20 3c 69 6e 69 74 3e 20 + 28 6c 69 6e 65 20 31 2c 20 63 6f 6c 75 6d 6e 20 + 31 20 28 6f 66 66 73 65 74 3a 20 30 29 29 0a 22 + 54 68 69 73 20 77 69 6c 6c 20 63 61 75 73 65 20 + 61 20 73 79 6e 74 61 78 20 65 72 72 6f 72 22 0a + 20 5e 00 00 + + +# Further requests are ignored +Client: PULL_ALL + + 00 02 b0 3f 00 00 + +Server: IGNORED + + 00 02 b0 7e 00 00 + + +# Until the error is acknowledged +Client: ACK_FAILURE + + 00 02 b0 0f 00 00 + +Server: SUCCESS {} + + 00 03 b1 70 a0 00 00 + + +# Server is now ready for new statements +Client: RUN "RETURN 1 AS num" {} + + 00 13 b2 10 8f 52 45 54 55 52 4e 20 31 20 41 53 + 20 6e 75 6d a0 00 00 + +Server: SUCCESS { fields: ['num'] } + + 00 0f b1 70 a1 86 66 69 65 6c 64 73 91 83 6e 75 + 6d 00 00 +---- \ No newline at end of file diff --git a/community/ndp/v1-docs/src/docs/dev/images/failure-ack.png b/community/ndp/v1-docs/src/docs/dev/images/failure-ack.png new file mode 100644 index 0000000000000..d6a64a9a88de3 Binary files /dev/null and b/community/ndp/v1-docs/src/docs/dev/images/failure-ack.png differ diff --git a/community/ndp/v1-docs/src/docs/dev/images/failure-optimistic.png b/community/ndp/v1-docs/src/docs/dev/images/failure-optimistic.png new file mode 100644 index 0000000000000..b34e8f167d89d Binary files /dev/null and b/community/ndp/v1-docs/src/docs/dev/images/failure-optimistic.png differ diff --git a/community/ndp/v1-docs/src/docs/dev/images/packstream-integers.png b/community/ndp/v1-docs/src/docs/dev/images/packstream-integers.png new file mode 100644 index 0000000000000..8378b4e472ded Binary files /dev/null and b/community/ndp/v1-docs/src/docs/dev/images/packstream-integers.png differ diff --git a/community/ndp/v1-docs/src/docs/dev/images/packstream-sized.png b/community/ndp/v1-docs/src/docs/dev/images/packstream-sized.png new file mode 100644 index 0000000000000..368a7f8f261a4 Binary files /dev/null and b/community/ndp/v1-docs/src/docs/dev/images/packstream-sized.png differ diff --git a/community/ndp/v1-docs/src/docs/dev/images/packstream-text.png b/community/ndp/v1-docs/src/docs/dev/images/packstream-text.png new file mode 100644 index 0000000000000..293715e708d39 Binary files /dev/null and b/community/ndp/v1-docs/src/docs/dev/images/packstream-text.png differ diff --git a/community/ndp/v1-docs/src/docs/dev/images/packstream-tinytext.png b/community/ndp/v1-docs/src/docs/dev/images/packstream-tinytext.png new file mode 100644 index 0000000000000..d31e8435880d0 Binary files /dev/null and b/community/ndp/v1-docs/src/docs/dev/images/packstream-tinytext.png differ diff --git a/community/ndp/v1-docs/src/docs/dev/images/pipelining.png b/community/ndp/v1-docs/src/docs/dev/images/pipelining.png new file mode 100644 index 0000000000000..4ae374699cc40 Binary files /dev/null and b/community/ndp/v1-docs/src/docs/dev/images/pipelining.png differ diff --git a/community/ndp/v1-docs/src/docs/dev/images/simple-exchange.png b/community/ndp/v1-docs/src/docs/dev/images/simple-exchange.png new file mode 100644 index 0000000000000..3a60e1787129b Binary files /dev/null and b/community/ndp/v1-docs/src/docs/dev/images/simple-exchange.png differ diff --git a/community/ndp/v1-docs/src/docs/dev/images/typical-interaction.png b/community/ndp/v1-docs/src/docs/dev/images/typical-interaction.png new file mode 100644 index 0000000000000..b820a18c44428 Binary files /dev/null and b/community/ndp/v1-docs/src/docs/dev/images/typical-interaction.png differ diff --git a/community/ndp/v1-docs/src/docs/dev/index.asciidoc b/community/ndp/v1-docs/src/docs/dev/index.asciidoc new file mode 100644 index 0000000000000..79aa79b4fc3e6 --- /dev/null +++ b/community/ndp/v1-docs/src/docs/dev/index.asciidoc @@ -0,0 +1,27 @@ +[[ndp]] +Neo4j Data Protocol, Version 1 +============================== + +WARNING: This component is in active development and *will* change. + +== Overview + +This section describes the Neo4j Data Protocol, version 1. +It is written primarily for those implementing client drivers as well as those who want to understand the low-level communication details of such interactions. + +Neo4j supports a client-server protocol where messages are exchanged between a client who drives an interaction and a server which processes and responds to client requests. +Every exchange of messages is initiated by the client with one or more request messages; in turn these requests are consumed by the server and corresponding response messages are returned. + +The diagram below illustrates a typical interaction including the order of messages sent and the life spans of the session and job (which are described in more detail in other chapters). + +image:typical-interaction.png[] + +The protocol is divided into two layers - <> and <>. + +include::transport.asciidoc[] + +include::messaging.asciidoc[] + +include::serialization.asciidoc[] + +include::examples.asciidoc[] \ No newline at end of file diff --git a/community/ndp/v1-docs/src/docs/dev/messaging.asciidoc b/community/ndp/v1-docs/src/docs/dev/messaging.asciidoc new file mode 100644 index 0000000000000..71f368e4e0792 --- /dev/null +++ b/community/ndp/v1-docs/src/docs/dev/messaging.asciidoc @@ -0,0 +1,259 @@ +[[ndp-messaging]] +== Messaging layer + +This section discusses the semantic meaning and layout of protocol messages. +For details on how Neo4j types are represented in binary form, see <>. + +Clients may send request messages at any time. +Clients may <> requests, sending multiple requests together. + +Servers must fully respond to each request before the next request is processed and processing must be done in the same order requests arrived in for a given session. + +Servers must ignore messages sent by the client after a failure occurs on the server, until the client has acknowledged the failure. See Failure & Acknowledgement. + +For each request message sent, clients should expect to receive zero or more detail messages followed by exactly one summary message. +The detail messages deliver the response content, while a summary message denotes the end of the response and any response metadata. +Note that these are classifications of message, not specific message types. +For example, `RECORD` messages are classed as detail messages and `SUCCESS` messages as summary messages. + +The diagrams below illustrates a basic exchange wherein the client sends a fictional request message and receives a series of response messages. + +image:simple-exchange.png[] + +[[ndp-messaging-pipelining]] +=== Pipelining + +The client is not required to wait for a response before sending more messages. +Sending multiple messages together like this is called pipelining: + +image:pipelining.png[] + +For performance reasons, it is recommended that clients use pipelining as much as possible. +Through pipelining, multiple messages can be transmitted together in the same network package, significantly reducing latency and increasing throughput. + +TIP: A common technique is to buffer outgoing messages on the client until the last possible moment, such as when a +commit is issued or a result is read by the application, and then sending all messages in the buffer together. + +=== Failure handling + +Because the protocol leverages pipelining, the client and the server need to agree on what happens when a failure +occurs, otherwise messages that were sent assuming no failure would occur might have unintended effects. + +When requests fail on the server, this is signalled to the client via a `FAILURE` message. +After a failure, the server will not accept any messages it recieves until the client sends an `ACK_FAILURE` message. +The server should issue an IGNORED message in response to any other kind of message sent by the client until an `ACK_FAILURE` has been received. + +The diagram below illustrates a typical flow involving `ACK_FAILURE` messages: + +image:failure-ack.png[] + +Here, the original failure is acknowledged immediately by the client, allowing the subsequent RUN to be actioned as expected. + +This second diagram shows a sequence where a pair of request messages are sent together: + +image:failure-optimistic.png[] + +Here, the client optimistically sends a pair of messages. The first of these fails and the second is consequently +`IGNORED`. +Once the client acknowledges the failure, it is then able to resend a corrected RUN message. + + +[[ndp-message-structs]] +=== Messages + +Protocol messages are represented as <>. + +==== RUN + +The `RUN` message is a client message that is used to start a new job on the server. It has the following structure: + +[source,ndp_message_struct] +---- +RunMessage (signature=0x10) { + Text statement + Map parameters +} +---- + +On receipt of a `RUN` message, a server should start a new job by executing the statement with the parameters supplied. +If successful, the subsequent response should consist of a single `SUCCESS` message; if not, a `FAILURE` response should + be sent instead. +A successful job will always produce a result stream which must then be explicitly consumed, even if empty. + +In the case where a previous result stream has not yet been fully consumed (via `PULL_ALL` or `DISCARD_ALL`), an +attempt to `RUN` a new job should trigger a `FAILURE` response. + +If an unacknowledged failure is pending from a previous exchange, the server must immediately respond with a single +`IGNORED` message and take no further action. + +.Example +[source,ndp_packstream_type] +---- +Value: RUN "RETURN 1 AS num" {} + +B2 10 8F 52 45 54 55 52 4E 20 31 20 41 53 20 6E 75 6D A0 +---- + +==== DISCARD_ALL + +The `DISCARD_ALL` message is a client message that is used to discard all remaining items from the active result +stream. It has the following structure: + +[source,ndp_message_struct] +---- +DiscardAllMessage (signature=0x2F) { +} +---- + +On receipt of a `DISCARD_ALL` message, a server should dispose of all remaining items from the active result stream. +If no result stream is currently active, the server should respond with a single FAILURE message. + +If an unacknowledged failure is pending from a previous exchange, the server must immediately respond with a single IGNORED message and take no further action. + +.Example +[source,ndp_packstream_type] +---- +Value: DISCARD_ALL + +B0 7E +---- + +==== PULL_ALL +The `PULL_ALL` message is a client message that is used to retrieve all remaining items from the active result stream. +It has the following structure: + +[source,ndp_message_struct] +---- +PullAllMessage (signature=0x3F) { +} +---- + +On receipt of a `PULL_ALL` message, a server should send all remaining result data items to the client, each in a +single `RECORD` message, followed by a single `SUCCESS` message which may contain summary information on the data items +sent. +If an error is encountered, the server must instead send a `FAILURE` message and discard all remaining data items. + +If an unacknowledged failure is pending from a previous exchange, the server must immediately respond with a single IGNORED message and take no further action. + +.Example +[source,ndp_packstream_type] +---- +Value: PULL_ALL + +B0 3F +---- + +==== ACK_FAILURE + +The `ACK_FAILURE` message is a client message that is used to signal that a client has acknowledged a previous `FAILURE` +. It has the following structure: + +[source,ndp_message_struct] +---- +AcknowledgeFailureMessage (signature=0x0F) { +} +---- + +On receipt of an `ACK_FAILURE` message, a server should clear any pending failure state and respond with a single +`SUCCESS` message. If no such failure state is pending, a FAILURE message should be sent instead. + +An `ACK_FAILURE` should never be ignored. + +.Example +[source,ndp_packstream_type] +---- +Value: ACK_FAILURE + +B0 0F +---- + +==== RECORD + +The `RECORD` message is a server detail message used to deliver data from the server to the client. Each record +message contains a single List, which in turn contains the fields of the record in order. It has the following structure: + +[source,ndp_message_struct] +---- +RecordMessage (signature=0x71) { + List fields +} +---- + +.Example +[source,ndp_packstream_type] +---- +Value: RECORD [1,2,3] + +B1 71 93 01 02 03 +---- + +==== SUCCESS + +The `SUCCESS` message is a server summary message that is used to signal that a corresponding client message has been +received and actioned as intended. It has the following structure: + +[source,ndp_message_struct] +---- +SuccessMessage (signature=0x70) { + Map metadata +} +---- + +.Example +[source,ndp_packstream_type] +---- +Value: SUCCESS {fields:["name", "age"]} + +B1 70 A1 86 66 69 65 6C 64 73 92 84 6E 61 6D 65 +83 61 67 65 +---- + +==== FAILURE + +The `FAILURE` message is a server summary message that is used to signal that a corresponding client message has +encountered an error while being actioned. It has the following structure: + +[source,ndp_message_struct] +---- +FailureMessage (signature=0x7F) { + Map metadata +} +---- + +`FAILURE` messages contain metadata which describe the detail of the primary failure that has occurred. This metadata +is a simple map containing a code and a message. These codes map to the standard Neo4j status codes. + +.Example +[source,ndp_packstream_type] +---- +Value: FAILURE {code:"Neo.ClientError.Statement.InvalidSyntax", message:"Invalid syntax."} + +B1 7F A2 84 63 6F 64 65 D0 27 4E 65 6F 2E 43 6C +69 65 6E 74 45 72 72 6F 72 2E 53 74 61 74 65 6D +65 6E 74 2E 49 6E 76 61 6C 69 64 53 79 6E 74 61 +78 87 6D 65 73 73 61 67 65 8F 49 6E 76 61 6C 69 +64 20 73 79 6E 74 61 78 2E +---- + +==== IGNORED + +The `IGNORED` message is a server summary message that is used to signal that a corresponding client message has been +ignored and not actioned. It has the following structure: + +[source,ndp_message_struct] +---- +IgnoredMessage (signature=0x7E) { + Map metadata +} +---- + +A client message will be ignored if an earlier failure has not yet been acknowledged by the client via an `ACK_FAILURE` message. +This will occur if the client optimistically sends a group of messages, one of which fails during execution: all subsequent messages in that group will then be ignored. + +.Example +[source,ndp_packstream_type] +---- +Value: IGNORED + +B0 7E +---- \ No newline at end of file diff --git a/community/ndp/v1-docs/src/docs/dev/serialization.asciidoc b/community/ndp/v1-docs/src/docs/dev/serialization.asciidoc new file mode 100644 index 0000000000000..8bc7f1b6bdd0e --- /dev/null +++ b/community/ndp/v1-docs/src/docs/dev/serialization.asciidoc @@ -0,0 +1,477 @@ +== Message Serialization +=== Overview + +This section discusses how messages and the Neo4j type system are represented by the protocol using a custom binary serialization format. + +For details on the layout and meaning of specific messages, see <>. + +[[ndp-type-system-mapping]] +.Types overview +[options="header",name="value-translation-table"] +|======================= +|Type |Description +|<> |Represents the absence of a value +|<> |Boolean true or false +|<> |64-bit signed integer +|<> |64-bit floating point number +|<> |Unicode text string +|<> |Ordered collection of values +|<> |Unordered, keyed collection of values +|<> |An opaque identifier that references a Node, a Relationship, or any other object Cypher wants to give the user a back-reference to. +|<> |The best place in the world to put your data +|<> |A directed, typed connection between two nodes. Each relationship may have properties and always has an identity +|<> |The record of a directed walk through the graph, a sequence of zro or more segments*. A path with zero segments consists of a single node. +|======================= + +NOTE: *A segment is the record of a single step traversal through a graph, encompassing a start node, a relationship +traversed either forwards or backwards and an end node. + +==== Why a custom format? + +This format draws upon some of the best serialization formats available, but tailors it for the needs of Neo4j. + +Specifically, the Neo4j Serialization format has the following features: + +* Mixed unstructured and structured data, meaning we can easily represent self-describing data structures, while also leveraging schema-defined ones where a well-known structure exists. +* No versioning, meaning lower overhead and lower complexity than formats with versioned messages. +Versioning is instead handled by the wrapping transport protocol. +* Efficient streaming serialization and deserialization of large (no size boundary) messages. + +=== Markers +Every value consists of at least one marker byte. +The marker contains information on data type as well as direct or indirect size information for those types that require it. +How that size information is encoded varies by marker type. + +Some values, such as `true`, can be encoded within a single marker byte and many small integers (specifically between -16 and +127) are also encoded within a single byte. + +A number of marker bytes are reserved for future expansion of the format itself. +These bytes should not be used, and encountering them in a stream should cause an error. + +=== Sized Values +Some value types require variable length representations and, as such, have their size encoded. +These values generally consist of a single marker byte followed by a size followed by the data content itself. +Here, the marker denotes both type and scale and therefore determines the number of bytes used to represent the size of the data. +The size itself is either an 8-bit, 16-bit or 32-bit big-endian unsigned integer. + +The diagram below illustrates the general layout for a sized value, here with a 16-bit size: + +image:packstream-sized.png[] + +[[ndp-packstream-null]] +=== Null +Null is always encoded using the single marker byte `0xC0`. + +.Absence of value - null +[source,ndp_packstream_type] +---- +Value: null + +C0 +---- + +[[ndp-packstream-booleans]] +=== Booleans +Boolean values are encoded within a single marker byte, using `0xC3` to denote true and `0xC2` to denote false. + +.Boolean true +[source,ndp_packstream_type] +---- +Value: true + +C3 +---- + +.Boolean false +[source,ndp_packstream_type] +---- +Value: false + +C2 +---- + +[[ndp-packstream-floats]] +=== Floating Point Numbers +These are double-precision floating points for approximations of any number, notably for representing fractions and decimal numbers. +Floats are encoded as a single `0xC1` marker byte followed by 8 bytes, formatted according to the IEEE 754 floating-point "double format" bit layout. + +Bit 63 (the bit that is selected by the mask `0x8000000000000000`) represents the sign of the number. + +Bits 62-52 (the bits that are selected by the mask `0x7ff0000000000000`) represent the exponent. + +Bits 51-0 (the bits that are selected by the mask `0x000fffffffffffff`) represent the significand (sometimes called the mantissa) of the number. + +.Simple floating point +[source,ndp_packstream_type] +---- +Value: 1.1 + +C1 3F F1 99 99 99 99 99 9A +---- + +.Negative floating point +[source,ndp_packstream_type] +---- +Value: -1.1 + +C1 BF F1 99 99 99 99 99 9A +---- + +[[ndp-packstream-ints]] +=== Integers +Integer values occupy either 1, 2, 3, 5 or 9 bytes depending on magnitude and are stored as big-endian signed values. +Several markers are designated specifically as `TINY_INT` values and can therefore be used to pass a small number in a single byte. +These markers can be identified by a zero high-order bit or by a high-order nibble containing only ones. + +The available encodings are illustrated below and each shows a valid representation for the decimal value 42, with marker bytes in green: + +image:packstream-integers.png[] + +Note that while encoding small numbers in wider formats is supported, it is generally recommended to use the most compact representation possible. +The following table shows the optimal representation for every possible integer: + +.Simple integer +[source,ndp_packstream_type] +---- +Value: 1 + +01 +---- + +.Min integer +[source,ndp_packstream_type] +---- +Value: -9223372036854775808 + +CB 80 00 00 00 00 00 00 00 +---- + +.Max integer +[source,ndp_packstream_type] +---- +Value: 9223372036854775807 + +CB 7F FF FF FF FF FF FF FF +---- + +.Suggested integer representations +[options="header",name="packstream-integer-range-table"] +|======================= +|Range Minimum |Range Maximum |Suggested representation +|-9 223 372 036 854 775 808 |-2 147 483 649 |`INT_64` +|-2 147 483 648 |-32 769 |`INT_32` +|-32 768 |-129 |`INT_16` +|-128 |-17 |`INT_8` +|-16 |+127 |`TINY_INT` +|+128 |+32 767 |`INT_16` +|+32 768 |+2 147 483 647 |`INT_32` +|+2 147 483 648 |+9 223 372 036 854 775 807 |`INT_64` +|======================= + +[[ndp-packstream-text]] +=== Text +Text data is represented as UTF-8 encoded binary data. +Note that sizes used for text are the byte counts of the UTF-8 encoded data, not the character count of the original text. + +.Text markers +[options="header",name="packstream-text-marker-table"] +|======================= +|Marker |Size |Maximum data size +|`0x80`..`0x8F` |contained within low-order nibble of marker |15 bytes +|`0xD0` |8-bit big-endian unsigned integer |255 bytes +|`0xD1` |16-bit big-endian unsigned integer |65 535 bytes +|`0xD2` |32-bit big-endian unsigned integer |4 294 967 295 bytes +|======================= + +==== Tiny Text Strings & Empty Text Strings +For encoded text containing fewer than 16 bytes, including empty strings, the marker byte should contain the high-order nibble `1000` followed by a low-order nibble containing the size. +The encoded data then immediately follows the marker. +The example below shows how the string "Hello" would be represented: + +// TODO: Convert this to a code-segment that can be tested +image:packstream-tinytext.png[] + +==== Regular Text Strings +For encoded text containing 16 bytes or more, the marker `0xD0`, `0xD1` or `0xD2` should be used, depending on scale. +This marker is followed by the size and the UTF-8 encoded data as in the example below: + +// TODO: Convert this to a code-segment that can be tested +image:packstream-text.png[] + +==== Examples + +.Tiny text +[source,ndp_packstream_type] +---- +Value: "a" + +81 61 +---- + +.Regular text +[source,ndp_packstream_type] +---- +Value: "abcdefghijklmonpqrstuvwxyz" + +D0 1A 61 62 63 64 65 66 67 68 69 6A 6B 6C 6D 6F +6E 70 71 72 73 74 75 76 77 78 79 7A +---- + +.Text with special characters +[source,ndp_packstream_type] +---- +Value: "En å flöt över ängen" + +D0 18 45 6E 20 C3 A5 20 66 6C C3 B6 74 20 C3 B6 +76 65 72 20 C3 A4 6E 67 65 6E +---- + +[[ndp-packstream-lists]] +=== Lists +Lists are sized heterogeneous sequences of values and permit a mixture of types within the same list. +The size of a list denotes the number of items within that list, not the total packed byte size. +The markers used to denote a list are described in the table below: + +.List markers +[options="header",name="packstream-list-marker-table"] +|======================= +|Marker |Size |Maximum list size +|`0x90`..`0x9F` |contained within low-order nibble of marker |15 bytes +|`0xD4` |8-bit big-endian unsigned integer |255 items +|`0xD5` |16-bit big-endian unsigned integer |65 535 items +|`0xD6` |32-bit big-endian unsigned integer |4 294 967 295 items +|======================= + +==== Tiny Lists & Empty Lists +For lists containing fewer than 16 items, including empty lists, the marker byte should contain the high-order nibble `1001` followed by a low-order nibble containing the size. +The items within the list are then serialised in order immediately after the marker. + +==== Regular Lists +For lists containing 16 items or more, the marker `0xD4`, `0xD5` or `0xD6` should be used, depending on scale. +This marker is followed by the size and list items, serialized in order. + +==== Examples + +.Empty list +[source,ndp_packstream_type] +---- +Value: [] + +90 +---- + +.Tiny list +[source,ndp_packstream_type] +---- +Value: [1,2,3] + +93 01 02 03 +---- + +.Regular list +[source,ndp_packstream_type] +---- +Value: [1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6,7,8,9,0] + +D4 14 01 02 03 04 05 06 07 08 09 00 01 02 03 04 +05 06 07 08 09 00 +---- + +[[ndp-packstream-maps]] +=== Maps +Maps are sized sequences of pairs of values and permit a mixture of types within the same map. +The size of a map denotes the number of pairs within that map, not the total packed byte size. +The markers used to denote a map are described in the table below: + +.Map markers +[options="header",name="packstream-map-marker-table"] +|======================= +|Marker |Size |Maximum map size +|`0xA0`..`0xAF` |contained within low-order nibble of marker |15 key-value pairs +|`0xD8` |8-bit big-endian unsigned integer |255 key-value pairs +|`0xD9` |16-bit big-endian unsigned integer |65 535 key-value pairs +|`0xDA` |32-bit big-endian unsigned integer |4 294 967 295 key-value pairs +|======================= + +==== Tiny Maps & Empty Maps +For maps containing fewer than 16 key-value pairs, including empty maps, the marker byte should contain the high-order nibble `1010` followed by a low-order nibble containing the size. +The items within the map are then serialised in key-value-key-value order immediately after the marker. + +==== Regular Maps +For maps containing 16 pairs or more, the marker `0xD8`, `0xD9` or `0xDA` should be used, depending on scale. +This marker is followed by the size and map entries, serialised in key-value-key-value order. + +==== Examples + +.Empty map +[source,ndp_packstream_type] +---- +Value: {} + +A0 +---- + +.Tiny map +[source,ndp_packstream_type] +---- +Value: {a:1} + +A1 81 61 01 +---- + +.Regular map +[source,ndp_packstream_type] +---- +Value: {a:1,b:1,c:3,d:4,e:5,f:6,g:7,h:8,i:9,j:0,k:1,l:2,m:3,n:4,o:5,p:6} + +D8 10 81 61 01 81 62 01 81 63 03 81 64 04 81 65 +05 81 66 06 81 67 07 81 68 08 81 69 09 81 6A 00 +81 6B 01 81 6C 02 81 6D 03 81 6E 04 81 6F 05 81 +70 06 +---- + +[[ndp-packstream-structures]] +=== Structures +Structures represent composite values and consist, beyond the marker, of a single byte signature followed by a sequence of fields, each an individual value. +The size of a structure is measured as the number of fields, not the total packed byte size. +The markers used to denote a structure are described in the table below: + +.Structure markers +[options="header",name="packstream-structure-marker-table"] +|======================= +|Marker |Size |Maximum structure size +|`0xB0`..`0xBF` |contained within low-order nibble of marker |15 fields +|`0xDC` |8-bit big-endian unsigned integer |255 fields +|`0xDD` |16-bit big-endian unsigned integer |65 535 fields +|======================= + +==== Signature +The signature byte is used to identify the type or class of the structure. +Refer to the <> and <> for structures used in the protocol. + +Signature bytes may hold any value between 0 and +127. Bytes with the high bit set are reserved for future expansion. + +==== Tiny Structures +For structures containing fewer than 16 fields, the marker byte should contain the high-order nibble `1011` followed by a low-order nibble containing the size. +The marker is immediately followed by the signature byte and the field values. + +==== Regular Structures +For structures containing 16 fields or more, the marker `0xDC` or `0xDD` should be used, depending on scale. +This marker is followed by the size, the signature byte and the actual fields, serialised in order. + +==== Examples + +Assuming a struct with the signature `0x01` and three fields with values 1,2,3: + +.Tiny structure +[source,ndp_packstream_type] +---- +Value: Struct (signature=0x01) { 1,2,3 } + +B3 01 01 02 03 +---- + +.Regular structure +[source,ndp_packstream_type] +---- +Value: Struct (signature=0x01) { 1,2,3,4,5,6,7,8,9,0,1,2,3,4,5,6 } + +DC 10 01 01 02 03 04 05 06 07 08 09 00 01 02 03 +04 05 06 +---- + +[[ndp-value-structs]] +==== Neo4j value structs + +Neo4j values that are represented as <>. + +[[ndp-value-identitystruct]] +===== Identity +An identity is an opaque unique handle that references a specific graph entity. The general serialised structure is as follows: + +[source,ndp_value_struct] +---- +Identity (signature=0x49) { + Text identityData +} +---- + +[[ndp-value-nodestruct]] +===== Node +A Node represents a node from a Neo4j graph and consists of a unique identifier (within the scope of its origin graph), a list of labels and a map of properties. The general serialised structure is as follows: + +[source,ndp_value_struct] +---- +Node (signature=0x4E) { + Identity identity + List labels + Map properties +} +---- + +[[ndp-value-relstruct]] +===== Relationship +A Relationship represents a relationship from a Neo4j graph and consists of a unique identifier (within the scope of its origin graph), identifiers for the start and end nodes of that relationship, a type and a map of properties. The general serialised structure is as follows: + +[source,ndp_value_struct] +---- +Relationship (signature=0x52) { + Identity identity + Identity startNode + Identity endNode + Identity type + Map properties +} +---- + +[[ndp-value-pathstruct]] +===== Path +A Path consists of a list of alternating nodes and relationships, always starting and ending with a node. The general serialised structure is as follows: + +[source,ndp_value_struct] +---- +Path (signature=0x50) { + List entities +} +---- + +=== Marker table + +These are all the marker bytes: + +[[ndp-packstream-markers]] +.Marker table +[options="header",name="ndp-packstream-marker-table"] +|======================= +|Marker |Binary |Type |Description +|`0x00`..`0x7F` |`0xxxxxxx` |`+TINY_INT` |Integer 0 to 127 +|`0x80`..`0x8F` |`1000xxxx` |`TINY_TEXT` |UTF-8 encoded text string (fewer than 24 bytes) +|`0x90`..`0x9F` |`1001xxxx` |`TINY_LIST` |List (fewer than 24 items) +|`0xA0`..`0xAF` |`1010xxxx` |`TINY_MAP` |Map (fewer than 24 key-value pairs) +|`0xB0`..`0xBF` |`1011xxxx` |`TINY_STRUCT` |Structure (fewer than 24 fields) +|`0xC0` |`11000000` |`NULL` |Null +|`0xC1` |`11000001` |`FLOAT_64` |64-bit floating point number (double) +|`0xC2` |`11000010` |`FALSE` |Boolean false +|`0xC3` |`11000011` |`TRUE` |Boolean true +|`0xC4`..`0xC7` |`110001xx` | |Reserved +|`0xC8` |`11001000` |`INT_8` |8-bit signed integer +|`0xC9` |`11001001` |`INT_16` |16-bit signed integer +|`0xCA` |`11001010` |`INT_32` |32-bit signed integer +|`0xCB` |`11001011` |`INT_64` |64-bit signed integer +|`0xCC`..`0xCF` |`11001100` | |Reserved +|`0xD0` |`11010000` |`TEXT_8` |UTF-8 encoded text string (fewer than 28 bytes) +|`0xD1` |`11010001` |`TEXT_16` |UTF-8 encoded text string (fewer than 216 bytes) +|`0xD2` |`11010010` |`TEXT_32` |UTF-8 encoded text string (fewer than 232 bytes) +|`0xD3` |`11010011` | |Reserved +|`0xD4` |`11010100` |`LIST_8` |List (fewer than 28 items) +|`0xD5` |`11010101` |`LIST_16` |List (fewer than 216 items) +|`0xD6` |`11010110` |`LIST_32` |List (fewer than 232 items) +|`0xD7` |`11010111` | |Reserved +|`0xD8` |`11011000` |`MAP_8` |Map (fewer than 28 key-value pairs) +|`0xD9` |`11011001` |`MAP_16` |Map (fewer than 216 key-value pairs) +|`0xDA` |`11011010` |`MAP_32` |Map (fewer than 232 key-value pairs) +|`0xDB` |`11011011` | |Reserved +|`0xDC` |`11011100` |`STRUCT_8` |Structure (fewer than 28 fields) +|`0xDD` |`11011101` |`STRUCT_16` |Structure (fewer than 216 fields) +|`0xDE`..`0xEF` |`1110xxxx` | |Reserved +|`0xF0`..`0xFF` |`1111xxxx` |`-TINY_INT` |Integer -1 to -16 +|======================= diff --git a/community/ndp/v1-docs/src/docs/dev/transport.asciidoc b/community/ndp/v1-docs/src/docs/dev/transport.asciidoc new file mode 100644 index 0000000000000..916b144c0cd42 --- /dev/null +++ b/community/ndp/v1-docs/src/docs/dev/transport.asciidoc @@ -0,0 +1,109 @@ +[[ndp-transport]] +== Transport layer +The protocol uses a TCP transport for sending and receiving messages. +The transport protocol is responsible for: + +* Negotiating Neo4j protocol version +* Establishing and terminating sessions +* Routing messages from clients to specific sessions and back + +The transport is versioned along with the rest of the data protocol - however, versioning applies only after a version negotiation handshake takes place. +Before that handshake, the protocol will remain stable across versions to support backwards compatibility. + +=== Sessions + +A connection to the server via a transport is called a session. +Each session with the server is isolated and the server tracks the current state based on the requests and responses delivered within that session. + +Neo4j uses 'sticky sessions', which means that, in a database cluster, each session is tied to one specific Neo4j instance. + +=== Connecting + +To initialize a new session, the client connects using a regular TCP socket to the address the server has been configured to use. +Once the socket is connected, a handshake to establish which protocol version to use takes place. + +In the handshake, the client proposes up to four protocol versions it supports, in order of preference. +The proposal is always represented as four 32-bit unsigned big-endian integers. +Each integer represents a proposed protocol version to use, or zero (`00 00 00 00`) for "none". + +The server will respond with a single 32-bit unsigned big-endian integer representing the chosen protocol, this will always be the highest-priority protocol version the server supports. +If none of the proposed protocols are supported, the server responds with zero (`00 00 00 00`) and closes the connection. + +.Initial handshake +[source,ndp_exchange] +---- +Client: +Client: 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 + Version 1 None None None + +Server: 00 00 00 01 + Choose + version 1 +---- + +.No supported version +[source,ndp_exchange] +---- +Client: +Client: 00 00 00 06 00 00 00 00 00 00 00 00 00 00 00 00 + Version 6 None None None + +Server: 00 00 00 00 + None + supported + +Server: +---- + +=== Message framing + +The transport protocol uses a framing layer to wrap messages. + +Each message is transferred as one or more `chunks` of data. +Each chunk starts with a two-byte header, an unsigned big-endian 16-bit integer, representing the size of the chunk. +The header is not counted towards this size. +A message can be divided across multiple chunks, allowing client and server alike to efficiently transfer messages larger than their network buffers. + +Each message ends with two bytes with the value `00 00`, these are not counted towards the chunk length. + +.A message in one chunk +[source,ndp_chunking_example] +---- +Chunk size: 16 +Message data: 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F + +00 10 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F 00 00 +chunk | Message | End +header | Data | Marker +---- + +.A message split in two chunks +[source,ndp_chunking_example] +---- +Chunk size: 8 +Message data: 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F + +00 08 00 01 02 03 04 05 06 07 00 08 08 09 0A 0B 0C 0D 0E 0F 00 00 +chunk1 | Message | chunk2 | Message | End +header | Data | header | Data | Marker +---- + +.Two messages +[source,ndp_chunking_example] +---- +Chunk size: 16 +Message 1 data: 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F +Message 2 data: 0F 0E 0D 0C 0B 0A 09 08 + +00 10 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F 00 00 +chunk | Message 1 | End +header | Data | Marker + +00 08 0F 0E 0D 0C 0B 0A 09 08 00 00 +chunk | Message 2 | End +header | Data | Marker +---- + +=== Disconnecting + +To end a session, the client simply closes the TCP socket. diff --git a/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/Chunker.java b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/Chunker.java new file mode 100644 index 0000000000000..d5893f5d1b63e --- /dev/null +++ b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/Chunker.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.ndp.docs.v1; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.UnpooledByteBufAllocator; +import io.netty.channel.ChannelHandlerContext; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.neo4j.ndp.transport.socket.ChunkedOutput; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** Helper to chunk up serialized data for testing */ +public class Chunker +{ + public static byte[] chunk( int maxChunkSize, byte[][] messages ) throws IOException + { + ChunkedOutput out = new ChunkedOutput( maxChunkSize + 2 /* for chunk header */ ); + final ByteBuffer outputBuffer = ByteBuffer.allocate( 512 ); + + ChannelHandlerContext ch = mock( ChannelHandlerContext.class ); + when( ch.alloc() ).thenReturn( UnpooledByteBufAllocator.DEFAULT ); + when( ch.writeAndFlush( any() ) ).then( new Answer() + { + @Override + public Object answer( InvocationOnMock inv ) throws Throwable + { + ByteBuf buf = (ByteBuf) inv.getArguments()[0]; + outputBuffer.limit( outputBuffer.position() + buf.readableBytes() ); + buf.readBytes( outputBuffer ); + return null; + } + } ); + out.setTargetChannel( ch ); + + for ( byte[] message : messages ) + { + out.put( message, 0, message.length ); + out.messageBoundaryHook().run(); + } + out.flush(); + + byte[] bytes = new byte[outputBuffer.limit()]; + outputBuffer.position( 0 ); + outputBuffer.get( bytes ); + return bytes; + } +} diff --git a/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocExchangeExample.java b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocExchangeExample.java new file mode 100644 index 0000000000000..e8354b1b2c02f --- /dev/null +++ b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocExchangeExample.java @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.ndp.docs.v1; + +import org.jsoup.nodes.Element; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +import org.neo4j.function.Function; +import org.neo4j.kernel.impl.util.Codecs; + +/** + * Client: + * Client: 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 + * Version 1 None None None + *

+ * Server: 00 00 00 01 + * Choose + * version 1 + */ +public class DocExchangeExample implements Iterable +{ + public static Function exchange_example + = new Function() + { + @Override + public DocExchangeExample apply( Element s ) throws RuntimeException + { + return new DocExchangeExample( s.text() ); + } + }; + + public enum Type + { + CONNECT, + DISCONNECT, + SEND + } + + public class Event + { + + private final String from; + private final Type type; + private final byte[] payload; + private final String message; + + public Event( String from, Type type ) + { + this( from, type, new byte[0], type.name() ); + } + + public Event( String from, Type type, byte[] payload, String message ) + { + this.from = from; + this.type = type; + this.payload = payload; + this.message = message; + } + + public String from() + { + return from; + } + + public Type type() + { + return type; + } + + public byte[] payload() + { + return payload; + } + + public String humanReadableMessage() + { + return message; + } + + @Override + public String toString() + { + return "Event{" + + "from='" + from + '\'' + + ", type=" + type + + ", payload=" + Arrays.toString( payload ) + + '}'; + } + + public boolean hasHumanReadableValue() + { + return message.trim().length() > 0; + } + } + + private final List events = new ArrayList<>(); + private final String raw; + + public DocExchangeExample( String raw ) + { + this.raw = raw; + + // Generally "client" or "server", but up to the spec we're parsing + String currentActor = null; + String currentPayload = ""; + + // If the example contains messaging, we parse out the message as well to ensure the serialization is correct + String currentMessage = ""; + Type type = Type.SEND; + + // Parse all the lines, breaking them into events + for ( String line : raw.split( "\n" ) ) + { + if ( line.matches( "[a-zA-Z]+\\s*:.*" ) ) + { + if ( currentActor != null ) + { + addEvent( currentActor, currentPayload, currentMessage, type ); + } + String[] parts = line.split( ":", 2 ); + currentActor = parts[0].trim(); + currentPayload = ""; + currentMessage = ""; + line = parts[1].trim(); + type = Type.SEND; + } + + if ( line.matches( "^[(RUN)|(PULL_ALL)|(DISCARD_ALL)|(RECORD)|(SUCCESS)|(FAILURE)|(ACK_FAILURE)].+$" ) ) + { + currentMessage = line; + } + else if ( line.matches( "^[a-fA-f0-9\\s]+$" ) ) + { + currentPayload += line; + } + else if ( line.equals( "" ) ) + { + type = Type.CONNECT; + } + else if ( line.equals( "" ) ) + { + type = Type.DISCONNECT; + } + } + + if ( currentActor != null ) + { + addEvent( currentActor, currentPayload, currentMessage, type ); + } + } + + private void addEvent( String actor, String payload, String currentMessage, Type type ) + { + events.add( new Event( + actor, type, Codecs.decodeHexString( payload.replace( " ", "" ) ), currentMessage ) ); + } + + @Override + public Iterator iterator() + { + return events.iterator(); + } + + @Override + public String toString() + { + return raw; + } +} diff --git a/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocSerialization.java b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocSerialization.java new file mode 100644 index 0000000000000..bdc00f744f6e8 --- /dev/null +++ b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocSerialization.java @@ -0,0 +1,265 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.ndp.docs.v1; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import org.neo4j.kernel.api.exceptions.Status; +import org.neo4j.kernel.impl.util.HexPrinter; +import org.neo4j.ndp.messaging.v1.PackStreamMessageFormatV1; +import org.neo4j.ndp.messaging.v1.RecordingByteChannel; +import org.neo4j.ndp.messaging.v1.RecordingMessageHandler; +import org.neo4j.ndp.messaging.v1.message.Message; +import org.neo4j.ndp.runtime.internal.Neo4jError; +import org.neo4j.ndp.transport.socket.ChunkedInput; +import org.neo4j.packstream.BufferedChannelOutput; +import org.neo4j.packstream.PackStream; +import org.neo4j.stream.ImmutableRecord; + +import static java.util.Arrays.asList; +import static org.neo4j.helpers.collection.MapUtil.map; +import static org.neo4j.ndp.docs.v1.Chunker.chunk; + +/** + * Takes human-readable value descriptions and packs them to binary data, and vice versa. + *

+ * Examples: + *

+ * - 1 + * - "hello, world" + * - RUN "RETURN a", {a:12} + * - SUCCESS {} + */ +public class DocSerialization +{ + public static byte[] packAndChunk( String value, int chunkSize ) throws IOException + { + return chunk( chunkSize, new byte[][]{pack( value )} ); + } + + public static byte[] pack( String value ) throws IOException + { + RecordingByteChannel ch = new RecordingByteChannel(); + PackStream.Packer packer = new PackStream.Packer( new BufferedChannelOutput( ch, 128 ) ); + PackStreamMessageFormatV1.Writer writer = new PackStreamMessageFormatV1.Writer( packer, new Runnable() + { + @Override + public void run() + { + + } + } ); + + pack( value, packer, writer ); + + packer.flush(); + return ch.getBytes(); + } + + /** + * @param value + * @param packer + * @param writer a message writer that delegates to the packer, for packing protocol messages + * @throws IOException + */ + public static void pack( String value, PackStream.Packer packer, PackStreamMessageFormatV1.Writer writer ) + throws IOException + { + // NOTE: This currently has hard-coded handling of specific messages, it did not seem worth the time + // to write a custom parser for this yet. We may want to come back and do that as the docs evolve. + if ( value.equalsIgnoreCase( "null" ) ) + { + packer.packNull(); + } + else if ( value.equalsIgnoreCase( "true" ) ) + { + packer.pack( true ); + } + else if ( value.equalsIgnoreCase( "false" ) ) + { + packer.pack( false ); + } + else if ( value.startsWith( "\"" ) ) + { + packer.pack( value.substring( 1, value.length() - 1 ) ); + } + else if ( value.startsWith( "[" ) ) + { + if ( value.equals( "[]" ) ) + { + packer.packListHeader( 0 ); + } + else + { + String[] values = value.substring( 1, value.length() - 1 ).split( "," ); + packer.packListHeader( values.length ); + for ( String s : values ) + { + pack( s, packer, writer ); + } + } + } + else if ( value.startsWith( "{" ) ) + { + if ( value.equals( "{}" ) ) + { + packer.packMapHeader( 0 ); + } + else + { + String[] pairs = value.substring( 1, value.length() - 1 ).split( "," ); + packer.packMapHeader( pairs.length ); + for ( String pair : pairs ) + { + String[] split = pair.split( ":" ); + packer.pack( + split[0] ); // Key, different from packing value because it doesn't use quotation marks + pack( split[1], packer, writer ); // Value + } + } + } + else if ( value.startsWith( "Struct" ) ) + { + DocStructExample struct = new DocStructExample( value ); + packer.packStructHeader( struct.size(), (char) struct.signature() ); + + for ( String s : struct ) + { + pack( s, packer, writer ); + } + } + else if ( value.matches( "-?[0-9]+\\.[0-9]+" ) ) + { + packer.pack( Double.parseDouble( value ) ); + } + else if ( value.matches( "-?[0-9]+" ) ) + { + packer.pack( Long.parseLong( value ) ); + } + else if ( value.equals( "DISCARD_ALL" ) ) + { + writer.handleIgnoredMessage(); + } + else if ( value.equals( "PULL_ALL" ) ) + { + writer.handlePullAllMessage(); + } + else if ( value.equals( "ACK_FAILURE" ) ) + { + writer.handleAckFailureMessage(); + } + else if ( value.equals( "IGNORED" ) ) // kiss.. + { + writer.handleIgnoredMessage(); + } + else if ( value.equals( "RUN \"RETURN 1 AS num\" {}" ) ) // kiss.. + { + writer.handleRunMessage( "RETURN 1 AS num", Collections.emptyMap() ); + } + else if ( value.equals( "RUN \"This will cause a syntax error\" {}" ) ) // kiss.. + { + writer.handleRunMessage( "This will cause a syntax error", Collections.emptyMap() ); + } + else if ( value.equals( "RECORD [1,2,3]" ) ) // kiss.. + { + writer.handleRecordMessage( new ImmutableRecord( new Object[]{1, 2, 3} ) ); + } + else if ( value.equals( "RECORD [1]" ) ) // kiss.. + { + writer.handleRecordMessage( new ImmutableRecord( new Object[]{1} ) ); + } + else if ( value.equals( "SUCCESS {fields:[\"name\", \"age\"]}" ) ) // kiss.. + { + writer.handleSuccessMessage( map( "fields", asList( "name", "age" ) ) ); + } + else if ( value.equals( "SUCCESS { fields: ['num'] }" ) ) // kiss.. + { + writer.handleSuccessMessage( map( "fields", asList( "num" ) ) ); + } + else if ( value.equals( "SUCCESS {}" ) ) // kiss.. + { + writer.handleSuccessMessage( map() ); + } + else if ( value.equals( "FAILURE {code:\"Neo.ClientError.Statement.InvalidSyntax\", " + + "message:\"Invalid syntax.\"}" ) ) // kiss.. + { + writer.handleFailureMessage( new Neo4jError( Status.Statement.InvalidSyntax, "Invalid syntax." ) ); + } + else if ( value.equals( "FAILURE {code:\"Neo.ClientError.Statement.InvalidSyntax\"," ) ) // kiss.. + { + writer.handleFailureMessage( new Neo4jError( Status.Statement.InvalidSyntax, + "Invalid input 'T': expected (line 1, column 1 (offset: 0))\n" + + "\"This will cause a syntax error\"\n" + + " ^" ) ); + } + else + { + throw new RuntimeException( "Unknown value: " + value ); + } + } + + public static List unpackChunked( byte[] data ) throws Exception + { + ChunkedInput input = new ChunkedInput(); + PackStreamMessageFormatV1.Reader reader = new PackStreamMessageFormatV1.Reader( input ); + RecordingMessageHandler messages = new RecordingMessageHandler(); + + ByteBuf buf = Unpooled.wrappedBuffer( data ); + while ( buf.readableBytes() > 0 ) + { + int chunkSize = buf.readUnsignedShort(); + if ( chunkSize > 0 ) + { + input.addChunk( buf.readSlice( chunkSize ) ); + } + else + { + reader.read( messages ); + input.clear(); + } + } + return messages.asList(); + } + + public static String normalizedHex( byte[] data ) + { + return normalizedHex( HexPrinter.hex( data ) ); + } + + /** Convert a hex string into a normalized format for string comparison */ + public static String normalizedHex( String dirtyHex ) + { + StringBuilder str = new StringBuilder( dirtyHex.replace( "\n", "" ).replace( " ", "" ) ); + int idx = str.length() - 2; + + while ( idx > 0 ) + { + str.insert( idx, " " ); + idx = idx - 2; + } + + return str.toString().toUpperCase(); + } +} diff --git a/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocSerializationExample.java b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocSerializationExample.java new file mode 100644 index 0000000000000..2889fb3fda37e --- /dev/null +++ b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocSerializationExample.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.ndp.docs.v1; + +import org.jsoup.nodes.Element; + +import java.util.HashMap; +import java.util.Map; + +import org.neo4j.function.Function; + +/** + * Value: 1.1 + *

+ * C1 3F F1 99 99 99 99 99 9A + */ +public class DocSerializationExample +{ + public static Function serialization_example + = new Function() + { + @Override + public DocSerializationExample apply( Element s ) throws RuntimeException + { + return new DocSerializationExample( s.text() ); + } + }; + + private final Map attributes = new HashMap<>(); + private final String raw; + private String serializedData = ""; + + public DocSerializationExample( String raw ) + { + this.raw = raw; + boolean readingHeaders = true; + for ( String line : raw.split( "\n" ) ) + { + if ( line.trim().equals( "" ) ) + { + // Blank line denotes header/binary data body split + readingHeaders = false; + } + else if ( readingHeaders ) + { + String[] split = line.split( ":", 2 ); + attributes.put( split[0].trim(), split[1].trim() ); + } + else + { + if ( line.matches( "^[a-fA-f0-9\\s]+$" ) ) + { + serializedData += line; + } + } + } + } + + public String attribute( String name ) + { + return attributes.get( name ); + } + + public String serializedData() + { + return serializedData; + } + + @Override + public String toString() + { + return raw; + } +} diff --git a/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocStruct.java b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocStruct.java new file mode 100644 index 0000000000000..73f7c6bf0a2a5 --- /dev/null +++ b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocStruct.java @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.ndp.docs.v1; + +import org.jsoup.nodes.Element; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.neo4j.function.Function; + +import static java.util.regex.Pattern.CASE_INSENSITIVE; +import static java.util.regex.Pattern.COMMENTS; +import static java.util.regex.Pattern.DOTALL; + +/** + * Node (signature=0x4E) { + * Identity identity + * List labels + * Map properties + * } + */ +public class DocStruct implements Iterable +{ + public static Function struct_definition = new Function() + { + @Override + public DocStruct apply( Element s ) throws RuntimeException + { + return new DocStruct( s.text() ); + } + }; + + private final String name; + private final Map attributes; + private final List fields; + private final String raw; + + public static class Field + { + private final String typeSignature; + private final String name; + + public Field( String typeSignature, String name ) + { + this.typeSignature = typeSignature; + this.name = name; + } + + @Override + public String toString() + { + return typeSignature + " " + name; + } + + public String type() + { + return typeSignature; + } + } + + public static final Pattern STRUCT_PATTERN = Pattern.compile( + "^(?[a-z]+)\\s+ # StructName \n" + + " \\( (?[^)]+) \\) \\s* # (identifier=0x01,..) \n" + + "\\{ # \\{\n" + + "\\s* (?[^\\}]+) # Type fieldName..\n" + + "\\}\\s* # \\}\n" + + "(.*)$" + + "", CASE_INSENSITIVE | COMMENTS | DOTALL ); + + public DocStruct( String structDefinition ) + { + Matcher matcher = STRUCT_PATTERN.matcher( structDefinition ); + if ( !matcher.matches() ) + { + throw new RuntimeException( "Unable to parse struct definition: \n" + structDefinition ); + } + this.raw = structDefinition; + this.name = matcher.group( "name" ); + this.attributes = parseAttributes( matcher.group( "attrs" ) ); + this.fields = parseFields( matcher.group( "fields" ) ); + } + + public String name() + { + return name; + } + + public String attribute( String key ) + { + return attributes.get( key ); + } + + @Override + public Iterator iterator() + { + return fields.iterator(); + } + + private List parseFields( String raw ) + { + List out = new LinkedList<>(); + for ( String s : raw.split( "\n" ) ) + { + String[] parts = s.trim().split( "\\s+" ); + out.add( new Field( parts[0], parts[1] ) ); + } + return out; + } + + public static Map parseAttributes( String raw ) + { + Map out = new HashMap<>(); + for ( String attr : raw.split( "," ) ) + { + String[] split = attr.split( "=" ); + out.put( split[0].trim(), split[1].trim() ); + } + return out; + } + + @Override + public String toString() + { + return raw; + } + + public int signature() + { + return Integer.parseInt( attribute( "signature" ).substring( 2 ), 16 ); + } + + public int size() + { + return fields.size(); + } +} diff --git a/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocStructExample.java b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocStructExample.java new file mode 100644 index 0000000000000..af299dd30ae1c --- /dev/null +++ b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocStructExample.java @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.ndp.docs.v1; + +import org.jsoup.nodes.Element; + +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; + +import org.neo4j.function.Function; + +/** + * Unlike {@link DocStruct}, this represents a "real" example struct with values, rather than the blueprint for one. + */ +public class DocStructExample implements Iterable +{ + public static Function struct_example = new Function() + { + @Override + public DocStructExample apply( Element s ) throws RuntimeException + { + return new DocStructExample( s.text() ); + } + }; + + private final String name; + private final Map attributes; + private final List fields; + private final String raw; + + public DocStructExample( String structDefinition ) + { + Matcher matcher = DocStruct.STRUCT_PATTERN.matcher( structDefinition ); + if ( !matcher.matches() ) + { + throw new RuntimeException( "Unable to parse struct definition: \n" + structDefinition ); + } + this.raw = structDefinition; + this.name = matcher.group( "name" ); + this.attributes = DocStruct.parseAttributes( matcher.group( "attrs" ) ); + this.fields = parseFields( matcher.group( "fields" ) ); + } + + public String name() + { + return name; + } + + public String attribute( String key ) + { + return attributes.get( key ); + } + + @Override + public Iterator iterator() + { + return fields.iterator(); + } + + private List parseFields( String raw ) + { + List out = new LinkedList<>(); + for ( String s : raw.split( "," ) ) + { + out.add( s.trim() ); + } + return out; + } + + @Override + public String toString() + { + return raw; + } + + public int signature() + { + return Integer.parseInt( attribute( "signature" ).substring( 2 ), 16 ); + } + + public int size() + { + return fields.size(); + } +} diff --git a/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocTable.java b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocTable.java new file mode 100644 index 0000000000000..9f1bcccc126ac --- /dev/null +++ b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocTable.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.ndp.docs.v1; + +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.neo4j.function.Function; + +/** A table from an asciidoc file */ +public class DocTable implements Iterable +{ + public static Function table = new Function() + { + @Override + public DocTable apply( Element s ) throws RuntimeException + { + return new DocTable( s ); + } + }; + + private final List rows; + + public static class Row + { + private final Elements cells; + + public Row( Element htmlRow ) + { + this.cells = htmlRow.select( "td" ); + } + + String get( int columnIndex ) + { + return cells.get( columnIndex ).text(); + } + + int numColumns() + { + return cells.size(); + } + } + + public DocTable( Element htmlTable ) + { + if ( htmlTable == null || !htmlTable.tagName().equals( "table" ) ) + { + throw new RuntimeException( "Expected a 'table' element, but got: " + htmlTable ); + } + this.rows = new ArrayList<>(); + + for ( Element tr : htmlTable.select( "tr" ) ) + { + if ( tr.select( "td" ).size() > 0 ) + { + rows.add( new Row( tr ) ); + } + } + } + + @Override + public Iterator iterator() + { + return rows.iterator(); + } +} diff --git a/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocsRepository.java b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocsRepository.java new file mode 100644 index 0000000000000..d5e33c86f7ae1 --- /dev/null +++ b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/DocsRepository.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.ndp.docs.v1; + +import org.asciidoctor.Asciidoctor; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import java.io.File; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.neo4j.function.Function; + +import static org.asciidoctor.OptionsBuilder.options; + +/** + * A utility for accessing asciidoc documentation for this module in a semantically meaningful way. + */ +public class DocsRepository +{ + private final static String SEP = File.separator; + private final static File docsDir = findBackwards( "community" + SEP + "ndp" + SEP + "v1-docs" + SEP + + "src" + SEP + "docs" + SEP, 5 ); + + private final Asciidoctor asciidoc; + private static final Map docCache = new HashMap<>(); + + public static DocsRepository docs() + { + return new DocsRepository(); + } + + public DocsRepository() + { + asciidoc = Asciidoctor.Factory.create(); + } + + /** + * Read exerpts from the documentation and parse them into a specified representation. + * + * @param fileName is a file name relative to the 'docs' dir of the v1-docs module, + * for instance 'dev/index.asciidoc' + * @param cssSelector is a regular css selector, for instance "code[data-lang=\"ndp-struct\"]" + * @param parser is something that converts the documentation excerpt to your desired representation + */ + public List read( String fileName, String cssSelector, Function parser ) + { + List out = new LinkedList<>(); + for ( Element el : doc( fileName ).select( cssSelector ) ) + { + out.add( parser.apply( el ) ); + } + return out; + } + + private Document doc( String fileName ) + { + File file = new File( docsDir, fileName ).getAbsoluteFile(); + if ( !file.exists() ) + { + throw new RuntimeException( "Cannot find: " + file.getAbsolutePath() ); + } + if ( !docCache.containsKey( file ) ) + { + docCache.put( file, Jsoup.parse( asciidoc.renderFile( file, options().toFile( false ) ) ) ); + } + return docCache.get( file ); + } + + private static File findBackwards( String dir, int maxDepth ) + { + if ( maxDepth == 0 ) + { + throw new RuntimeException( "Couldn't find " + dir ); + } + else if ( !new File( dir ).exists() ) + { + return findBackwards( ".." + SEP + dir, maxDepth - 1 ); + } + else + { + return new File( dir ); + } + } +} diff --git a/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/NDPFullExchangesDocTest.java b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/NDPFullExchangesDocTest.java new file mode 100644 index 0000000000000..0619992d30651 --- /dev/null +++ b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/NDPFullExchangesDocTest.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.ndp.docs.v1; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.ArrayList; +import java.util.Collection; + +import org.neo4j.ndp.transport.socket.integration.NDPConn; +import org.neo4j.ndp.transport.socket.integration.Neo4jWithSocket; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.neo4j.kernel.impl.util.HexPrinter.hex; +import static org.neo4j.ndp.docs.v1.DocExchangeExample.exchange_example; +import static org.neo4j.ndp.docs.v1.DocSerialization.packAndChunk; +import static org.neo4j.ndp.docs.v1.DocsRepository.docs; + +@RunWith( Parameterized.class ) +public class NDPFullExchangesDocTest +{ + @Rule + public Neo4jWithSocket neo4j = new Neo4jWithSocket(); + + @Parameterized.Parameter( 0 ) + public DocExchangeExample example; + + @Parameterized.Parameters( name = "{0}" ) + public static Collection documentedFullProtocolExamples() + { + Collection mappings = new ArrayList<>(); + + // Load the documented mappings + for ( DocExchangeExample ex : docs().read( + "dev/transport.asciidoc", + "code[data-lang=\"ndp_exchange\"]", + exchange_example ) ) + { + mappings.add( new Object[]{ex} ); + } + + for ( DocExchangeExample ex : docs().read( + "dev/examples.asciidoc", + "code[data-lang=\"ndp_exchange\"]", + exchange_example ) ) + { + mappings.add( new Object[]{ex} ); + } + + return mappings; + } + + @Test + public void serverShouldBehaveAsDocumented() throws Throwable + { + try ( NDPConn client = new NDPConn() ) + { + for ( DocExchangeExample.Event event : example ) + { + if ( event.from().equalsIgnoreCase( "client" ) ) + { + // Play out a client action + switch ( event.type() ) + { + case CONNECT: + client.connect( neo4j.address() ); + break; + case DISCONNECT: + client.disconnect(); + break; + case SEND: + // Ensure the documented binary representation matches the human-readable version in the docs + if ( event.hasHumanReadableValue() ) + { + assertThat( "'" + event.humanReadableMessage() + "' should serialize to the documented " + + "binary data.", + hex( event.payload() ), + equalTo( hex( packAndChunk( event.humanReadableMessage(), 64 ) ) ) ); + } + client.send( event.payload() ); + break; + default: + throw new RuntimeException( "Unknown client event: " + event.type() ); + } + } + else if ( event.from().equalsIgnoreCase( "server" ) ) + { + // Assert that the server does what the docs say + // Play out a client action + switch ( event.type() ) + { + case DISCONNECT: + // There's not really a good way to verify that the remote connection is closed, we can read and + // time out, or write perhaps, but that's buggy and racy.. not sure how to test this on this + // level. + break; + case SEND: + if ( event.hasHumanReadableValue() ) + { + assertThat( "'" + event.humanReadableMessage() + "' should serialize to the documented " + + "binary data.", + hex( event.payload() ), + equalTo( hex( packAndChunk( event.humanReadableMessage(), 512 ) ) ) ); + } + + byte[] recieved = client.recv( event.payload().length ); + + assertThat( + hex( recieved ), + equalTo( hex( event.payload() ) ) ); + break; + default: + throw new RuntimeException( "Unknown server event: " + event.type() ); + } + } + } + } + } +} diff --git a/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/NDPMessageStructsDocTest.java b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/NDPMessageStructsDocTest.java new file mode 100644 index 0000000000000..b85567d1b3e6c --- /dev/null +++ b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/NDPMessageStructsDocTest.java @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.ndp.docs.v1; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; + +import org.neo4j.ndp.messaging.v1.PackStreamMessageFormatV1; +import org.neo4j.ndp.messaging.v1.RecordingByteChannel; +import org.neo4j.ndp.messaging.v1.RecordingMessageHandler; +import org.neo4j.ndp.messaging.v1.util.ArrayByteChannel; +import org.neo4j.packstream.BufferedChannelOutput; +import org.neo4j.packstream.PackStream; + +import static org.junit.Assert.assertEquals; +import static org.neo4j.ndp.docs.v1.DocStruct.struct_definition; +import static org.neo4j.ndp.docs.v1.DocsRepository.docs; + +/** This tests that message data structures look the way we say they do */ +@RunWith( Parameterized.class ) +public class NDPMessageStructsDocTest +{ + @Parameterized.Parameter( 0 ) + public DocStruct struct; + + @Parameterized.Parameters( name = "{0}" ) + public static Collection documentedTypeMapping() + { + Collection mappings = new ArrayList<>(); + + // Load the documented mappings + for ( DocStruct struct : docs().read( + "dev/messaging.asciidoc", + "code[data-lang=\"ndp_message_struct\"]", + struct_definition ) ) + { + mappings.add( new Object[]{struct} ); + } + + return mappings; + } + + @Test + public void ensureSerializingMessagesAsDocumentedWorks() throws Throwable + { + // Given + RecordingByteChannel ch = new RecordingByteChannel(); + PackStream.Packer packer = new PackStream.Packer( new BufferedChannelOutput( ch, 128 ) ); + + // When I pack a message according to the documentation + packer.packStructHeader( struct.size(), (char) struct.signature() ); + for ( DocStruct.Field field : struct ) + { + packValueOf( field.type(), packer ); + } + packer.flush(); + + // Then it should get interpreted as the documented message + RecordingMessageHandler messages = new RecordingMessageHandler(); + PackStreamMessageFormatV1.Reader reader = new PackStreamMessageFormatV1.Reader(); + reader.reset( new ArrayByteChannel( ch.getBytes() ) ); + reader.read( messages ); + + // Hello, future traveler. The assertion below is not strictly necessary. What we're trying to do here + // is simply to ensure that the documented message type is what we get back out when we deserialize, the + // name of the class does not strictly have to map to the name in the docs, if that is causing you trouble. + assertEquals( struct.name(), messages.asList().get( 0 ).getClass().getSimpleName() ); + } + + private void packValueOf( String type, PackStream.Packer packer ) throws IOException + { + if ( type.equalsIgnoreCase( "Text" ) ) + { + packer.pack( "Hello, world!" ); + } + else if ( type.startsWith( "Map" ) ) + { + packer.packMapHeader( 1 ); + packer.pack( "k" ); + packer.pack( 12345 ); + } + else if ( type.startsWith( "List" ) ) + { + packer.packListHeader( 2 ); + packer.pack( 1 ); + packer.pack( 2 ); + } + else + { + throw new RuntimeException( "Unknown type: " + type ); + } + } +} diff --git a/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/NDPPackStreamDocTest.java b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/NDPPackStreamDocTest.java new file mode 100644 index 0000000000000..d9bce3666ad84 --- /dev/null +++ b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/NDPPackStreamDocTest.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.ndp.docs.v1; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.ArrayList; +import java.util.Collection; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.neo4j.ndp.docs.v1.DocSerialization.normalizedHex; +import static org.neo4j.ndp.docs.v1.DocSerialization.pack; +import static org.neo4j.ndp.docs.v1.DocSerializationExample.serialization_example; +import static org.neo4j.ndp.docs.v1.DocsRepository.docs; + +@RunWith( Parameterized.class ) +public class NDPPackStreamDocTest +{ + @Parameterized.Parameter( 0 ) + public DocSerializationExample example; + + @Parameterized.Parameters( name = "{0}" ) + public static Collection documentedPackstreamExamples() + { + Collection mappings = new ArrayList<>(); + + // Load the documented mappings + for ( DocSerializationExample ex : docs().read( + "dev/serialization.asciidoc", + "code[data-lang=\"ndp_packstream_type\"]", + serialization_example ) ) + { + mappings.add( new Object[]{ex} ); + } + + for ( DocSerializationExample ex : docs().read( + "dev/messaging.asciidoc", + "code[data-lang=\"ndp_packstream_type\"]", + serialization_example ) ) + { + mappings.add( new Object[]{ex} ); + } + + return mappings; + } + + @Test + public void serializingLeadsToSpecifiedOutput() throws Throwable + { + assertThat( "Serialized version of value should match documented data: " + example, + normalizedHex( pack( example.attribute( "Value" ) ) ), + equalTo( normalizedHex( example.serializedData() ) ) ); + } +} diff --git a/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/NDPTransportChunkingDocTest.java b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/NDPTransportChunkingDocTest.java new file mode 100644 index 0000000000000..4bd4f959a40b7 --- /dev/null +++ b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/NDPTransportChunkingDocTest.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.ndp.docs.v1; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; + +import static java.lang.Integer.parseInt; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.neo4j.kernel.impl.util.Codecs.decodeHexString; +import static org.neo4j.kernel.impl.util.HexPrinter.hex; +import static org.neo4j.ndp.docs.v1.DocSerializationExample.serialization_example; +import static org.neo4j.ndp.docs.v1.DocsRepository.docs; + +@RunWith( Parameterized.class ) +public class NDPTransportChunkingDocTest +{ + @Parameterized.Parameter( 0 ) + public DocSerializationExample example; + + @Parameterized.Parameters( name = "{0}" ) + public static Collection documentedChunkingExamples() + { + Collection mappings = new ArrayList<>(); + + // Load the documented mappings + for ( DocSerializationExample ex : docs().read( + "dev/transport.asciidoc", + "code[data-lang=\"ndp_chunking_example\"]", + serialization_example ) ) + { + mappings.add( new Object[]{ex} ); + } + + return mappings; + } + + @Test + public void serializingLeadsToSpecifiedOutput() throws Throwable + { + int chunkSize = parseInt( example.attribute( "Chunk size" ) ); + + assertThat( "Serialized data should match documented representation:\n" + example, + normalizedHex( example.serializedData() ), + equalTo( serialize( chunkSize, messages( example ) ) ) ); + } + + private String serialize( int maxChunkSize, byte[][] messages ) throws IOException + { + return normalizedHex( hex( Chunker.chunk( maxChunkSize, messages ) ) ); + } + + private byte[][] messages( DocSerializationExample ex ) + { + // Not very generic, but gets the job done for now. + if ( ex.attribute( "Message data" ) != null ) + { + String hex = ex.attribute( "Message data" ).replace( " ", "" ); + return new byte[][]{decodeHexString( hex )}; + } + else + { + return new byte[][]{ + decodeHexString( ex.attribute( "Message 1 data" ).replace( " ", "" ) ), + decodeHexString( ex.attribute( "Message 2 data" ).replace( " ", "" ) )}; + } + } + + /** Convert a hex string into a normalized format for string comparison */ + private static String normalizedHex( String dirtyHex ) + { + StringBuilder str = new StringBuilder( dirtyHex.replace( "\n", "" ).replace( " ", "" ) ); + int idx = str.length() - 2; + + while ( idx > 0 ) + { + str.insert( idx, " " ); + idx = idx - 2; + } + + return str.toString().toUpperCase(); + } +} diff --git a/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/NDPValueDocTest.java b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/NDPValueDocTest.java new file mode 100644 index 0000000000000..b0731ee99cc16 --- /dev/null +++ b/community/ndp/v1-docs/src/test/java/org/neo4j/ndp/docs/v1/NDPValueDocTest.java @@ -0,0 +1,214 @@ +/* + * Copyright (c) 2002-2015 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.ndp.docs.v1; + +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeMatcher; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; + +import org.neo4j.graphdb.DynamicRelationshipType; +import org.neo4j.ndp.messaging.v1.MessageFormat; +import org.neo4j.ndp.messaging.v1.PackStreamMessageFormatV1; +import org.neo4j.ndp.messaging.v1.RecordingByteChannel; +import org.neo4j.ndp.messaging.v1.infrastructure.ValueNode; +import org.neo4j.ndp.messaging.v1.infrastructure.ValuePath; +import org.neo4j.ndp.messaging.v1.infrastructure.ValueRelationship; +import org.neo4j.ndp.messaging.v1.message.RecordMessage; +import org.neo4j.ndp.messaging.v1.util.ArrayByteChannel; +import org.neo4j.packstream.PackStream; +import org.neo4j.packstream.PackType; + +import static java.util.Arrays.asList; +import static junit.framework.TestCase.fail; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.neo4j.graphdb.DynamicLabel.label; +import static org.neo4j.helpers.collection.MapUtil.map; +import static org.neo4j.ndp.docs.v1.DocTable.table; +import static org.neo4j.ndp.docs.v1.DocsRepository.docs; +import static org.neo4j.stream.Records.record; + +/** This tests that Neo4j value mappings described in the documentation work the way we say they do. */ +@RunWith( Parameterized.class ) +public class NDPValueDocTest +{ + @Parameterized.Parameter( 0 ) + public String type; + + @Parameterized.Parameters( name = "{0}" ) + public static Collection documentedTypeMapping() + { + Collection mappings = new ArrayList<>(); + + // Load the documented mappings + for ( DocTable.Row row : docs().read( "dev/serialization.asciidoc", "#ndp-type-system-mapping", table ) + .get( 0 ) ) + { + mappings.add( new Object[]{row.get( 0 )} ); + } + + return mappings; + } + + @Test + public void mappingShouldBeCorrect() throws Throwable + { + assertThat( serialize( neoValue( type ) ), isPackstreamType( type ) ); + } + + private Matcher isPackstreamType( final String packStreamType ) + { + return new TypeSafeMatcher() + { + @Override + protected boolean matchesSafely( byte[] recordWithValue ) + { + if ( packStreamType.equalsIgnoreCase( "identity" ) ) + { + // TODO: Skipping this for now, the docs are in fact out of sync with reality, need to resolve + // outside of this documentation PR + return true; + } + + PackStream.Unpacker unpacker = new PackStream.Unpacker( 64 ); + unpacker.reset( new ArrayByteChannel( recordWithValue ) ); + + try + { + // Wrapped in a "Record" struct + unpacker.unpackStructHeader(); + unpacker.unpackStructSignature(); + unpacker.unpackListHeader(); + + PackType type = unpacker.peekNextType(); + if ( type.name().equalsIgnoreCase( "struct" ) ) + { + String structName = null; + unpacker.unpackStructHeader(); + char sig = unpacker.unpackStructSignature(); + switch ( sig ) + { + case PackStreamMessageFormatV1.NODE: + structName = "node"; + break; + case PackStreamMessageFormatV1.RELATIONSHIP: + structName = "relationship"; + break; + case PackStreamMessageFormatV1.PATH: + structName = "path"; + break; + default: + fail( "Unknown struct type: " + sig ); + } + assertThat( structName, equalTo( packStreamType.toLowerCase() ) ); + } + else + { + assertThat( type.name().toLowerCase(), equalTo( packStreamType.toLowerCase() ) ); + } + } + catch ( IOException e ) + { + throw new RuntimeException( e ); + } + + return true; + } + + @Override + public void describeTo( Description description ) + { + description.appendText( "PackStream type " + packStreamType ); + } + }; + } + + private Object neoValue( String type ) + { + if ( type.equalsIgnoreCase( "float" ) ) + { + return 12345.12345d; + } + else if ( type.equalsIgnoreCase( "integer" ) ) + { + return 1337l; + } + else if ( type.equalsIgnoreCase( "boolean" ) ) + { + return true; + } + else if ( type.equalsIgnoreCase( "text" ) ) + { + return "Steven Brookreson"; + } + else if ( type.equalsIgnoreCase( "list" ) ) + { + return asList( 1, 2, 3 ); + } + else if ( type.equalsIgnoreCase( "map" ) ) + { + return map( "k", 1 ); + } + else if ( type.equalsIgnoreCase( "node" ) ) + { + return new ValueNode( 12, asList( label( "User" ) ), map() ); + } + else if ( type.equalsIgnoreCase( "relationship" ) ) + { + return new ValueRelationship( 12, 1, 2, DynamicRelationshipType.withName( "KNOWS" ), map() ); + } + else if ( type.equalsIgnoreCase( "path" ) ) + { + return new ValuePath(); + } + else if ( type.equalsIgnoreCase( "null" ) ) + { + return null; + } + else if ( type.equalsIgnoreCase( "identity" ) ) + { + return null; // TODO: No representation for identity yet + } + else + { + throw new RuntimeException( "Unknown neo type: " + type ); + } + } + + private byte[] serialize( Object neoValue ) throws IOException + { + MessageFormat format = new PackStreamMessageFormatV1(); + RecordMessage msg = new RecordMessage( record( neoValue ) ); + MessageFormat.Writer writer = format.newWriter(); + RecordingByteChannel channel = new RecordingByteChannel(); + + writer.reset( channel ).write( msg ).flush(); + + channel.eof(); + return channel.getBytes(); + } +}