Skip to content

Commit

Permalink
Native index validate correct string length
Browse files Browse the repository at this point in the history
  • Loading branch information
burqen committed Jul 27, 2018
1 parent 74f7f82 commit 349561a
Show file tree
Hide file tree
Showing 4 changed files with 133 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -563,7 +563,7 @@ public String providerName()
"This improves read and write performance for non-composite indexed numbers. " +
"lucene+native-2.0: Store strings in a native index and remaining value types like lucene+native-1.0. " +
"This improves write performance for non-composite indexed strings. " +
"This version of the native string index has a value limit of 4046 bytes, such that byte-representation " +
"This version of the native string index has a value limit of 4039 bytes, such that byte-representation " +
"of a string to index cannot be larger than that limit, or the transaction trying to index such a value will fail. " +
"This version of the native string index also has reduced performance for CONTAINS and ENDS WITH queries, " +
"due to resorting to index scan+filter internally. " +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,11 @@ void instantiateTree( RecoveryCleanupWorkCollector recoveryCleanupWorkCollector,
ensureDirectoryExist();
GBPTree.Monitor monitor = treeMonitor();
tree = new GBPTree<>( pageCache, storeFile, layout, 0, monitor, NO_HEADER_READER, headerWriter, recoveryCleanupWorkCollector );
afterTreeInstantiation( tree );
}

protected void afterTreeInstantiation( GBPTree<KEY,VALUE> tree )
{ // no-op per default
}

private GBPTree.Monitor treeMonitor( )
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@
import java.io.File;
import java.io.IOException;

import org.neo4j.index.internal.gbptree.GBPTree;
import org.neo4j.index.internal.gbptree.Layout;
import org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector;
import org.neo4j.index.internal.gbptree.TreeNodeDynamicSize;
import org.neo4j.io.fs.FileSystemAbstraction;
import org.neo4j.io.pagecache.PageCache;
import org.neo4j.kernel.api.index.IndexAccessor;
Expand All @@ -41,7 +41,7 @@
*/
public class StringSchemaIndexAccessor extends NativeSchemaIndexAccessor<StringSchemaKey,NativeSchemaValue>
{
private static final Validator<Value> VALIDATOR = new IndexTextValueLengthValidator( TreeNodeDynamicSize.MAX_KEY_SIZE );
private Validator<Value> validator;

StringSchemaIndexAccessor(
PageCache pageCache,
Expand All @@ -57,6 +57,12 @@ public class StringSchemaIndexAccessor extends NativeSchemaIndexAccessor<StringS
super( pageCache, fs, storeFile, layout, recoveryCleanupWorkCollector, monitor, descriptor, indexId, samplingConfig );
}

@Override
protected void afterTreeInstantiation( GBPTree<StringSchemaKey,NativeSchemaValue> tree )
{
validator = new IndexTextValueLengthValidator( tree.keyValueSizeCap() - StringSchemaKey.ENTITY_ID_SIZE );
}

@Override
public IndexReader newReader()
{
Expand All @@ -67,6 +73,6 @@ public IndexReader newReader()
@Override
public void validateBeforeCommit( Value[] tuple )
{
VALIDATOR.validate( tuple[0] );
validator.validate( tuple[0] );
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
/*
* Copyright (c) 2002-2018 "Neo4j,"
* Neo4j Sweden AB [http://neo4j.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.neo4j.kernel.api.impl.schema;

import org.hamcrest.Matchers;
import org.junit.Rule;
import org.junit.Test;

import java.util.concurrent.TimeUnit;

import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Transaction;
import org.neo4j.graphdb.factory.GraphDatabaseSettings;
import org.neo4j.index.internal.gbptree.TreeNodeDynamicSize;
import org.neo4j.io.pagecache.PageCache;
import org.neo4j.test.rule.DatabaseRule;
import org.neo4j.test.rule.EmbeddedDatabaseRule;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import static org.neo4j.test.TestLabels.LABEL_ONE;

public class StringLengthIndexValidationIT
{
@Rule
public DatabaseRule db = new EmbeddedDatabaseRule()
.withSetting( GraphDatabaseSettings.default_schema_provider, GraphDatabaseSettings.SchemaIndex.NATIVE20.providerName() );

private static final String propKey = "largeString";
private static final int keySizeLimit = TreeNodeDynamicSize.keyValueSizeCapFromPageSize( PageCache.PAGE_SIZE ) - Long.BYTES;

@Test
public void shouldSuccessfullyWriteAndReadWithinIndexKeySizeLimit()
{
createIndex();
String propValue = getString( keySizeLimit );
long expectedNodeId;

// Write
try ( Transaction tx = db.beginTx() )
{
Node node = db.createNode( LABEL_ONE );
node.setProperty( propKey, propValue );
expectedNodeId = node.getId();
tx.success();
}

// Read
try ( Transaction tx = db.beginTx() )
{
Node node = db.findNode( LABEL_ONE, propKey, propValue );
assertNotNull( node );
assertEquals( "node id", expectedNodeId, node.getId() );
tx.success();
}
}

@Test
public void txMustFailIfExceedingIndexKeySizeLimit()
{
createIndex();

// Write
try ( Transaction tx = db.beginTx() )
{
String propValue = getString( keySizeLimit + 1 );
db.createNode( LABEL_ONE ).setProperty( propKey, propValue );
tx.success();
}
catch ( IllegalArgumentException e )
{
assertThat( e.getMessage(), Matchers.containsString( String.format(
"Property value bytes length: %d is longer than %d, which is maximum supported length of indexed property value", keySizeLimit + 1, keySizeLimit ) ) );
}
}

private String getString( int stringSize )
{
StringBuilder buf = new StringBuilder();
for ( int i = 0; i < stringSize; i++ )
{
buf.append( "0" );
}
return buf.toString();
}

private void createIndex()
{
try ( Transaction tx = db.beginTx() )
{
db.schema().indexFor( LABEL_ONE ).on( propKey ).create();
tx.success();
}
try ( Transaction tx = db.beginTx() )
{
db.schema().awaitIndexesOnline( 1, TimeUnit.MINUTES );
tx.success();
}
}
}

0 comments on commit 349561a

Please sign in to comment.