From f814082891d4717799cd1487a8d48a78a7012478 Mon Sep 17 00:00:00 2001 From: MishaDemianenko Date: Wed, 16 Nov 2016 10:36:02 +0100 Subject: [PATCH] Value validation for indexed properties. Introduce validation of property values for legacy and schema indexes. Create validators for empty/null/allowable cases. Perform validation during property updated and index population. --- .../impl/api/IndexSimpleValueValidator.java | 45 ++++ .../impl/api/IndexValueLengthValidator.java | 49 +++++ .../kernel/impl/api/IndexValueValidator.java | 42 ++++ .../impl/api/LegacyIndexValueValidator.java | 41 ++++ .../api/StateHandlingStatementOperations.java | 6 + .../storeview/StoreViewNodeStoreScan.java | 5 +- .../neo4j/kernel/impl/util/Validators.java | 7 + .../api/IndexSimpleValueValidatorTest.java | 77 +++++++ .../api/IndexValueLengthValidatorTest.java | 52 +++++ .../impl/api/IndexValueValidatorTest.java | 52 +++++ .../api/LegacyIndexValueValidatorTest.java | 46 ++++ .../impl/lucene/legacy/LuceneLegacyIndex.java | 14 +- .../impl/schema/LuceneDocumentStructure.java | 30 +-- .../impl/lucene/legacy/TestLuceneIndex.java | 56 +++++ .../schema/LuceneDocumentStructureTest.java | 19 +- .../schema/IndexValuesValidationTest.java | 197 ++++++++++++++++++ 16 files changed, 683 insertions(+), 55 deletions(-) create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/api/IndexSimpleValueValidator.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/api/IndexValueLengthValidator.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/api/IndexValueValidator.java create mode 100644 community/kernel/src/main/java/org/neo4j/kernel/impl/api/LegacyIndexValueValidator.java create mode 100644 community/kernel/src/test/java/org/neo4j/kernel/impl/api/IndexSimpleValueValidatorTest.java create mode 100644 community/kernel/src/test/java/org/neo4j/kernel/impl/api/IndexValueLengthValidatorTest.java create mode 100644 community/kernel/src/test/java/org/neo4j/kernel/impl/api/IndexValueValidatorTest.java create mode 100644 community/kernel/src/test/java/org/neo4j/kernel/impl/api/LegacyIndexValueValidatorTest.java create mode 100644 community/neo4j/src/test/java/schema/IndexValuesValidationTest.java diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/IndexSimpleValueValidator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/IndexSimpleValueValidator.java new file mode 100644 index 0000000000000..0d448dbfc6d8f --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/IndexSimpleValueValidator.java @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.api; + +import org.neo4j.kernel.impl.util.Validator; + +public class IndexSimpleValueValidator implements Validator +{ + + public static IndexSimpleValueValidator INSTANCE = new IndexSimpleValueValidator(); + + private IndexSimpleValueValidator() + { + } + + @Override + public void validate( Object value ) + { + if ( value == null ) + { + throw new IllegalArgumentException( "Null value" ); + } + if ( value instanceof String ) + { + IndexValueLengthValidator.INSTANCE.validate( ((String) value).getBytes() ); + } + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/IndexValueLengthValidator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/IndexValueLengthValidator.java new file mode 100644 index 0000000000000..7371bd7ae2ecf --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/IndexValueLengthValidator.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.api; + +import org.neo4j.kernel.impl.util.Validator; + +public class IndexValueLengthValidator implements Validator +{ + // Maximum bytes value length that supported by indexes. + // Absolute hard maximum length for a term, in bytes once + // encoded as UTF8. If a term arrives from the analyzer + // longer than this length, an IllegalArgumentException + // when lucene writer trying to add or update document + private static final int MAX_TERM_LENGTH = (1 << 15) - 2; + + public static final IndexValueLengthValidator INSTANCE = new IndexValueLengthValidator(); + + private IndexValueLengthValidator() + { + } + + @Override + public void validate( byte[] bytes ) + { + if ( bytes.length > MAX_TERM_LENGTH ) + { + throw new IllegalArgumentException( "Property value bytes length: " + bytes.length + + " is longer then " + MAX_TERM_LENGTH + ", which is maximum supported length" + + " of indexed property value." ); + } + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/IndexValueValidator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/IndexValueValidator.java new file mode 100644 index 0000000000000..d0d31e17c10dd --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/IndexValueValidator.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.api; + +import org.neo4j.kernel.api.index.ArrayEncoder; +import org.neo4j.kernel.impl.util.Validator; + +public class IndexValueValidator implements Validator +{ + public static IndexValueValidator INSTANCE = new IndexValueValidator(); + + private IndexValueValidator() + { + } + + @Override + public void validate( Object value ) + { + IndexSimpleValueValidator.INSTANCE.validate( value ); + if ( value.getClass().isArray() ) + { + IndexValueLengthValidator.INSTANCE.validate( ArrayEncoder.encode( value ).getBytes() ); + } + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/LegacyIndexValueValidator.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/LegacyIndexValueValidator.java new file mode 100644 index 0000000000000..a85e768f6a3b2 --- /dev/null +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/LegacyIndexValueValidator.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.api; + +import org.neo4j.kernel.impl.util.Validator; + +public class LegacyIndexValueValidator implements Validator +{ + public static LegacyIndexValueValidator INSTANCE = new LegacyIndexValueValidator(); + + private LegacyIndexValueValidator() + { + } + + @Override + public void validate( Object value ) + { + IndexSimpleValueValidator.INSTANCE.validate( value ); + if ( !(value instanceof Number) && value.toString() == null ) + { + throw new IllegalArgumentException( "Value of type " + value.getClass() + " has null toString" ); + } + } +} diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/StateHandlingStatementOperations.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/StateHandlingStatementOperations.java index 233f1acdd8693..35f6d756bae4f 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/api/StateHandlingStatementOperations.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/api/StateHandlingStatementOperations.java @@ -79,6 +79,7 @@ import org.neo4j.kernel.impl.index.IndexEntityType; import org.neo4j.kernel.impl.index.LegacyIndexStore; import org.neo4j.kernel.impl.util.Cursors; +import org.neo4j.kernel.impl.util.Validators; import org.neo4j.register.Register.DoubleLongRegister; import org.neo4j.storageengine.api.EntityType; import org.neo4j.storageengine.api.LabelItem; @@ -1117,6 +1118,11 @@ private void indexUpdateProperty( KernelStatement state, long nodeId, int labelI IndexDescriptor descriptor = indexGetForLabelAndPropertyKey( state, labelId, propertyKey ); if ( descriptor != null ) { + if (after != null) + { + Validators.INDEX_VALUE_VALIDATOR.validate( after.value() ); + } + state.txState().indexDoUpdateProperty( descriptor, nodeId, before, after ); } } diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/storeview/StoreViewNodeStoreScan.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/storeview/StoreViewNodeStoreScan.java index 12dcd1a6a87c2..b2a0e3141d014 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/storeview/StoreViewNodeStoreScan.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/transaction/state/storeview/StoreViewNodeStoreScan.java @@ -40,6 +40,7 @@ import org.neo4j.kernel.impl.store.record.PropertyBlock; import org.neo4j.kernel.impl.store.record.PropertyRecord; import org.neo4j.kernel.impl.store.record.Record; +import org.neo4j.kernel.impl.util.Validators; import static org.neo4j.collection.primitive.PrimitiveLongCollections.EMPTY_LONG_ARRAY; import static org.neo4j.kernel.api.labelscan.NodeLabelUpdate.labelChanges; @@ -103,7 +104,9 @@ public void process( NodeRecord node ) throws FAILURE if ( propertyKeyIdFilter.test( propertyKeyId ) ) { // This node has a property of interest to us - updates.add( propertyKeyId, valueOf( property ), labels ); + Object value = valueOf( property ); + Validators.INDEX_VALUE_VALIDATOR.validate( value ); + updates.add( propertyKeyId, value, labels ); } } if ( updates.containsUpdates() ) diff --git a/community/kernel/src/main/java/org/neo4j/kernel/impl/util/Validators.java b/community/kernel/src/main/java/org/neo4j/kernel/impl/util/Validators.java index c3d26470d7e27..dd3b49845b7a7 100644 --- a/community/kernel/src/main/java/org/neo4j/kernel/impl/util/Validators.java +++ b/community/kernel/src/main/java/org/neo4j/kernel/impl/util/Validators.java @@ -29,11 +29,18 @@ import org.neo4j.io.fs.DefaultFileSystemAbstraction; import org.neo4j.io.fs.FileSystemAbstraction; +import org.neo4j.kernel.impl.api.IndexValueValidator; +import org.neo4j.kernel.impl.api.LegacyIndexValueValidator; import org.neo4j.kernel.impl.store.MetaDataStore; import org.neo4j.kernel.impl.storemigration.StoreFileType; public class Validators { + + public static final IndexValueValidator INDEX_VALUE_VALIDATOR = IndexValueValidator.INSTANCE; + + public static final LegacyIndexValueValidator LEGACY_INDEX_VALUE_VALIDATOR = LegacyIndexValueValidator.INSTANCE; + public static final Validator REGEX_FILE_EXISTS = file -> { if ( matchingFiles( file ).isEmpty() ) diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/IndexSimpleValueValidatorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/IndexSimpleValueValidatorTest.java new file mode 100644 index 0000000000000..b68aab5a60e0d --- /dev/null +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/IndexSimpleValueValidatorTest.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.api; + +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import org.neo4j.kernel.impl.util.Validator; + +import static org.neo4j.kernel.impl.api.IndexSimpleValueValidator.INSTANCE; + +public class IndexSimpleValueValidatorTest +{ + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void nullIsNotAllowed() throws Exception + { + expectedException.expect( IllegalArgumentException.class ); + expectedException.expectMessage( "Null value" ); + getValidator().validate( null ); + } + + @Test + public void tooLongStringIsNotAllowed() + { + expectedException.expect( IllegalArgumentException.class ); + expectedException.expectMessage( + "Property value bytes length: 35000 is longer then 32766, " + + "which is maximum supported length of indexed property value." ); + getValidator().validate( RandomStringUtils.randomAlphabetic( 35000 ) ); + } + + @Test + public void numberIsValidValue() + { + getValidator().validate( 5 ); + getValidator().validate( 5.0d ); + getValidator().validate( 5.0f ); + getValidator().validate( 5L ); + } + + @Test + public void shortStringIsValidValue() + { + getValidator().validate( RandomStringUtils.randomAlphabetic( 5 ) ); + getValidator().validate( RandomStringUtils.randomAlphabetic( 10 ) ); + getValidator().validate( RandomStringUtils.randomAlphabetic( 250 ) ); + getValidator().validate( RandomStringUtils.randomAlphabetic( 450 ) ); + } + + protected Validator getValidator() + { + return INSTANCE; + } +} diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/IndexValueLengthValidatorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/IndexValueLengthValidatorTest.java new file mode 100644 index 0000000000000..c274b16a10f6a --- /dev/null +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/IndexValueLengthValidatorTest.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.api; + +import org.apache.commons.lang3.RandomUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static org.neo4j.kernel.impl.api.IndexValueLengthValidator.INSTANCE; + +public class IndexValueLengthValidatorTest +{ + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void tooLongByteArrayIsNotAllowed() + { + expectedException.expect( IllegalArgumentException.class ); + expectedException.expectMessage( "Property value bytes length: 35000 is longer then 32766, which is maximum supported length of indexed property value." ); + INSTANCE.validate( RandomUtils.nextBytes( 35000 ) ); + } + + @Test + public void shortByteArrayIsValid() + { + INSTANCE.validate( RandomUtils.nextBytes( 3 ) ); + INSTANCE.validate( RandomUtils.nextBytes( 30 ) ); + INSTANCE.validate( RandomUtils.nextBytes( 300 ) ); + INSTANCE.validate( RandomUtils.nextBytes( 4303 ) ); + INSTANCE.validate( RandomUtils.nextBytes( 13234 ) ); + } + +} diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/IndexValueValidatorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/IndexValueValidatorTest.java new file mode 100644 index 0000000000000..7388ecb788b7e --- /dev/null +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/IndexValueValidatorTest.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.api; + +import org.apache.commons.lang3.RandomUtils; +import org.junit.Test; + +import org.neo4j.kernel.impl.util.Validator; + +public class IndexValueValidatorTest extends IndexSimpleValueValidatorTest +{ + @Test + public void tooLongArrayIsNotAllowed() throws Exception + { + expectedException.expect( IllegalArgumentException.class ); + expectedException.expectMessage( "is longer then 32766, " + + "which is maximum supported length of indexed property value." ); + getValidator().validate( RandomUtils.nextBytes(39978 ) ); + } + + @Test + public void shortArrayIsAllowed() throws Exception + { + getValidator().validate( RandomUtils.nextBytes( 3 ) ); + getValidator().validate( RandomUtils.nextBytes( 30 ) ); + getValidator().validate( RandomUtils.nextBytes( 450 ) ); + getValidator().validate( RandomUtils.nextBytes( 4556 ) ); + } + + @Override + protected Validator getValidator() + { + return IndexValueValidator.INSTANCE; + } +} diff --git a/community/kernel/src/test/java/org/neo4j/kernel/impl/api/LegacyIndexValueValidatorTest.java b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/LegacyIndexValueValidatorTest.java new file mode 100644 index 0000000000000..d20132a79ff9b --- /dev/null +++ b/community/kernel/src/test/java/org/neo4j/kernel/impl/api/LegacyIndexValueValidatorTest.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.neo4j.kernel.impl.api; + +import org.junit.Test; +import org.mockito.Mockito; + +import org.neo4j.kernel.impl.util.Validator; + +public class LegacyIndexValueValidatorTest extends IndexSimpleValueValidatorTest +{ + + @Test + public void nullToStringIsNotAllowed() throws Exception + { + expectedException.expect( IllegalArgumentException.class ); + expectedException.expectMessage( "has null toString" ); + + Object testValue = Mockito.mock( Object.class ); + Mockito.when( testValue.toString() ).thenReturn( null ); + getValidator().validate( testValue ); + } + + @Override + protected Validator getValidator() + { + return LegacyIndexValueValidator.INSTANCE; + } +} diff --git a/community/lucene-index/src/main/java/org/neo4j/index/impl/lucene/legacy/LuceneLegacyIndex.java b/community/lucene-index/src/main/java/org/neo4j/index/impl/lucene/legacy/LuceneLegacyIndex.java index b5cfbec072b58..e113c80897a4b 100644 --- a/community/lucene-index/src/main/java/org/neo4j/index/impl/lucene/legacy/LuceneLegacyIndex.java +++ b/community/lucene-index/src/main/java/org/neo4j/index/impl/lucene/legacy/LuceneLegacyIndex.java @@ -52,6 +52,7 @@ import org.neo4j.kernel.api.LegacyIndexHits; import org.neo4j.kernel.api.impl.index.collector.DocValuesCollector; import org.neo4j.kernel.impl.util.IoPrimitiveUtils; +import org.neo4j.kernel.impl.util.Validators; import org.neo4j.kernel.spi.legacyindex.IndexCommandFactory; import static org.neo4j.collection.primitive.Primitive.longSet; @@ -119,7 +120,7 @@ protected Object getCorrectValue( Object value ) Object result = value instanceof ValueContext ? ((ValueContext) value).getCorrectValue() : value.toString(); - assertValidValue( value ); + assertValidValue( result ); return result; } @@ -131,16 +132,9 @@ private static void assertValidKey( String key ) } } - private static void assertValidValue( Object singleValue ) + protected void assertValidValue( Object value ) { - if ( singleValue == null ) - { - throw new IllegalArgumentException( "Null value" ); - } - if ( !(singleValue instanceof Number) && singleValue.toString() == null ) - { - throw new IllegalArgumentException( "Value of type " + singleValue.getClass() + " has null toString" ); - } + Validators.LEGACY_INDEX_VALUE_VALIDATOR.validate( value ); } /** diff --git a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/LuceneDocumentStructure.java b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/LuceneDocumentStructure.java index f8120bd0af8e9..abb73b2a257dd 100644 --- a/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/LuceneDocumentStructure.java +++ b/community/lucene-index/src/main/java/org/neo4j/kernel/api/impl/schema/LuceneDocumentStructure.java @@ -25,7 +25,6 @@ import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.FilteredTermsEnum; -import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; @@ -63,12 +62,6 @@ public class LuceneDocumentStructure public static final String NODE_ID_KEY = "id"; - // Absolute hard maximum length for a term, in bytes once - // encoded as UTF8. If a term arrives from the analyzer - // longer than this length, an IllegalArgumentException - // when lucene writer trying to add or update document - private static final int MAX_FIELD_LENGTH = IndexWriter.MAX_TERM_LENGTH; - private static final ThreadLocal perThreadDocument = new ThreadLocal() { @Override @@ -286,28 +279,7 @@ private void setValue( ValueEncoding encoding, Object value ) { removeAllValueFields(); Field reusableField = getFieldWithValue( encoding, value ); - if ( isArrayOrString( reusableField ) ) - { - if ( isShorterThenMaximum( reusableField ) ) - { - document.add( reusableField ); - } - } - else - { - document.add( reusableField ); - } - } - - private boolean isShorterThenMaximum( Field reusableField ) - { - return reusableField.stringValue().getBytes().length <= MAX_FIELD_LENGTH; - } - - private boolean isArrayOrString( Field reusableField ) - { - return ValueEncoding.Array.key().equals( reusableField.name() ) || - ValueEncoding.String.key().equals( reusableField.name() ); + document.add( reusableField ); } private void removeAllValueFields() diff --git a/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/legacy/TestLuceneIndex.java b/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/legacy/TestLuceneIndex.java index e940dae11aa71..a016b1945c873 100644 --- a/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/legacy/TestLuceneIndex.java +++ b/community/lucene-index/src/test/java/org/neo4j/index/impl/lucene/legacy/TestLuceneIndex.java @@ -19,6 +19,7 @@ */ package org.neo4j.index.impl.lucene.legacy; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper; import org.apache.lucene.index.Term; import org.apache.lucene.queryparser.classic.QueryParser.Operator; @@ -1995,4 +1996,59 @@ public void shouldNotBeAbleToAddNullValuesToRelationshipIndex() throws Exception // THEN Good } } + + @Test + public void tooLongValuesAreNotAllowedInNodeIndex() + { + Index index = nodeIndex( EXACT_CONFIG ); + String tooLongValue = RandomStringUtils.randomAlphabetic( 36000 ); + try + { + index.add( graphDb.createNode(), "key", tooLongValue ); + fail( "Validation exception expected. Such long values are not allowed in the index." ); + } + catch ( IllegalArgumentException e ) + { + // expected + } + + try + { + index.add( graphDb.createNode(), "key", new String[] {"a", tooLongValue, "c"} ); + fail( "Validation exception expected. Such long values are not allowed in the index." ); + } + catch ( IllegalArgumentException e ) + { + // expected + } + + } + + @Test + public void tooLongValuesAreNotAllowedInRelationshipIndex() + { + RelationshipIndex index = relationshipIndex( EXACT_CONFIG ); + String tooLongValue = RandomStringUtils.randomAlphabetic( 36000 ); + try + { + index.add( graphDb.createNode().createRelationshipTo( graphDb.createNode(), MyRelTypes.TEST ), "key", + tooLongValue ); + fail( "Validation exception expected. Such long values are not allowed in the index." ); + } + catch ( IllegalArgumentException e ) + { + // expected + } + + try + { + index.add( graphDb.createNode().createRelationshipTo( graphDb.createNode(), MyRelTypes.TEST ), "key", + new String[] {"a", tooLongValue, "c"} ); + fail( "Validation exception expected. Such long values are not allowed in the index." ); + } + catch ( IllegalArgumentException e ) + { + // expected + } + } } diff --git a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/LuceneDocumentStructureTest.java b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/LuceneDocumentStructureTest.java index 2791b81703cfa..6c01ad2063590 100644 --- a/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/LuceneDocumentStructureTest.java +++ b/community/lucene-index/src/test/java/org/neo4j/kernel/api/impl/schema/LuceneDocumentStructureTest.java @@ -28,9 +28,10 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.WildcardQuery; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; -import static junit.framework.Assert.assertNull; import static junit.framework.TestCase.assertEquals; import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertThat; @@ -43,21 +44,9 @@ public class LuceneDocumentStructureTest { - @Test - public void tooLongStringShouldBeSkipped() - { - String string = RandomStringUtils.randomAscii( 358749 ); - Document document = LuceneDocumentStructure.documentRepresentingProperty( 123, string ); - assertNull( document.getField( String.key() ) ); - } - @Test - public void tooLongArrayShouldBeSkipped() - { - byte[] bytes = RandomStringUtils.randomAscii( IndexWriter.MAX_TERM_LENGTH + 10 ).getBytes(); - Document document = LuceneDocumentStructure.documentRepresentingProperty( 123, bytes ); - assertNull( document.getField( Array.key() ) ); - } + @Rule + public final ExpectedException expectedException = ExpectedException.none(); @Test public void stringWithMaximumLengthShouldBeAllowed() diff --git a/community/neo4j/src/test/java/schema/IndexValuesValidationTest.java b/community/neo4j/src/test/java/schema/IndexValuesValidationTest.java new file mode 100644 index 0000000000000..7ff10d5b70557 --- /dev/null +++ b/community/neo4j/src/test/java/schema/IndexValuesValidationTest.java @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2002-2016 "Neo Technology," + * Network Engine for Objects in Lund AB [http://neotechnology.com] + * + * This file is part of Neo4j. + * + * Neo4j is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package schema; + +import org.apache.commons.lang3.StringUtils; +import org.apache.lucene.index.IndexWriter; +import org.hamcrest.Matchers; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.concurrent.TimeUnit; + +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.graphdb.Label; +import org.neo4j.graphdb.Node; +import org.neo4j.graphdb.Relationship; +import org.neo4j.graphdb.RelationshipType; +import org.neo4j.graphdb.Transaction; +import org.neo4j.graphdb.factory.GraphDatabaseFactory; +import org.neo4j.graphdb.schema.IndexDefinition; +import org.neo4j.test.rule.TestDirectory; + +import static org.junit.Assert.assertThat; + + +public class IndexValuesValidationTest +{ + + @ClassRule + public static final TestDirectory directory = TestDirectory.testDirectory(); + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private static GraphDatabaseService database; + + @BeforeClass + public static void setUp() + { + database = new GraphDatabaseFactory().newEmbeddedDatabase( directory.graphDbDir() ); + } + + @AfterClass + public static void tearDown() + { + database.shutdown(); + } + + @Test + public void validateIndexedNodeProperties() throws Exception + { + Label label = Label.label( "indexedNodePropertiesTestLabel" ); + String propertyName = "indexedNodePropertyName"; + + createIndex( label, propertyName ); + + try ( Transaction ignored = database.beginTx() ) + { + database.schema().awaitIndexesOnline( 5, TimeUnit.MINUTES ); + } + + expectedException.expect( IllegalArgumentException.class ); + expectedException.expectMessage( "Property value bytes length: 32767 is longer then 32766, " + + "which is maximum supported length of indexed property value." ); + + try ( Transaction transaction = database.beginTx() ) + { + Node node = database.createNode( label ); + node.setProperty( propertyName, StringUtils.repeat( "a", IndexWriter.MAX_TERM_LENGTH + 1 ) ); + transaction.success(); + } + } + + @Test + public void validateNodePropertiesOnPopulation() + { + Label label = Label.label( "populationTestNodeLabel" ); + String propertyName = "populationTestPropertyName"; + + try ( Transaction transaction = database.beginTx() ) + { + Node node = database.createNode( label ); + node.setProperty( propertyName, StringUtils.repeat( "a", IndexWriter.MAX_TERM_LENGTH + 1 ) ); + transaction.success(); + } + + IndexDefinition indexDefinition = createIndex( label, propertyName ); + try + { + try ( Transaction ignored = database.beginTx() ) + { + database.schema().awaitIndexesOnline( 5, TimeUnit.MINUTES ); + } + } + catch ( IllegalStateException e ) + { + try ( Transaction ignored = database.beginTx() ) + { + String indexFailure = database.schema().getIndexFailure( indexDefinition ); + assertThat( "", indexFailure, Matchers.startsWith( "java.lang.IllegalArgumentException: " + + "Property value bytes length: 32767 is longer then 32766, " + + "which is maximum supported length of indexed property value." ) ); + } + } + } + + @Test + public void validateLegacyIndexedNodeProperties() + { + Label label = Label.label( "legacyIndexedNodePropertiesTestLabel" ); + String propertyName = "legacyIndexedNodeProperties"; + String legacyIndexedNodeIndex = "legacyIndexedNodeIndex"; + + try ( Transaction transaction = database.beginTx() ) + { + Node node = database.createNode( label ); + database.index().forNodes( legacyIndexedNodeIndex ) + .add( node, propertyName, "shortString" ); + transaction.success(); + } + + expectedException.expect( IllegalArgumentException.class ); + expectedException.expectMessage( "Property value bytes length: 32767 is longer then 32766, " + + "which is maximum supported length of indexed property value." ); + try ( Transaction transaction = database.beginTx() ) + { + Node node = database.createNode( label ); + String longValue = StringUtils.repeat( "a", IndexWriter.MAX_TERM_LENGTH + 1 ); + database.index().forNodes( legacyIndexedNodeIndex ) + .add( node, propertyName, longValue ); + transaction.success(); + } + } + + @Test + public void validateLegacyIndexedRelationshipProperties() + { + Label label = Label.label( "legacyIndexedRelationshipPropertiesTestLabel" ); + String propertyName = "legacyIndexedRelationshipProperties"; + String legacyIndexedRelationshipIndex = "legacyIndexedRelationshipIndex"; + RelationshipType indexType = RelationshipType.withName( "legacyIndexType" ); + + try ( Transaction transaction = database.beginTx() ) + { + Node source = database.createNode( label ); + Node destination = database.createNode( label ); + Relationship relationship = source.createRelationshipTo( destination, indexType ); + database.index().forRelationships( legacyIndexedRelationshipIndex ) + .add( relationship, propertyName, "shortString" ); + transaction.success(); + } + + expectedException.expect( IllegalArgumentException.class ); + expectedException.expectMessage( "Property value bytes length: 32767 is longer then 32766, " + + "which is maximum supported length of indexed property value." ); + try ( Transaction transaction = database.beginTx() ) + { + Node source = database.createNode( label ); + Node destination = database.createNode( label ); + Relationship relationship = source.createRelationshipTo( destination, indexType ); + String longValue = StringUtils.repeat( "a", IndexWriter.MAX_TERM_LENGTH + 1 ); + database.index().forRelationships( legacyIndexedRelationshipIndex ) + .add( relationship, propertyName, longValue ); + transaction.success(); + } + } + + private IndexDefinition createIndex( Label label, String propertyName ) + { + try ( Transaction transaction = database.beginTx() ) + { + IndexDefinition indexDefinition = database.schema().indexFor( label ).on( propertyName ).create(); + transaction.success(); + return indexDefinition; + } + } +}