Skip to content
This repository has been archived by the owner on Jan 5, 2022. It is now read-only.

Commit

Permalink
Added most recent changes and a test that exposes potential issue whe…
Browse files Browse the repository at this point in the history 
…n selective indexing properties.
  • Loading branch information
@GERey
GERey committed Mar 24, 2016
1 parent c0b96a5 commit 1491264
Show file tree
Hide file tree
Showing 2 changed files with 120 additions and 82 deletions.
148 changes: 66 additions & 82 deletions stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
View file
Expand Up @@ -196,14 +196,16 @@ private Map<String, Object> getFilteredStringObjectMap( final ApplicationScope a


MapManager mm = mapManagerFactory.createMapManager( ms ); MapManager mm = mapManagerFactory.createMapManager( ms );


String jsonMap = mm.getString( indexEdge.getEdgeName().split( "\\|" )[1] ); //determines is the map manager has a collection schema associated with it.
String jsonSchemaMap = mm.getString( indexEdge.getEdgeName().split( "\\|" )[1] );


Set<String> defaultProperties = null; Set<String> defaultProperties = null;
ArrayList fieldsToKeep = null; ArrayList fieldsToKeep = null;


if(jsonMap != null) { //If we do have a schema then parse it and add it to a list of properties we want to keep.Otherwise return.
if(jsonSchemaMap != null) {


Map jsonMapData = ( Map ) JsonUtils.parse( jsonMap ); Map jsonMapData = ( Map ) JsonUtils.parse( jsonSchemaMap );
Schema schema = Schema.getDefaultSchema(); Schema schema = Schema.getDefaultSchema();
defaultProperties = schema.getRequiredProperties( indexEdge.getEdgeName().split( "\\|" )[1]); defaultProperties = schema.getRequiredProperties( indexEdge.getEdgeName().split( "\\|" )[1]);
//TODO: additional logic to //TODO: additional logic to
Expand All @@ -213,105 +215,87 @@ private Map<String, Object> getFilteredStringObjectMap( final ApplicationScope a
else else
return null; return null;



Map<String, Object> map = indexOperation.convertedEntityToBeIndexed( applicationScope,indexEdge,entity ); Map<String, Object> map = indexOperation.convertedEntityToBeIndexed( applicationScope,indexEdge,entity );
HashSet mapFields = ( HashSet ) map.get( "fields" ); HashSet mapFields = ( HashSet ) map.get( "fields" );


if(defaultProperties!=null) { if(defaultProperties!=null) {
final Set<String> finalDefaultProperties = defaultProperties; final Set<String> finalDefaultProperties = defaultProperties;
//the iterator is based off of this valu
Iterator collectionIterator = mapFields.iterator(); Iterator collectionIterator = mapFields.iterator();

//TODO:I think there is a bug when we modify it and continue iterating then it fails
//Loop that goes through all the fields that the passed in entity contains
while ( collectionIterator.hasNext() ) { while ( collectionIterator.hasNext() ) {
EntityField testedField = ( EntityField ) collectionIterator.next(); EntityField testedField = ( EntityField ) collectionIterator.next();
String fieldName = ( String ) (testedField).get( "name" ); String fieldName = ( String ) (testedField).get( "name" );


//TODO: need to change the logic here such that default properties does a check against each every field.
//it is very likely that the below does O(n) comparisons anyways so doing it for each value in a loop
//would be the right equivalent here. Even if it isn't very efficient.
//maybe if it was a hashmap, and if the hashmap didn't contain the value then we remove the field.
//but now for each hash value we have to do a check to see it doesn't start with a certain value.
//making this an O(n) operation anyways.

//so for each value in the finalDefaultProperties check to see if it matches the current property
//then if it doesn't do a split of the properties and see if any of them match for the split
//if they do match then allow in, otherwise don't allow.

//will only enter the loop for properties that aren't default properties so we only need to check //will only enter the loop for properties that aren't default properties so we only need to check
//the properties that the user imposed. //the properties that the user imposed.
if ( !finalDefaultProperties.contains( fieldName ) ) { if ( !finalDefaultProperties.contains( fieldName ) ) {
innerLoop( fieldsToKeep, mapFields, testedField, fieldName );
}
}
}
return map;
}


//loop through and string split it all and if they still dont' match after that loop then remove them.
boolean toRemoveFlag = true;
String[] flattedStringArray = fieldName.split( "\\." );
Iterator fieldIterator = fieldsToKeep.iterator();
while(fieldIterator.hasNext()) {
String requiredInclusionString = ( String ) fieldIterator.next();
String[] flattedRequirementString = new String[0];

if(!requiredInclusionString.contains( "." )){
flattedRequirementString = new String[]{requiredInclusionString};
}
else {
flattedRequirementString= requiredInclusionString.split( "\\." );
}

//what we're aiming to do here is check the entire length of the string against
//
for ( int index = 0; index < flattedRequirementString.length;index++){
//if the array contains a string that it is equals to then set the remove flag to true
//otherwise remain false.

//TODO: document that whatever comes first in the index will be filtered out in the way specified
//so if the filtering spec had to filter one and one.two.three. and there is a value of
// one.three. Then it will be filtered out because the rule one.two.three is applied
//without a reference or care that one exists.
if(flattedStringArray.length <= index){
toRemoveFlag = true;
break;
}
//this has a condition issue where if we evaluate that it passes on the first pass, we dont' want to continue the while lo
if(flattedRequirementString[index].equals( flattedStringArray[index] )){
toRemoveFlag=false;
}
else{
toRemoveFlag=true;
}
}
if(toRemoveFlag==false){
break;
}
}


if(toRemoveFlag){ private void innerLoop( final ArrayList fieldsToKeep, final HashSet mapFields, final EntityField testedField,
mapFields.remove( testedField ); final String fieldName ) {//loop through and string split it all and if they still dont' match after that loop then remove them.
} boolean toRemoveFlag = true;
String[] flattedStringArray = null;


} if(!fieldName.contains( "." )){
//create a single array that is made of a the single value.
flattedStringArray = new String[]{fieldName};
}
else {
flattedStringArray= fieldName.split( "\\." );
}


Iterator fieldIterator = fieldsToKeep.iterator();


// Iterator iterator = finalDefaultProperties.iterator(); //goes through a loop of all the fields ( excluding default ) that we want to query on
// while(iterator.hasNext()){ while(fieldIterator.hasNext()) {
// String includedFieldName = ( String ) iterator.next(); String requiredInclusionString = ( String ) fieldIterator.next();
// //if the field name String[] flattedRequirementString;
// if(!includedFieldName.equals( fieldName )){
// //in the case you only have a filter that is a single field.Otherwise split it.
// } if(!requiredInclusionString.contains( "." )){
// flattedRequirementString = new String[]{requiredInclusionString};
// } }

else {
// for(int index = 0; index < finalDefaultProperties.size();index++){ flattedRequirementString= requiredInclusionString.split( "\\." );
// finalDefaultProperties. }
//
// }

//loop each split array value to see if it matches an equivalent value
// //in the field.
// if ( !finalDefaultProperties.contains( fieldName ) ) { for ( int index = 0; index < flattedRequirementString.length;index++){
// mapFields.remove( testedField ); //if the array contains a string that it is equals to then set the remove flag to true
// } //otherwise remain false.
// else{
// if(flattedStringArray.length <= index){
// } toRemoveFlag = true;
break;
}
//this has a condition issue where if we evaluate that it passes on the first pass, we dont' want to continue the while lo
if(flattedRequirementString[index].equals( flattedStringArray[index] )){
toRemoveFlag=false;
}
else{
toRemoveFlag=true;
}
}
if(toRemoveFlag==false){
break;
} }
} }
return map;
if(toRemoveFlag){
mapFields.remove( testedField );
}
} }




Expand Down
54 changes: 54 additions & 0 deletions ...src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
View file
Expand Up @@ -327,6 +327,60 @@ public void postToCollectionSchemaArrayWithTopLevelIndexing() throws Exception {


} }


@Test
public void postToCollectionSchemaArrayWithSelectiveTopLevelIndexing() throws Exception {

//Include the property labeled two to be index.
ArrayList<String> indexingArray = new ArrayList<>( );
indexingArray.add( "two" );
//this should work such that one.key and one.anotherKey should work.
indexingArray.add( "one.key" );

//field "fields" is required.
Entity payload = new Entity();
payload.put( "fields", indexingArray);

//Post index to the collection metadata
this.app().collection( "testCollection" ).collection( "_indexes" ).post( payload );
refreshIndex();

Map<String,Object> arrayFieldsForTestingSelectiveIndexing = new HashMap<>();

arrayFieldsForTestingSelectiveIndexing.put( "wowMoreKeys","value" );
arrayFieldsForTestingSelectiveIndexing.put( "thisShouldBeQueryableToo","value2");

Map<String,Object> arrayFieldsForTesting = new HashMap<>();

arrayFieldsForTesting.put( "key",arrayFieldsForTestingSelectiveIndexing );
arrayFieldsForTesting.put( "anotherKey","value2");

//Create test collection with a test entity that is partially indexed.
Entity testEntity = new Entity();
testEntity.put( "one", arrayFieldsForTesting );
testEntity.put( "two","query" );

//Post entity.
this.app().collection( "testCollection" ).post( testEntity );
refreshIndex();

//Do a query to see if you can find the indexed query.
String query = "one.key.wowMoreKeys = 'value'";
QueryParameters queryParameters = new QueryParameters().setQuery(query);

//having a name breaks it. Need to get rid of the stack trace and also
Collection tempEntity = this.app().collection( "testCollection" ).get(queryParameters,true);
Entity reindexedEntity = tempEntity.getResponse().getEntity();
assertEquals( "value2",((Map)reindexedEntity.get( "one" )).get( "anotherKey" ) );

//Verify if you can query on an entity that was not indexed and that no entities are returned.
//TODO: check that the below gets indexed as well. although the above should prove that at least one thing is getting indexed.
query = "one.anotherKey = 'value2'";
queryParameters = new QueryParameters().setQuery(query);
tempEntity = this.app().collection( "testCollection" ).get(queryParameters,true);
assertEquals(0,tempEntity.getResponse().getEntities().size());

}







Expand Down

0 comments on commit 1491264

Please sign in to comment.