Skip to content
This repository has been archived by the owner on Jan 5, 2022. It is now read-only.

Commit

Permalink
add deletion of orphans to collection iterator
Browse files Browse the repository at this point in the history
  • Loading branch information
Mike Dunker committed Aug 26, 2016
1 parent fd7a75e commit cda2879
Showing 1 changed file with 81 additions and 26 deletions.
107 changes: 81 additions & 26 deletions stack/tools/src/main/java/org/apache/usergrid/tools/CollectionIterator.java
View file
Expand Up @@ -20,31 +20,19 @@
import java.text.DateFormat; import java.text.DateFormat;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.*; import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;


import com.google.common.base.*;
import com.google.common.base.Optional; import com.google.common.base.Optional;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.model.Column;
import com.netflix.astyanax.model.ConsistencyLevel;
import com.netflix.astyanax.util.RangeBuilder;
import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage; import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
import org.apache.usergrid.corepersistence.results.EntityQueryExecutor;
import org.apache.usergrid.corepersistence.results.IdQueryExecutor; import org.apache.usergrid.corepersistence.results.IdQueryExecutor;
import org.apache.usergrid.corepersistence.service.CollectionSearch; import org.apache.usergrid.corepersistence.service.CollectionSearch;
import org.apache.usergrid.corepersistence.service.CollectionService; import org.apache.usergrid.corepersistence.service.CollectionService;
import org.apache.usergrid.persistence.*; import org.apache.usergrid.persistence.*;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
import org.apache.usergrid.persistence.collection.serialization.impl.*;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl; import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.index.utils.UUIDUtils; import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.persistence.model.entity.*; import org.apache.usergrid.persistence.model.entity.*;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.field.StringField;
import org.apache.usergrid.persistence.schema.CollectionInfo; import org.apache.usergrid.persistence.schema.CollectionInfo;
import org.apache.usergrid.utils.InflectionUtils; import org.apache.usergrid.utils.InflectionUtils;
import static org.apache.commons.lang.StringUtils.isBlank;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;


Expand All @@ -54,11 +42,6 @@
import org.apache.commons.cli.Options; import org.apache.commons.cli.Options;




import org.apache.usergrid.persistence.core.astyanax.MultiTenantColumnFamily;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer;
import rx.*;

import static org.apache.usergrid.persistence.Schema.getDefaultSchema; import static org.apache.usergrid.persistence.Schema.getDefaultSchema;




Expand All @@ -72,6 +55,14 @@ public class CollectionIterator extends ToolBase {


private static final String REMOVE_CONNECTIONS_ARG = "removeConnections"; private static final String REMOVE_CONNECTIONS_ARG = "removeConnections";


private static final String LATEST_TIMESTAMP_ARG = "latestTimestamp";

private static final String EARLIEST_TIMESTAMP_ARG = "earliestTimestamp";

private static final String SECONDS_IN_PAST_ARG = "secondsInPast";

private static final Long DEFAULT_SECONDS_IN_PAST = 60L * 60L; // hour

private EntityManager em; private EntityManager em;


@Override @Override
Expand All @@ -95,11 +86,29 @@ public Options createOptions() {
options.addOption( collectionOption ); options.addOption( collectionOption );


Option removeConnectionsOption = Option removeConnectionsOption =
OptionBuilder.withArgName(REMOVE_CONNECTIONS_ARG).hasArg().isRequired( false ).withDescription( "remove orphaned connections" ) OptionBuilder.withArgName(REMOVE_CONNECTIONS_ARG).hasArg().isRequired( false ).withDescription( "remove orphaned connections" )
.create(REMOVE_CONNECTIONS_ARG); .create(REMOVE_CONNECTIONS_ARG);


options.addOption( removeConnectionsOption ); options.addOption( removeConnectionsOption );


Option earliestTimestampOption =
OptionBuilder.withArgName(EARLIEST_TIMESTAMP_ARG).hasArg().isRequired( false ).withDescription( "earliest timestamp to delete" )
.create(EARLIEST_TIMESTAMP_ARG);

options.addOption( earliestTimestampOption );

Option latestTimestampOption =
OptionBuilder.withArgName(LATEST_TIMESTAMP_ARG).hasArg().isRequired( false ).withDescription( "latest timestamp to delete" )
.create(LATEST_TIMESTAMP_ARG);

options.addOption( latestTimestampOption );

Option secondsInPastOption =
OptionBuilder.withArgName(SECONDS_IN_PAST_ARG).hasArg().isRequired( false ).withDescription( "how many seconds old orphan must be to be deleted" )
.create(SECONDS_IN_PAST_ARG);

options.addOption( secondsInPastOption );



return options; return options;
} }
Expand All @@ -116,18 +125,62 @@ public void runTool( CommandLine line ) throws Exception {


startSpring(); startSpring();


if (line.getOptionValue(APPLICATION_ARG).isEmpty()) { String applicationOption = line.getOptionValue(APPLICATION_ARG);
String entityTypeOption = line.getOptionValue(ENTITY_TYPE_ARG);
String removeConnectionsOption = line.getOptionValue(REMOVE_CONNECTIONS_ARG);
String earliestTimestampOption = line.getOptionValue(EARLIEST_TIMESTAMP_ARG);
String latestTimestampOption = line.getOptionValue(LATEST_TIMESTAMP_ARG);
String secondsInPastOption = line.getOptionValue(SECONDS_IN_PAST_ARG);

if (isBlank(applicationOption)) {
throw new RuntimeException("Application ID not provided."); throw new RuntimeException("Application ID not provided.");
} }
final UUID app = UUID.fromString(line.getOptionValue(APPLICATION_ARG)); final UUID app = UUID.fromString(line.getOptionValue(APPLICATION_ARG));


String removeOrphansOption = line.getOptionValue(REMOVE_CONNECTIONS_ARG); if (isBlank(entityTypeOption)) {
final boolean removeOrphans = !removeOrphansOption.isEmpty() && removeOrphansOption.toLowerCase().equals("yes"); throw new RuntimeException("Entity type (singular collection name) not provided.");
}
String entityType = entityTypeOption;

final boolean removeOrphans = !isBlank(removeConnectionsOption) && removeConnectionsOption.toLowerCase().equals("yes");

if (!isBlank(secondsInPastOption) && !isBlank(latestTimestampOption)) {
throw new RuntimeException("Can't specify both latest timestamp and seconds in past options.");
}


String entityType = line.getOptionValue(ENTITY_TYPE_ARG); long earliest = 0L;
if (!isBlank(earliestTimestampOption)) {
try {
earliest = Long.parseLong(earliestTimestampOption);
} catch (Exception e) {
throw new RuntimeException("Cannot convert earliest timestamp to long: " + earliestTimestampOption);
}
}
final long earliestTimestamp = earliest;

long currentTimestamp = System.currentTimeMillis();

// default to DEFAULT_SECONDS_IN_PAST
long latest = currentTimestamp - (DEFAULT_SECONDS_IN_PAST * 1000L);
if (!isBlank(latestTimestampOption)) {
try {
latest = Long.parseLong(latestTimestampOption);
} catch (Exception e) {
throw new RuntimeException("Cannot convert latest timestamp to long: " + latestTimestampOption);
}
} else if (!isBlank(secondsInPastOption)) {
try {
long secondsInPast = Long.parseLong(secondsInPastOption);
latest = currentTimestamp - (secondsInPast * 1000L);
} catch (Exception e) {
throw new RuntimeException("Cannot convert seconds in past to long: " + secondsInPastOption);
}
}
final long latestTimestamp = latest;


logger.info("Starting Tool: CollectionIterator"); logger.info("Starting Tool: CollectionIterator");
logger.info("Orphans {} be deleted", removeOrphans ? "WILL" : "will not"); logger.info("Orphans {} be deleted", removeOrphans ? "WILL" : "will not");
logger.info("Timestamp range {} to {}", Long.toString(earliestTimestamp), Long.toString(latestTimestamp));
logger.info("Using Cassandra consistency level: {}", System.getProperty("usergrid.read.cl", "CL_LOCAL_QUORUM")); logger.info("Using Cassandra consistency level: {}", System.getProperty("usergrid.read.cl", "CL_LOCAL_QUORUM"));


em = emf.getEntityManager( app ); em = emf.getEntityManager( app );
Expand Down Expand Up @@ -184,15 +237,17 @@ protected rx.Observable<ResultsPage<Id>> buildNewResultsPage(


logger.info("{} - {} - entity data found", uuid, dateString); logger.info("{} - {} - entity data found", uuid, dateString);
}else{ }else{
if (removeOrphans) { if (removeOrphans && timestamp >= earliestTimestamp && timestamp <= latestTimestamp) {
logger.info("{} - {} - entity data NOT found, REMOVING", uuid, dateString); logger.info("{} - {} - entity data NOT found, REMOVING", uuid, dateString);
try { try {
em.removeFromCollection(headEntity, collectionName, entityRef ); em.removeFromCollection(headEntity, collectionName, entityRef );
} catch (Exception e) { } catch (Exception e) {
logger.error("{} - exception while trying to remove orphaned connection, {}", uuid, e.getMessage()); logger.error("{} - exception while trying to remove orphaned connection, {}", uuid, e.getMessage());
} }
} else if (removeOrphans) {
logger.info("{} - {} ({}) - entity data NOT found, not removing because timestamp not in range", uuid, dateString, timestamp);
} else { } else {
logger.info("{} - {} - entity data NOT found", uuid, dateString); logger.info("{} - {} ({}) - entity data NOT found", uuid, dateString, timestamp);
} }
} }
} catch (Exception e) { } catch (Exception e) {
Expand Down

0 comments on commit cda2879

Please sign in to comment.