Skip to content
This repository has been archived by the owner on Jan 5, 2022. It is now read-only.

Commit

Permalink
Ignore users with no orgs or creds, log total orgs exported and bette…
Browse files Browse the repository at this point in the history 
…r logging.
  • Loading branch information
@snoopdave
snoopdave committed Jul 17, 2015
1 parent 16de78d commit 125ffe9
Show file tree
Hide file tree
Showing 2 changed files with 78 additions and 41 deletions.
117 changes: 77 additions & 40 deletions stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
View file
Expand Up @@ -61,15 +61,25 @@
* cassandra.lock.keyspace=My_Usergrid_Locks
*/
public class ExportAdmins extends ExportingToolBase {

static final Logger logger = LoggerFactory.getLogger( ExportAdmins.class );

public static final String ADMIN_USERS_PREFIX = "admin-users";
public static final String ADMIN_USER_METADATA_PREFIX = "admin-user-metadata";

// map admin user UUID to list of organizations to which user belongs
private Map<UUID, List<Org>> userToOrgsMap = new HashMap<UUID, List<Org>>(50000);

private Map<String, UUID> orgNameToUUID = new HashMap<String, UUID>(50000);

private Set<UUID> orgsWritten = new HashSet<UUID>(50000);

private Set<UUID> duplicateOrgs = new HashSet<UUID>();

private static final String READ_THREAD_COUNT = "readThreads";
private Map<String, List<Org>> orgMap = new HashMap<String, List<Org>>(80000);
private int readThreadCount;
AtomicInteger count = new AtomicInteger( 0 );

AtomicInteger userCount = new AtomicInteger( 0 );


/**
Expand Down Expand Up @@ -173,7 +183,7 @@ public void runTool(CommandLine line) throws Exception {
while ( !done ) {
writeThread.join( 10000, 0 );
done = !writeThread.isAlive();
logger.info( "Wrote {} users", count.get() );
logger.info( "Wrote {} users", userCount.get() );
}
}

Expand Down Expand Up @@ -211,19 +221,22 @@ private void buildOrgMap() throws Exception {
organizations = em.searchCollection( em.getApplicationRef(), "groups", query );
for ( Entity organization : organizations.getEntities() ) {
execService.submit( new OrgMapWorker( organization ) );
count++;
}
count++;
if ( count % 1000 == 0 ) {
logger.info("Processed {} orgs for org map", count);
logger.info("Queued {} org map workers", count);
}
query.setCursor( organizations.getCursor() );
}
while ( organizations != null && organizations.hasCursor() );

execService.shutdown();
while ( !execService.awaitTermination( 10, TimeUnit.SECONDS ) ) {
logger.info("Processed {} orgs for map", orgMap.size() );
logger.info( "Processed {} orgs for map", userToOrgsMap.size() );
}

logger.info("Org map complete, counted {} organizations", count);
}


Expand All @@ -239,17 +252,33 @@ public void run() {
try {
final String orgName = orgEntity.getProperty( "path" ).toString();
final UUID orgId = orgEntity.getUuid();

for (UserInfo user : managementService.getAdminUsersForOrganization( orgEntity.getUuid() )) {
try {
Entity admin = managementService.getAdminUserEntityByUuid( user.getUuid() );
List<Org> orgs = orgMap.get( admin.getProperty( "username" ) );
if (orgs == null) {
orgs = new ArrayList<Org>();
orgMap.put( admin.getProperty( "username" ).toString().toLowerCase(), orgs );
Org org = new Org( orgId, orgName );

synchronized (userToOrgsMap) {
List<Org> userOrgs = userToOrgsMap.get( admin.getUuid() );
if (userOrgs == null) {
userOrgs = new ArrayList<Org>();
userToOrgsMap.put( admin.getUuid(), userOrgs );
}
userOrgs.add( org );
}
orgs.add( new Org( orgId, orgName ) );

//logger.debug("Added org {} for user {}", orgName, admin.getProperty( "username" ));
synchronized (orgNameToUUID) {
UUID existingOrgId = orgNameToUUID.get( orgName );
;
if (existingOrgId != null && !orgId.equals( existingOrgId )) {
if ( !duplicateOrgs.contains( orgId )) {
logger.info( "Org {}:{} is a duplicate", orgId, orgName );
duplicateOrgs.add(orgId);
}
} else {
orgNameToUUID.put( orgName, orgId );
}
}

} catch (Exception e) {
logger.warn( "Cannot get orgs for userId {}", user.getUuid() );
Expand Down Expand Up @@ -301,10 +330,33 @@ private void readAndQueueAdminUsers() throws Exception {
AdminUserWriteTask task = new AdminUserWriteTask();
task.adminUser = entity;

addDictionariesToTask( task, entity );
addDictionariesToTask( task, entity );
addOrganizationsToTask( task );

writeQueue.add( task );
String actionTaken = "Processed";

if (task.orgNamesByUuid.isEmpty()
|| task.dictionariesByName.isEmpty()
|| task.dictionariesByName.get( "credentials" ).isEmpty()) {
actionTaken = "Ignored";

} else {
writeQueue.add( task );
}

Map<String, Object> creds = (Map<String, Object>) (task.dictionariesByName.isEmpty() ?
0 : task.dictionariesByName.get( "credentials" ));

logger.error( "{} admin user {}:{}:{} has organizations={} dictionaries={} credentials={}",
new Object[]{
actionTaken,
task.adminUser.getProperty( "username" ),
task.adminUser.getProperty( "email" ),
task.adminUser.getUuid(),
task.orgNamesByUuid.size(),
task.dictionariesByName.size(),
creds == null ? 0 : creds.size()
} );

} catch ( Exception e ) {
logger.error("Error reading data for user " + uuid, e );
Expand All @@ -327,43 +379,26 @@ private void addDictionariesToTask(AdminUserWriteTask task, Entity entity) throw

Map<Object, Object> credentialsDictionary = em.getDictionaryAsMap( entity, "credentials" );

if ( credentialsDictionary != null && !credentialsDictionary.isEmpty() ) {
if ( credentialsDictionary != null ) {
task.dictionariesByName.put( "credentials", credentialsDictionary );

if (credentialsDictionary.get( "password" ) == null) {
logger.error( "User {}:{} has no password in credential dictionary",
new Object[]{task.adminUser.getName(), task.adminUser.getUuid()} );
}
if (credentialsDictionary.get( "secret" ) == null) {
logger.error( "User {}:{} has no secret in credential dictionary",
new Object[]{task.adminUser.getName(), task.adminUser.getUuid()} );
}
} else {
logger.error( "User {}:{} has no or empty credentials dictionary",
new Object[]{task.adminUser.getName(), task.adminUser.getUuid()} );
}
}

private void addOrganizationsToTask(AdminUserWriteTask task) throws Exception {

task.orgNamesByUuid = managementService.getOrganizationsForAdminUser( task.adminUser.getUuid() );

List<Org> orgs = orgMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase() );
List<Org> orgs = userToOrgsMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase() );

if ( orgs != null && task.orgNamesByUuid.size() < orgs.size() ) {

// list of orgs from getOrganizationsForAdminUser() is less than expected, use userToOrgsMap
BiMap<UUID, String> bimap = HashBiMap.create();
for (Org org : orgs) {
bimap.put( org.orgId, org.orgName );
}
task.orgNamesByUuid = bimap;
}

if ( task.orgNamesByUuid.isEmpty() ) {
logger.error("{}:{}:{} has no orgs", new Object[] {
task.adminUser.getProperty("username"),
task.adminUser.getProperty("email"),
task.adminUser.getUuid() } );
}
}
}

Expand Down Expand Up @@ -425,7 +460,7 @@ private void writeEntities() throws Exception {
task.adminUser.getProperty("email"),
task.adminUser.getUuid() } );

count.addAndGet( 1 );
userCount.addAndGet( 1 );

} catch (InterruptedException e) {
throw new Exception("Interrupted", e);
Expand All @@ -438,7 +473,7 @@ private void writeEntities() throws Exception {
usersFile.writeEndArray();
usersFile.close();

logger.info( "Exported TOTAL {} admin users", count );
logger.info( "Exported TOTAL {} admin users and {} organizations", userCount.get(), orgsWritten.size() );
}


Expand Down Expand Up @@ -490,7 +525,9 @@ private void saveOrganizations( JsonGenerator jg, AdminUserWriteTask task ) thro

jg.writeEndObject();

logger.debug( "Exported organization {}:{}", uuid, orgs.get( uuid ) );
synchronized (orgsWritten) {
orgsWritten.add( uuid );
}
}

jg.writeEndArray();
Expand Down
2 changes: 1 addition & 1 deletion stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
View file
Expand Up @@ -636,7 +636,7 @@ public void run() {
long duration = stopTime - startTime;
durationSum += duration;

//logger.debug( "Audited {}th admin", count );
//logger.debug( "Audited {}th admin", userCount );

if ( count % 100 == 0 ) {
logger.info( "Audited {}. Average Audit Rate: {}(ms)", count, durationSum / count );
Expand Down

0 comments on commit 125ffe9

Please sign in to comment.