diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java index ae9c16b443..9dce86265e 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java @@ -61,15 +61,25 @@ * cassandra.lock.keyspace=My_Usergrid_Locks */ public class ExportAdmins extends ExportingToolBase { - + static final Logger logger = LoggerFactory.getLogger( ExportAdmins.class ); + public static final String ADMIN_USERS_PREFIX = "admin-users"; public static final String ADMIN_USER_METADATA_PREFIX = "admin-user-metadata"; + + // map admin user UUID to list of organizations to which user belongs + private Map> userToOrgsMap = new HashMap>(50000); + + private Map orgNameToUUID = new HashMap(50000); + + private Set orgsWritten = new HashSet(50000); + + private Set duplicateOrgs = new HashSet(); + private static final String READ_THREAD_COUNT = "readThreads"; - private Map> orgMap = new HashMap>(80000); private int readThreadCount; - - AtomicInteger count = new AtomicInteger( 0 ); + + AtomicInteger userCount = new AtomicInteger( 0 ); /** @@ -173,7 +183,7 @@ public void runTool(CommandLine line) throws Exception { while ( !done ) { writeThread.join( 10000, 0 ); done = !writeThread.isAlive(); - logger.info( "Wrote {} users", count.get() ); + logger.info( "Wrote {} users", userCount.get() ); } } @@ -211,10 +221,11 @@ private void buildOrgMap() throws Exception { organizations = em.searchCollection( em.getApplicationRef(), "groups", query ); for ( Entity organization : organizations.getEntities() ) { execService.submit( new OrgMapWorker( organization ) ); + count++; } - count++; + if ( count % 1000 == 0 ) { - logger.info("Processed {} orgs for org map", count); + logger.info("Queued {} org map workers", count); } query.setCursor( organizations.getCursor() ); } @@ -222,8 +233,10 @@ private void buildOrgMap() throws Exception { execService.shutdown(); while ( !execService.awaitTermination( 10, TimeUnit.SECONDS ) ) { - logger.info("Processed {} orgs for map", orgMap.size() ); + logger.info( "Processed {} orgs for map", userToOrgsMap.size() ); } + + logger.info("Org map complete, counted {} organizations", count); } @@ -239,17 +252,33 @@ public void run() { try { final String orgName = orgEntity.getProperty( "path" ).toString(); final UUID orgId = orgEntity.getUuid(); + for (UserInfo user : managementService.getAdminUsersForOrganization( orgEntity.getUuid() )) { try { Entity admin = managementService.getAdminUserEntityByUuid( user.getUuid() ); - List orgs = orgMap.get( admin.getProperty( "username" ) ); - if (orgs == null) { - orgs = new ArrayList(); - orgMap.put( admin.getProperty( "username" ).toString().toLowerCase(), orgs ); + Org org = new Org( orgId, orgName ); + + synchronized (userToOrgsMap) { + List userOrgs = userToOrgsMap.get( admin.getUuid() ); + if (userOrgs == null) { + userOrgs = new ArrayList(); + userToOrgsMap.put( admin.getUuid(), userOrgs ); + } + userOrgs.add( org ); } - orgs.add( new Org( orgId, orgName ) ); - //logger.debug("Added org {} for user {}", orgName, admin.getProperty( "username" )); + synchronized (orgNameToUUID) { + UUID existingOrgId = orgNameToUUID.get( orgName ); + ; + if (existingOrgId != null && !orgId.equals( existingOrgId )) { + if ( !duplicateOrgs.contains( orgId )) { + logger.info( "Org {}:{} is a duplicate", orgId, orgName ); + duplicateOrgs.add(orgId); + } + } else { + orgNameToUUID.put( orgName, orgId ); + } + } } catch (Exception e) { logger.warn( "Cannot get orgs for userId {}", user.getUuid() ); @@ -301,10 +330,33 @@ private void readAndQueueAdminUsers() throws Exception { AdminUserWriteTask task = new AdminUserWriteTask(); task.adminUser = entity; - addDictionariesToTask( task, entity ); + addDictionariesToTask( task, entity ); addOrganizationsToTask( task ); - writeQueue.add( task ); + String actionTaken = "Processed"; + + if (task.orgNamesByUuid.isEmpty() + || task.dictionariesByName.isEmpty() + || task.dictionariesByName.get( "credentials" ).isEmpty()) { + actionTaken = "Ignored"; + + } else { + writeQueue.add( task ); + } + + Map creds = (Map) (task.dictionariesByName.isEmpty() ? + 0 : task.dictionariesByName.get( "credentials" )); + + logger.error( "{} admin user {}:{}:{} has organizations={} dictionaries={} credentials={}", + new Object[]{ + actionTaken, + task.adminUser.getProperty( "username" ), + task.adminUser.getProperty( "email" ), + task.adminUser.getUuid(), + task.orgNamesByUuid.size(), + task.dictionariesByName.size(), + creds == null ? 0 : creds.size() + } ); } catch ( Exception e ) { logger.error("Error reading data for user " + uuid, e ); @@ -327,20 +379,8 @@ private void addDictionariesToTask(AdminUserWriteTask task, Entity entity) throw Map credentialsDictionary = em.getDictionaryAsMap( entity, "credentials" ); - if ( credentialsDictionary != null && !credentialsDictionary.isEmpty() ) { + if ( credentialsDictionary != null ) { task.dictionariesByName.put( "credentials", credentialsDictionary ); - - if (credentialsDictionary.get( "password" ) == null) { - logger.error( "User {}:{} has no password in credential dictionary", - new Object[]{task.adminUser.getName(), task.adminUser.getUuid()} ); - } - if (credentialsDictionary.get( "secret" ) == null) { - logger.error( "User {}:{} has no secret in credential dictionary", - new Object[]{task.adminUser.getName(), task.adminUser.getUuid()} ); - } - } else { - logger.error( "User {}:{} has no or empty credentials dictionary", - new Object[]{task.adminUser.getName(), task.adminUser.getUuid()} ); } } @@ -348,22 +388,17 @@ private void addOrganizationsToTask(AdminUserWriteTask task) throws Exception { task.orgNamesByUuid = managementService.getOrganizationsForAdminUser( task.adminUser.getUuid() ); - List orgs = orgMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase() ); + List orgs = userToOrgsMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase() ); if ( orgs != null && task.orgNamesByUuid.size() < orgs.size() ) { + + // list of orgs from getOrganizationsForAdminUser() is less than expected, use userToOrgsMap BiMap bimap = HashBiMap.create(); for (Org org : orgs) { bimap.put( org.orgId, org.orgName ); } task.orgNamesByUuid = bimap; } - - if ( task.orgNamesByUuid.isEmpty() ) { - logger.error("{}:{}:{} has no orgs", new Object[] { - task.adminUser.getProperty("username"), - task.adminUser.getProperty("email"), - task.adminUser.getUuid() } ); - } } } @@ -425,7 +460,7 @@ private void writeEntities() throws Exception { task.adminUser.getProperty("email"), task.adminUser.getUuid() } ); - count.addAndGet( 1 ); + userCount.addAndGet( 1 ); } catch (InterruptedException e) { throw new Exception("Interrupted", e); @@ -438,7 +473,7 @@ private void writeEntities() throws Exception { usersFile.writeEndArray(); usersFile.close(); - logger.info( "Exported TOTAL {} admin users", count ); + logger.info( "Exported TOTAL {} admin users and {} organizations", userCount.get(), orgsWritten.size() ); } @@ -490,7 +525,9 @@ private void saveOrganizations( JsonGenerator jg, AdminUserWriteTask task ) thro jg.writeEndObject(); - logger.debug( "Exported organization {}:{}", uuid, orgs.get( uuid ) ); + synchronized (orgsWritten) { + orgsWritten.add( uuid ); + } } jg.writeEndArray(); diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java index c6aada7aee..7e08a4c52a 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java @@ -636,7 +636,7 @@ public void run() { long duration = stopTime - startTime; durationSum += duration; - //logger.debug( "Audited {}th admin", count ); + //logger.debug( "Audited {}th admin", userCount ); if ( count % 100 == 0 ) { logger.info( "Audited {}. Average Audit Rate: {}(ms)", count, durationSum / count );