Skip to content

Commit

Permalink
merged from eee
Browse files Browse the repository at this point in the history
  • Loading branch information
@dmitrii
dmitrii committed Jan 27, 2012
2 parents 03d4bec + 622a215 commit 9f40094
Show file tree
Hide file tree
Showing 20 changed files with 449 additions and 113 deletions.
2 changes: 1 addition & 1 deletion clc/modules/cluster-manager/src/main/java/com/eucalyptus/vm/VmInstances.java
View file
Expand Up @@ -232,7 +232,7 @@ protected boolean inState( final VmState state ) {
public static String INSTANCE_SUBDOMAIN = ".eucalyptus";
@ConfigurableField( description = "Period (in seconds) between state updates for actively changing state.",
initial = "3" )
public static Long VOLATILE_STATE_INTERVAL_SEC = 3l;
public static Long VOLATILE_STATE_INTERVAL_SEC = Long.MAX_VALUE;
@ConfigurableField( description = "Timeout (in seconds) before a requested instance terminate will be repeated.",
initial = "60" )
public static Long VOLATILE_STATE_TIMEOUT_SEC = 60l;
Expand Down
43 changes: 29 additions & 14 deletions clc/modules/core/conf/upgrade/upgrade_20_30.groovy
View file
Expand Up @@ -25,6 +25,7 @@ import javax.persistence.Table;

import org.apache.log4j.Logger;
import org.mortbay.log.Log;
import org.hibernate.exception.ConstraintViolationException;

// For cluster / node keypairs
import java.security.KeyPair;
Expand Down Expand Up @@ -379,7 +380,7 @@ class upgrade_20_30 extends AbstractUpgradeScript {
}
for (String column : columnNames) {
if(!setterMap.containsKey(column) && !unmappedColumns.contains("${entityKey}.${column}".toString()) ) {
LOG.warn("No corresponding field for column: ${entityKey}.${column} found");
LOG.info("No corresponding field for column: ${entityKey}.${column} found");
}
}
if (entityKey.equals('vm_types')) {
Expand Down Expand Up @@ -478,10 +479,10 @@ class upgrade_20_30 extends AbstractUpgradeScript {
accountProxy = new DatabaseAccountProxy(account);
safeUserMap.put(it.auth_user_name, accountName);
accountIdMap.put(accountName, accountProxy.getAccountNumber());
} catch (AuthException e) {
} catch (ConstraintViolationException e) {
// The account already existed
userName = userName + "-";
accountName = userName;
account = null;
accountName += '-';
}
}

Expand All @@ -503,13 +504,18 @@ class upgrade_20_30 extends AbstractUpgradeScript {
if (uInfo != null) {
Map<String, String> info = new HashMap<String, String>( );
userInfoFields.each { k,v ->
if (uInfo[k] != null) { info.put(v, uInfo[k]); }
if (uInfo[k] != null) {
def truncatedInfo = uInfo[k].size() > 255 ? uInfo[k][0..254] : uInfo[k];
info.put(v, truncatedInfo);
}
}
LOG.debug("Setting user info: " + info);
user.setInfo( info );
}

EntityWrapper<UserEntity> dbUE = EntityWrapper.get(UserEntity.class);
def ue = DatabaseAuthUtils.getUniqueUser(dbUE, userName, accountName);
dbUE.rollback();
EntityWrapper<AccessKeyEntity> dbAuth = EntityWrapper.get(AccessKeyEntity.class);
AccessKeyEntity accessKey = new AccessKeyEntity();
initMetaClass(accessKey, accessKey.class);
Expand Down Expand Up @@ -628,17 +634,26 @@ class upgrade_20_30 extends AbstractUpgradeScript {

public boolean upgradeKeyPairs() {
connMap['eucalyptus_general'].rows('SELECT * FROM metadata_keypair').each{
EntityWrapper<SshKeyPair> dbkp = EntityWrapper.get(SshKeyPair.class);
if (!userIdMap.containsKey(it.metadata_user_name)) {
return;
}
UserFullName ufn = UserFullName.getInstance(userIdMap.get(it.metadata_user_name));
SshKeyPair kp = new SshKeyPair( ufn,
it.metadata_keypair_user_keyname,
def keyname = it.metadata_keypair_user_keyname;
def committed = false;
while (!committed) {
try {
EntityWrapper<SshKeyPair> dbkp = EntityWrapper.get(SshKeyPair.class);
SshKeyPair kp = new SshKeyPair( ufn,
keyname,
it.metadata_keypair_public_key,
it.metadata_keypair_finger_print );
dbkp.add(kp);
dbkp.commit();
dbkp.add(kp);
dbkp.commit();
committed = true;
} catch (ConstraintViolationException e) {
keyname += "-";
}
}
}
}

Expand Down Expand Up @@ -838,11 +853,11 @@ class upgrade_20_30 extends AbstractUpgradeScript {
ON peer.network_rule_peer_network_id=metadata_network_rule_has_peer_network.metadata_network_rule_peer_network_id
WHERE metadata_network_rule_has_peer_network.metadata_network_rule_id=?""", [ rule.metadata_network_rule_id ]).each { peer ->
peers.put(peer.network_rule_peer_network_user_query_key, peer.network_rule_peer_network_user_group);
LOG.debug("Peer: " + networkPeer);
// LOG.debug("Peer: " + networkPeer);
}
NetworkRule networkRule = NetworkRule.create(rule.metadata_network_rule_protocol,
rule.metadata_network_rule_low_port,
rule.metadata_network_rule_high_port,
NetworkRule networkRule = NetworkRule.create(rule.metadata_network_rule_protocol.toLowerCase(),
[rule.metadata_network_rule_low_port, 0].max(),
[[rule.metadata_network_rule_high_port, 65535].min(), 0].max(),
peers as Multimap<String, String>,
ipRanges as Collection<String>);
initMetaClass(networkRule, networkRule.class);
Expand Down
2 changes: 1 addition & 1 deletion clc/modules/dns/src/main/java/com/eucalyptus/dns/TransientZone.java
View file
Expand Up @@ -192,7 +192,7 @@ public SetResponse findRecords( Name name, int type ) {
} catch ( Exception e ) {
return super.findRecords( name, type );
}
} else if (name.toString().startsWith("eucalyptus.")) {
} else if (name.toString().startsWith("eucalyptus.") || (name.toString().startsWith("euare."))) {
SetResponse resp = new SetResponse(SetResponse.SUCCESSFUL);
try {
InetAddress cloudIp = Topology.lookup( Eucalyptus.class ).getInetAddress( );
Expand Down
2 changes: 1 addition & 1 deletion clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/Databases.java
View file
Expand Up @@ -168,7 +168,7 @@ public DatabaseStateException( String string ) {
}

private static final int MAX_TX_START_SYNC_RETRIES = 120;
private static final AtomicInteger counter = new AtomicInteger( 120 );
private static final AtomicInteger counter = new AtomicInteger( 500 );
private static final Predicate<Host> FILTER_SYNCING_DBS = Predicates.and( DbFilter.INSTANCE, Predicates.not( SyncedDbFilter.INSTANCE ) );
private static final ScriptedDbBootstrapper singleton = new ScriptedDbBootstrapper( );
private static Logger LOG = Logger.getLogger( Databases.class );
Expand Down
5 changes: 5 additions & 0 deletions clc/modules/msgs/src/main/java/com/eucalyptus/component/ServiceUris.java
View file
Expand Up @@ -358,6 +358,11 @@ public static URI remotePublicify( ServiceConfiguration config, String... pathPa
return make( config.getComponentId( ), config.getInetAddress( ), config.getPort( ), pathParts ).getPublicify( );
}

public static URI remotePublicify( final Class<? extends ComponentId> idClass, String... pathParts ) {
Component comp = Components.lookup( idClass );
return make( comp.getComponentId(), Internets.localHostInetAddress( ), comp.getComponentId().getPort(), pathParts ).getPublicify();
}

private static UriParserBuilder makeInternal( ComponentId compId, final InetAddress host, Integer port, String... pathParts ) {
return new UriParserBuilder( compId ).scheme( compId.getTransports( ).iterator( ).next( ) ).host( host ).port( port ).path( compId.getInternalServicePath( pathParts ) );
}
Expand Down
52 changes: 26 additions & 26 deletions clc/modules/msgs/src/main/java/com/eucalyptus/component/Topology.java
View file
Expand Up @@ -74,9 +74,8 @@
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.log4j.Logger;
import com.eucalyptus.bootstrap.Bootstrap;
import com.eucalyptus.bootstrap.BootstrapArgs;
Expand Down Expand Up @@ -169,33 +168,34 @@ public <C> Future<C> enqueue( final Callable<C> call ) {
private enum TopologyTimer implements EventListener<ClockTick> {
INSTANCE;
private static final AtomicInteger counter = new AtomicInteger( 0 );
private static final Lock canHas = new ReentrantLock( );
private static final AtomicBoolean busy = new AtomicBoolean( false );

@Override
public void fireEvent( final ClockTick event ) {
if ( Hosts.isCoordinator( ) && canHas.tryLock( ) ) {
Queue.INTERNAL.enqueue( new Callable<Object>( ) {
public Object call( ) {
try {
return RunChecks.INSTANCE.call( );
} finally {
canHas.unlock( );
}
}
} );
} else if ( counter.incrementAndGet( ) % 3 == 0 && canHas.tryLock( ) ) {
Queue.INTERNAL.enqueue( new Callable<Object>( ) {
public Object call( ) {
try {
return RunChecks.INSTANCE.call( );
} finally {
canHas.unlock( );
}
Callable<Object> call = new Callable<Object>( ) {
public Object call( ) {
try {
TimeUnit.SECONDS.sleep( 10 );
return RunChecks.INSTANCE.call( );
} catch ( InterruptedException ex ) {
return Exceptions.maybeInterrupted( ex );
} finally {
busy.set( false );
}
} );
} else if ( counter.incrementAndGet( ) > 10 ) {
counter.set( 0 );
Queue.INTERNAL.enqueue( RunChecks.INSTANCE );
}
};
if ( Hosts.isCoordinator( ) && busy.compareAndSet( false, true ) ) {
try {
Queue.INTERNAL.enqueue( call );
} catch ( Exception ex ) {
busy.set( false );
}
} else if ( counter.incrementAndGet( ) % 3 == 0 && busy.compareAndSet( false, true ) ) {
try {
Queue.INTERNAL.enqueue( call );
} catch ( Exception ex ) {
busy.set( false );
}
}
}

Expand Down Expand Up @@ -730,7 +730,7 @@ enum WaitForResults implements Predicate<Future> {
@Override
public boolean apply( final Future input ) {
try {
final Object conf = input.get( 30, TimeUnit.SECONDS );
final Object conf = input.get( 120, TimeUnit.SECONDS );
return true;
} catch ( final InterruptedException ex ) {
Thread.currentThread( ).interrupt( );
Expand Down
2 changes: 1 addition & 1 deletion clc/modules/reporting/src/main/java/com/eucalyptus/reporting/ReportGenerator.java
View file
Expand Up @@ -125,7 +125,7 @@ public void generateReport(String reportType, ReportFormat format,
List<ReportLine> reportLines = null;
ReportKey key = new ReportKey(reportType, period, criterion,
groupByCriterion, displayUnits, System.currentTimeMillis());
if (lineListMap.containsKey(key)) {
if (lineListMap.containsKey(key)) { //TODO: this isnt working all of the sudden even though hashcode and equals do not use timestampMs
log.info("Gathered report data from cache:" + key);
reportLines = lineListMap.get(key);
} else {
Expand Down
18 changes: 10 additions & 8 deletions clc/modules/reporting/src/main/java/com/eucalyptus/reporting/modules/s3/S3UsageLog.java
View file
Expand Up @@ -175,7 +175,8 @@ public Map<S3SummaryKey, S3UsageSummary> getUsageSummaryMap(Period period,
list = entityWrapper.createQuery(
"from S3UsageSnapshot as sus"
+ " WHERE sus.key.timestampMs > ?"
+ " AND sus.key.timestampMs < ?")
+ " AND sus.key.timestampMs < ?"
+ " ORDER BY sus.key.timestampMs")
.setLong(0, new Long(latestSnapshotBeforeMs))
.setLong(1, new Long(period.getEndingMs()))
.list();
Expand All @@ -184,7 +185,8 @@ public Map<S3SummaryKey, S3UsageSummary> getUsageSummaryMap(Period period,
"from S3UsageSnapshot as sus"
+ " WHERE sus.key.timestampMs > ?"
+ " AND sus.key.timestampMs < ?"
+ " AND sus.key.accountId = ?")
+ " AND sus.key.accountId = ?"
+ " ORDER BY sus.key.timestampMs")
.setLong(0, new Long(latestSnapshotBeforeMs))
.setLong(1, new Long(period.getEndingMs()))
.setString(2, accountId)
Expand Down Expand Up @@ -222,11 +224,11 @@ public Map<S3SummaryKey, S3UsageSummary> getUsageSummaryMap(Period period,
long endingMs = snapshotKey.getTimestampMs()-1;
long durationSecs = (endingMs - beginningMs) / 1000;

log.info(String.format("Accumulate usage, %d-%d, key:%s",
beginningMs, endingMs, summaryKey));
accumulator.accumulateUsage( durationSecs );
accumulator.setLastTimestamp(snapshotKey.getTimestampMs());
accumulator.setLastUsageData(snapshot.getUsageData());
log.info(String.format("Accumulate usage, %d-%d, key:%s",
beginningMs, endingMs, summaryKey));

}

Expand All @@ -241,9 +243,9 @@ public Map<S3SummaryKey, S3UsageSummary> getUsageSummaryMap(Period period,
accumulator.getLastTimestamp() );
long endingMs = period.getEndingMs() - 1;
long durationSecs = ( endingMs-beginningMs ) / 1000;
accumulator.accumulateUsage( durationSecs );
log.info(String.format("Accumulate endUsage, %d-%d, key:%s",
beginningMs, endingMs, key));
accumulator.accumulateUsage( durationSecs );

//add to results
usageMap.put( key, accumulator.getCurrentSummary() );
Expand Down Expand Up @@ -313,15 +315,15 @@ public void accumulateUsage( long durationSecs )
currentSummary.setBucketsNumMax(
Math.max(currentSummary.getBucketsNumMax(),
lastUsageData.getBucketsNum()));
log.info("Accumulate "
+ (lastUsageData.getObjectsMegs() * durationSecs));
log.info(String.format("Accumulated durationSecs:%d raw:%d multDurationSecs:%d",durationSecs,
lastUsageData.getObjectsMegs(),
(lastUsageData.getObjectsMegs() * durationSecs)));
this.lastUsageData = null;

}
}

}



}
21 changes: 12 additions & 9 deletions ...les/reporting/src/main/java/com/eucalyptus/reporting/modules/storage/StorageUsageLog.java
View file
Expand Up @@ -177,7 +177,8 @@ public Map<StorageSummaryKey, StorageUsageSummary> getUsageSummaryMap(
list = entityWrapper.createQuery(
"from StorageUsageSnapshot as sus"
+ " WHERE sus.key.timestampMs > ?"
+ " AND sus.key.timestampMs < ?")
+ " AND sus.key.timestampMs < ?"
+ " ORDER BY sus.key.timestampMs")
.setLong(0, (latestSnapshotBeforeMs!=null ? latestSnapshotBeforeMs : 0l))
.setLong(1, new Long(period.getEndingMs()))
.list();
Expand All @@ -186,7 +187,8 @@ public Map<StorageSummaryKey, StorageUsageSummary> getUsageSummaryMap(
"from StorageUsageSnapshot as sus"
+ " WHERE sus.key.timestampMs > ?"
+ " AND sus.key.timestampMs < ?"
+ " AND sus.key.accountId = ?")
+ " AND sus.key.accountId = ?"
+ " ORDER BY sus.key.timestampMs")
.setLong(0, (latestSnapshotBeforeMs!=null ? latestSnapshotBeforeMs : 0l))
.setLong(1, new Long(period.getEndingMs()))
.setString(2, accountId)
Expand Down Expand Up @@ -224,11 +226,11 @@ public Map<StorageSummaryKey, StorageUsageSummary> getUsageSummaryMap(
long endingMs = snapshotKey.getTimestampMs()-1;
long durationSecs = (endingMs - beginningMs) / 1000;

log.info(String.format("Accumulate usage, %d-%d, key:%s",
beginningMs, endingMs, summaryKey));
accumulator.accumulateUsage( durationSecs );
accumulator.setLastTimestamp(snapshotKey.getTimestampMs());
accumulator.setLastUsageData(snapshot.getUsageData());
log.info(String.format("Accumulate usage, %d-%d, key:%s",
beginningMs, endingMs, summaryKey));

}

Expand All @@ -243,9 +245,9 @@ public Map<StorageSummaryKey, StorageUsageSummary> getUsageSummaryMap(
accumulator.getLastTimestamp() );
long endingMs = period.getEndingMs() - 1;
long durationSecs = ( endingMs-beginningMs ) / 1000;
accumulator.accumulateUsage( durationSecs );
log.info(String.format("Accumulate endUsage, %d-%d, key:%s",
beginningMs, endingMs, key));
accumulator.accumulateUsage( durationSecs );

//add to results
usageMap.put( key, accumulator.getCurrentSummary() );
Expand Down Expand Up @@ -320,15 +322,16 @@ public void accumulateUsage( long durationSecs )
Math.max(currentSummary.getSnapshotsMegsMax(),
lastUsageData.getSnapshotsMegs()));

log.info("Accumulate "
+ (lastUsageData.getVolumesMegs() * durationSecs)
+ " " + (lastUsageData.getSnapshotsMegs() * durationSecs));
log.info(String.format("Accumulated durationSecs:%d raw:%d,%d multDurationSecs:%d,%d",durationSecs,
lastUsageData.getVolumesMegs(), lastUsageData.getSnapshotsMegs(),
(lastUsageData.getVolumesMegs() * durationSecs),
(lastUsageData.getSnapshotsMegs() * durationSecs)));
this.lastUsageData = null;

}
}

}


}

0 comments on commit 9f40094

Please sign in to comment.