Skip to content

Commit

Permalink
bfe timeout fixes, fix to initial synchronization failure restart, no…
Browse files Browse the repository at this point in the history 
…tifications
  • Loading branch information
chris grzegorczyk committed Jan 20, 2012
2 parents 130a0a7 + b67d322 commit de46c10
Show file tree
Hide file tree
Showing 8 changed files with 478 additions and 96 deletions.
30 changes: 30 additions & 0 deletions clc/modules/msgs/conf/scripts/notifications.groovy
View file
@@ -0,0 +1,30 @@
import com.eucalyptus.component.ServiceConfiguration
import com.eucalyptus.component.Faults.FaultRecord
import com.eucalyptus.scripting.Groovyness
import com.eucalyptus.util.Exceptions
import com.google.common.base.Throwables

summary = ""
details = ""
faults.each{ FaultRecord f ->
ServiceConfiguration s = Groovyness.expandoMetaClass(f.getServiceConfiguration( ));
summary += """
- ${s.getFullName( )} ${f.getTransitionRecord( ).getRule( ).getFromState( )}->${f.getFinalState( )} ${f.getError( ).getTimestamp( )}
${Throwables.getRootCause(f.getError( )).getMessage( )}
"""
details += """
- ${s.getFullName( )} ------------
${f.getTransitionRecord( )}
${Exceptions.causeString(f.getError())}
"""
}
content = """
Impacted Services Summary
=========================
${summary}
Details
=======
${details}
""".toString()
3 changes: 3 additions & 0 deletions clc/modules/msgs/conf/scripts/notifications_digest.groovy
View file
@@ -0,0 +1,3 @@
import com.eucalyptus.component.Components

Components.list().collect{ c -> c.services().collect{ s -> "\n" + s.toString() } }
120 changes: 62 additions & 58 deletions clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/Databases.java
View file
Expand Up @@ -168,6 +168,7 @@ public DatabaseStateException( String string ) {
}

private static final int MAX_TX_START_SYNC_RETRIES = 120;
private static final AtomicInteger counter = new AtomicInteger( 120 );
private static final Predicate<Host> FILTER_SYNCING_DBS = Predicates.and( DbFilter.INSTANCE, Predicates.not( SyncedDbFilter.INSTANCE ) );
private static final ScriptedDbBootstrapper singleton = new ScriptedDbBootstrapper( );
private static Logger LOG = Logger.getLogger( Databases.class );
Expand Down Expand Up @@ -260,8 +261,13 @@ public void run( ) {
if ( !Hosts.isCoordinator( ) && Hosts.localHost( ).hasDatabase( ) ) {
while ( !Databases.enable( Hosts.localHost( ) ) ) {
LOG.warn( LogUtil.subheader( "Synchronization of the database failed: " + Hosts.localHost( ) ) );
LOG.warn( "Sleeping for " + INITIAL_DB_SYNC_RETRY_WAIT + " seconds before trying again." );
TimeUnit.SECONDS.sleep( INITIAL_DB_SYNC_RETRY_WAIT );
if ( counter.decrementAndGet( ) == 0 ) {
LOG.fatal( "Restarting process to force re-synchronization." );
System.exit( 123 );
} else {
LOG.warn( "Sleeping for " + INITIAL_DB_SYNC_RETRY_WAIT + " seconds before trying again." );
TimeUnit.SECONDS.sleep( INITIAL_DB_SYNC_RETRY_WAIT );
}
}
Hosts.UpdateEntry.INSTANCE.apply( Hosts.localHost( ) );
LOG.info( LogUtil.subheader( "Database synchronization complete: " + Hosts.localHost( ) ) );
Expand Down Expand Up @@ -484,7 +490,8 @@ public void run( ) {
Logs.extreme( ).debug( "Skipping addition of db connections for host which already exists: " + hostName );
} catch ( IllegalStateException ex ) {
if ( Exceptions.isCausedBy( ex, InstanceAlreadyExistsException.class ) ) {
ManagementFactory.getPlatformMBeanServer( ).unregisterMBean( new ObjectName( "net.sf.hajdbc:cluster=" + ctx + ",database=" + hostName ) );
ManagementFactory.getPlatformMBeanServer( ).unregisterMBean(
new ObjectName( "net.sf.hajdbc:cluster=" + ctx + ",database=" + hostName ) );
cluster.add( hostName, realJdbcDriver, dbUrl );
} else {
throw ex;
Expand Down Expand Up @@ -626,7 +633,7 @@ static boolean disable( final String hostName ) {
}
}
}
private static final AtomicInteger counter = new AtomicInteger( 5 );

static boolean enable( final Host host ) {
if ( !host.hasDatabase( ) ) {
return false;
Expand All @@ -644,9 +651,6 @@ static boolean enable( final Host host ) {
SyncState.NOTSYNCED.set( );
LOG.error( ex );
Logs.extreme( ).error( ex, ex );
if ( Exceptions.isCausedBy( ex, InstanceNotFoundException.class ) && counter.decrementAndGet( ) == 0 ) {
System.exit( 123 );
}
return false;
}
} else if ( !SyncState.SYNCING.isCurrent( ) ) {
Expand Down Expand Up @@ -788,7 +792,7 @@ public Set<String> get( ) {
}
};
private static final AtomicBoolean last = new AtomicBoolean( false );

@Override
public abstract Set<String> get( );

Expand Down Expand Up @@ -1005,15 +1009,15 @@ public static <D> void dropForeignKeys( SynchronizationContext<D> context ) thro
}
statement.executeBatch( );
} catch ( SQLException sqle ) {
LOG.error( sqle );
Logs.extreme( ).error( sqle, sqle );
throw sqle;
LOG.error( sqle );
Logs.extreme( ).error( sqle, sqle );
throw sqle;
} finally {
try {
statement.close( );
} catch ( Exception e ) {
LOG.error( e );
}
try {
statement.close( );
} catch ( Exception e ) {
LOG.error( e );
}
}
}

Expand All @@ -1040,15 +1044,15 @@ public static <D> void restoreForeignKeys( SynchronizationContext<D> context ) t
}
statement.executeBatch( );
} catch ( SQLException sqle ) {
LOG.error( sqle );
Logs.extreme( ).error( sqle, sqle );
throw sqle;
LOG.error( sqle );
Logs.extreme( ).error( sqle, sqle );
throw sqle;
} finally {
try {
statement.close( );
} catch ( Exception e ) {
LOG.error( e );
}
try {
statement.close( );
} catch ( Exception e ) {
LOG.error( e );
}
}

}
Expand Down Expand Up @@ -1086,15 +1090,15 @@ public Long call( ) throws SQLException
long value = resultSet.getLong( 1 );
return value;
} catch ( SQLException sqle ) {
LOG.error(sqle);
Logs.extreme( ).error( sqle, sqle );
throw sqle;
LOG.error( sqle );
Logs.extreme( ).error( sqle, sqle );
throw sqle;
} finally {
try {
statement.close( );
} catch ( Exception e ) {
LOG.error( e );
}
try {
statement.close( );
} catch ( Exception e ) {
LOG.error( e );
}
}
}
};
Expand Down Expand Up @@ -1133,11 +1137,11 @@ public Long call( ) throws SQLException
Logs.extreme( ).error( sqle, sqle );
throw sqle;
} finally {
try {
targetStatement.close( );
} catch ( Exception e ) {
LOG.error( e );
}
try {
targetStatement.close( );
} catch ( Exception e ) {
LOG.error( e );
}
}
}
}
Expand All @@ -1148,10 +1152,10 @@ public Long call( ) throws SQLException
* @throws SQLException
*/
public static <D> void synchronizeIdentityColumns( SynchronizationContext<D> context ) throws SQLException {

Statement sourceStatement = null;
Statement targetStatement = null;
try {
try {
sourceStatement = context.getConnection( context.getSourceDatabase( ) ).createStatement( );
targetStatement = context.getConnection( context.getTargetDatabase( ) ).createStatement( );
Dialect dialect = context.getDialect( );
Expand Down Expand Up @@ -1181,20 +1185,20 @@ public static <D> void synchronizeIdentityColumns( SynchronizationContext<D> con
}
}
}
} catch (SQLException sqle ) {
LOG.error( sqle );
Logs.extreme( ).error( sqle, sqle );
throw sqle;
} catch ( SQLException sqle ) {
LOG.error( sqle );
Logs.extreme( ).error( sqle, sqle );
throw sqle;
} finally {
try {
sourceStatement.close( );
sourceStatement.close( );
} catch ( Exception e1 ) {
LOG.error( e1 );
LOG.error( e1 );
}
try {
targetStatement.close( );
} catch ( Exception e2 ) {
LOG.error( e2 );
LOG.error( e2 );
}
}
}
Expand All @@ -1220,15 +1224,15 @@ public static <D> void dropUniqueConstraints( SynchronizationContext<D> context
}
statement.executeBatch( );
} catch ( SQLException sqle ) {
LOG.error( sqle );
Logs.extreme( ).error( sqle, sqle );
throw sqle;
LOG.error( sqle );
Logs.extreme( ).error( sqle, sqle );
throw sqle;
} finally {
try {
statement.close( );
} catch (Exception e) {
LOG.error( e );
}
try {
statement.close( );
} catch ( Exception e ) {
LOG.error( e );
}
}
}

Expand All @@ -1254,14 +1258,14 @@ public static <D> void restoreUniqueConstraints( SynchronizationContext<D> conte
}
statement.executeBatch( );
} catch ( SQLException sqle ) {
LOG.error( sqle );
Logs.extreme( ).error( sqle, sqle );
throw sqle;
LOG.error( sqle );
Logs.extreme( ).error( sqle, sqle );
throw sqle;
} finally {
try {
statement.close( );
} catch ( Exception e ) {
LOG.error( e );
LOG.error( e );
}
}
}
Expand Down

0 comments on commit de46c10

Please sign in to comment.