Permalink
Browse files

Merge branch 'maint/3.2/testing' into team/clc-dev/CHUCK-33-MERGE

Conflicts:
	clc/modules/cluster-manager/src/main/java/com/eucalyptus/cluster/Cluster.java
	cluster/Makefile
	cluster/handlers.c
	gatherlog/Makefile
	net/vnetwork.c
	node/Makefile
	util/Makefile
  • Loading branch information...
2 parents dadb8ff + da360e7 commit f6a7ec699d9fd0b543f11cd4807d0e6aacfaef99 @gelinasc gelinasc committed Jan 17, 2013
Showing with 962 additions and 415 deletions.
  1. +1 −0 Makedefs.in
  2. +1 −1 clc/modules/bootstrap/src/main/native/Makefile
  3. +5 −3 clc/modules/cluster-manager/src/main/java/com/eucalyptus/blockstorage/Snapshots.java
  4. +2 −8 clc/modules/cluster-manager/src/main/java/com/eucalyptus/cloud/run/ClusterAllocator.java
  5. +23 −0 clc/modules/cluster-manager/src/main/java/com/eucalyptus/cluster/Cluster.java
  6. +1 −0 clc/modules/core/src/main/java/com/eucalyptus/util/StorageProperties.java
  7. +12 −0 clc/modules/core/src/main/java/edu/ucsb/eucalyptus/cloud/entities/SnapshotInfo.java
  8. +10 −4 clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/Databases.java
  9. +29 −22 clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/Hosts.java
  10. +2 −0 clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/SystemBootstrapper.java
  11. +2 −3 clc/modules/msgs/src/main/java/com/eucalyptus/entities/Entities.java
  12. +23 −4 clc/modules/msgs/src/main/java/com/eucalyptus/upgrade/Upgrades.java
  13. +2 −1 clc/modules/msgs/src/main/java/edu/ucsb/eucalyptus/cloud/entities/SystemConfiguration.java
  14. +0 −2 clc/modules/reporting/src/main/java/com/eucalyptus/reporting/domain/ReportingAccountCrud.java
  15. +43 −57 clc/modules/reporting/src/main/java/com/eucalyptus/reporting/domain/ReportingAccountDao.java
  16. +0 −2 clc/modules/reporting/src/main/java/com/eucalyptus/reporting/domain/ReportingUserCrud.java
  17. +55 −63 clc/modules/reporting/src/main/java/com/eucalyptus/reporting/domain/ReportingUserDao.java
  18. +3 −3 clc/modules/storage-controller/native/Makefile
  19. +17 −1 clc/modules/storage-controller/src/main/java/com/eucalyptus/storage/DASManager.java
  20. +28 −1 clc/modules/storage-controller/src/main/java/com/eucalyptus/storage/LogicalStorageManager.java
  21. +17 −1 clc/modules/storage-controller/src/main/java/com/eucalyptus/storage/OverlayManager.java
  22. +270 −76 clc/modules/storage-controller/src/main/java/edu/ucsb/eucalyptus/cloud/ws/BlockStorage.java
  23. +1 −2 clc/modules/storage-controller/src/main/java/edu/ucsb/eucalyptus/cloud/ws/HttpReader.java
  24. +12 −1 clc/modules/storage-controller/src/main/java/edu/ucsb/eucalyptus/cloud/ws/HttpTransfer.java
  25. +0 −2 clc/modules/storage-controller/src/main/java/edu/ucsb/eucalyptus/cloud/ws/HttpWriter.java
  26. +7 −1 ...modules/troubleshooting/src/main/java/com/eucalyptus/troubleshooting/checker/DBResourceCheck.java
  27. +40 −0 clc/modules/wsstack/src/main/java/com/eucalyptus/ws/handlers/WalrusAuthenticationHandler.java
  28. +6 −6 cluster/Makefile
  29. +6 −6 cluster/handlers.c
  30. +16 −6 cluster/handlers.h
  31. +3 −0 console/static/custom/Messages.properties
  32. 0 console/static/custom/Messages_FI.properties
  33. 0 console/static/custom/Messages_FI.properties
  34. 0 console/static/custom/Messages_en.properties
  35. 0 console/static/custom/Messages_en.properties
  36. 0 console/static/custom/Messages_en_US.properties
  37. 0 console/static/custom/Messages_en_US.properties
  38. +45 −32 console/static/custom/Messages_es_ES.properties
  39. +6 −1 console/static/custom/Messages_fi_FI.properties
  40. +45 −32 console/static/custom/Messages_fr_FR.properties
  41. +45 −32 console/static/custom/Messages_it_IT.properties
  42. +12 −0 console/static/custom/Messages_ja.properties
  43. 0 console/static/custom/Messages_ja_JP.properties
  44. 0 console/static/custom/Messages_ja_JP.properties
  45. 0 console/static/custom/Messages_ko.properties
  46. 0 console/static/custom/Messages_ko.properties
  47. +12 −0 console/static/custom/Messages_ko_KR.properties
  48. +12 −0 console/static/custom/Messages_ru.properties
  49. 0 console/static/custom/Messages_ru_RU.properties
  50. 0 console/static/custom/Messages_ru_RU.properties
  51. +6 −0 console/static/custom/Messages_zh_CN.properties
  52. +2 −1 console/static/custom/format_properties.py
  53. 0 console/static/help/en_US
  54. 0 console/static/help/en_US
  55. +0 −1 console/static/help/zh_CN
  56. +1 −0 console/static/help/zh_CN
  57. +8 −1 console/static/js/eip.js
  58. +21 −2 console/static/js/login.js
  59. +1 −1 console/static/js/support.js
  60. +4 −3 gatherlog/Makefile
  61. +1 −1 net/Makefile
  62. +27 −7 net/vnetwork.c
  63. +11 −10 node/Makefile
  64. 0 project/cluster
  65. 0 project/cluster
  66. +1 −0 project/devel
  67. 0 project/gatherlog
  68. 0 project/gatherlog
  69. 0 project/net
  70. 0 project/net
  71. 0 project/node
  72. 0 project/node
  73. 0 project/storage
  74. 0 project/storage
  75. 0 project/tools
  76. 0 project/tools
  77. 0 project/util
  78. 0 project/util
  79. 0 project/wsdl
  80. 0 project/wsdl
  81. +5 −5 storage/Makefile
  82. +9 −9 util/Makefile
  83. +50 −0 util/faults/en_US/1006.xml
  84. +1 −1 util/faults/en_US/1011.xml
View
@@ -39,6 +39,7 @@ TOP = @TOP@
# system dependents
CC = @CC@
CFLAGS = @CFLAGS@
+CPPFLAGS = @CPPFLAGS@
LDFLAGS = @LDFLAGS@
INCLUDES = @INCLUDES@
LIBS = @LIBS@ -pthread
@@ -74,7 +74,7 @@ OBJS = eucalyptus-bootstrap.o eucalyptus-opts.o
all: eucalyptus-cloud
build: eucalyptus-cloud
.c.o:
- $(CC) $(CFLAGS) -c $< -o $@
+ $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@
eucalyptus-cloud: eucalyptus-opts.h $(OBJS)
$(LDCMD) $(OBJS) -o eucalyptus-cloud $(LDFLAGS)
@@ -274,15 +274,17 @@ static Snapshot startCreateSnapshot( final Volume vol, final Snapshot snap ) thr
@Override
public void fire( Snapshot s ) {
- try {
- CreateStorageSnapshotType scRequest = new CreateStorageSnapshotType( vol.getDisplayName( ), snap.getDisplayName( ) );
+ String scSnapStatus = null;
+ try {
+ CreateStorageSnapshotType scRequest = new CreateStorageSnapshotType( vol.getDisplayName( ), snap.getDisplayName( ) );
CreateStorageSnapshotResponseType scReply = AsyncRequests.sendSync( sc, scRequest );
s.setMappedState( scReply.getStatus( ) );
+ scSnapStatus = scReply.getStatus();
} catch ( Exception ex ) {
throw Exceptions.toUndeclared( ex );
}
}
- } );
+ } );
} catch ( ConstraintViolationException ex ) {
throw new DuplicateMetadataException( "Duplicate snapshot creation: " + snap + ": " + ex.getMessage( ), ex );
} catch ( ExecutionException ex ) {
@@ -391,13 +391,7 @@ private Request makeRunRequest( final ResourceToken childToken, final VmTypeInfo
final String platform = this.allocInfo.getBootSet( ).getMachine( ).getPlatform( ).name( ) != null
? this.allocInfo.getBootSet( ).getMachine( ).getPlatform( ).name( )
: "linux"; // ASAP:FIXME:GRZE
- ExtantNetwork exNet;
- try {
- exNet = this.allocInfo.getExtantNetwork( );
- } catch ( final Exception ex ) {
- Logs.extreme( ).error( ex, ex );
- exNet = ExtantNetwork.bogus( this.allocInfo.getPrimaryNetwork( ) );
- }
+
//TODO:GRZE:FINISH THIS. Date date = Contexts.lookup( ).getContracts( ).get( Contract.Type.EXPIRATION );
final VmRunType run = VmRunType.builder( )
.instanceId( childToken.getInstanceId( ) )
@@ -409,7 +403,7 @@ private Request makeRunRequest( final ResourceToken childToken, final VmTypeInfo
.platform( platform )
.reservationId( childToken.getAllocationInfo( ).getReservationId( ) )
.userData( this.allocInfo.getRequest( ).getUserData( ) )
- .vlan( exNet.getTag( ) )
+ .vlan( childToken.getExtantNetwork( ) != null ? childToken.getExtantNetwork().getTag( ) : new Integer(-1) )
.vmTypeInfo( vmInfo )
.owner( this.allocInfo.getOwnerFullName( ) )
.create( );
@@ -64,7 +64,10 @@
import java.io.IOException;
import java.lang.reflect.UndeclaredThrowableException;
+import java.net.InetAddress;
import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.UnknownHostException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.List;
@@ -153,6 +156,7 @@
import com.eucalyptus.ws.WebServicesException;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
+import com.google.common.base.Strings;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.ObjectArrays;
@@ -166,6 +170,7 @@
private static Logger LOG = Logger.getLogger( Cluster.class );
private final StateMachine<Cluster, State, Transition> stateMachine;
private final ClusterConfiguration configuration;
+//TODO:GRZE: sigh. This stuff needs to be addressed by (1) move to Nodes.java for nodeMap, (2) handling it like any other registered service.
private final ConcurrentNavigableMap<String, NodeInfo> nodeMap;
private final BlockingQueue<Throwable> pendingErrors = new LinkedBlockingDeque<Throwable>( );
private final ClusterState state;
@@ -775,7 +780,25 @@ public String getName( ) {
}
public NodeInfo getNode( final String serviceTag ) {
+ if ( this.nodeMap.containsKey( serviceTag ) ) {
return this.nodeMap.get( serviceTag );
+ } else {
+ try {
+ URI tag = new URI( serviceTag );
+ String host = tag.getHost( );
+ InetAddress addr = InetAddress.getByName( host );
+ String hostAddr = addr.getHostAddress( );
+ String altTag = serviceTag.replace( host, hostAddr );
+ if ( this.nodeMap.containsKey( altTag ) ) {
+ return this.nodeMap.get( altTag );
+ } else {
+ return null;//TODO:GRZE: sigh.
+ }
+ } catch ( Exception ex ) {
+ return null;//TODO:GRZE: sigh.
+ }
+
+ }
}
public void updateNodeInfo( final ArrayList<String> serviceTags ) {
@@ -113,6 +113,7 @@
public static final String DEFAULT_EBS_MANAGER = "OverlayManager";
public static String DAS_DEVICE = "/dev/blockdev";
public static final String STORAGE_CLASS_PREFIX = "com.eucalyptus.storage.";
+ public static final long SNAP_POINT_MAX_TIME_MS = 15000; //15 sec turnaround
static { Groovyness.loadConfig("storageprops.groovy"); }
@@ -96,6 +96,10 @@
@Column(name = "should_transfer")
private Boolean shouldTransfer;
+ //TODO: zhill, persist the snapshot consistency point id here for cleanup. Should be removed upon snapshot completion
+ @Column(name = "snapshot_point_id")
+ private String snapPointId;
+
public SnapshotInfo() {
this.scName = StorageProperties.NAME;
}
@@ -201,4 +205,12 @@ public boolean equals(Object obj) {
return true;
}
+ public String getSnapPointId() {
+ return snapPointId;
+ }
+
+ public void setSnapPointId(String snapPointId) {
+ this.snapPointId = snapPointId;
+ }
+
}
@@ -154,6 +154,7 @@
import com.google.common.base.Suppliers;
import com.google.common.collect.Collections2;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
import com.google.common.collect.MapDifference;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -214,8 +215,8 @@ public void create() {
public static final String DB_LOCK_FILE = "DB_LOCK_FILE";
public void delete( ) {
- SubDirectory.DB.getChildFile("data",this.getLockName()).delete();
- LOG.info( "The " + this.getLockName( ) + " file was deleted" );
+ this.getLockFile( ).delete();
+ LOG.debug( "The " + this.getLockFile( ).getAbsolutePath( ) + " file was deleted" );
}
protected String getLockName() {
@@ -230,13 +231,18 @@ protected File getLockFile( ) {
return SubDirectory.DB.getChildFile( "data", this.getLockName() );
}
+ public void create( String reason ) {
+ LOG.error( this.getLockName( ) + ": Caused by: " + reason );
+ this.create( );
+ }
+
public void create( ) {
try {
if ( getLockFile( ).createNewFile( ) ) {
- LOG.debug( "The " + this.getLockName( ) + " file was created." );
+ LOG.debug( this.getLockName( ) + ": The " + this.getLockFile( ).getAbsolutePath( ) + " file was created." );
}
} catch ( IOException e ) {
- LOG.debug("Unable to create the " + this.getLockName( ) + " file: " + e.getMessage());
+ LOG.debug("Unable to create the " + this.getLockFile( ).getAbsolutePath( ) + " file: " + e.getMessage());
}
}
@@ -86,6 +86,7 @@
import org.jgroups.conf.ClassConfigurator;
import org.jgroups.stack.Protocol;
import org.jgroups.stack.ProtocolStack;
+import org.logicalcobwebs.proxool.ProxoolFacade;
import com.eucalyptus.component.Component;
import com.eucalyptus.component.Component.State;
import com.eucalyptus.component.ComponentId;
@@ -548,47 +549,53 @@ public void run( ) {
}
}
/**
- * At this point local state is up-to-date and only the previous coordinators need to make any kind of decisions based on that information.
+ * At this point local state is up-to-date and only CLCs need to proceed
*/
- if ( this.coordinator ) {
+ if ( BootstrapArgs.isCloudController( ) ) {
+ boolean partitioned = false;
/**
* Check if any DB was partitioned from the local view.
- * Check if db hosts have differing partitions: are there two distinct views which match a DB host.
*/
- Set<View> dbViews = Sets.newHashSet( );
+ Set<View> dbViews = Sets.newHashSet( );//used only for logging
for ( Host db : Hosts.listDatabases() ) {
View dbView = partitions.get( db.getDisplayName( ) );
if ( !dbView.equals( localView ) ) {
- LOG.warn( logPrefix( localView ) + " found partitioned database: " + dbView );
- Databases.Locks.PARTITIONED.create( );
- } else {
- dbViews.add( dbView );
+ partitioned = true;
}
- }
- /**
- * Check each database and determine if a partition arose between them.
- */
- if ( !dbViews.isEmpty( ) ) {
- LOG.warn( logPrefix( localView ) + " found partitioned database views: " + Joiner.on( ", " ).join( dbViews ) );
- Databases.Locks.PARTITIONED.create( );
+ dbViews.add( dbView );
}
/**
* Check to ensure that if this host was not the original coordinator it isn't restarted as the coordinator.
*/
Host newCoordinator = Coordinator.INSTANCE.get( );
if ( !coordinatorAddress.equals( newCoordinator.getDisplayName() ) ) {
- LOG.warn( logPrefix( localView ) + " found different coordinator " + newCoordinator );
- Databases.Locks.PARTITIONED.create( );
+ partitioned = true;
}
-
- if ( Hosts.hasCoordinator() && !Hosts.isCoordinator() && BootstrapArgs.isCloudController() ) {
- LOG.fatal( "PARTITION FAIL-STOP: Possibility for inconsistency detected for Host: " + Hosts.localHost() );
+
+ if ( !partitioned ) {//no partition, keep going ==> happy time.
+ return;
+ } else if ( !newCoordinator.isLocalHost( ) && this.coordinator ) {//was coordinator, am not now ==> failstop
+ LOG.error( "PARTITION FAIL-STOP: Possibility for inconsistency detected for Host: " + Hosts.localHost() );
+ LOG.error( "PARTITION FAIL-STOP: " + printMap( ) );
+ Databases.Locks.PARTITIONED.create( logPrefix( localView ) + " found partitioned database in subgroup views: " + Joiner.on( ", " ).join( dbViews ) );
+ Databases.Locks.PARTITIONED.failStop( );
+ } else if ( newCoordinator.isLocalHost( ) && this.coordinator ) {//was coordinator and continue to be ==> backup and keep going
+ LOG.error( "PARTITION CONTINUE: Possibility for inconsistency detected for hosts in the following views: " + Joiner.on( ", " ).join( dbViews ) );
+ } else if ( !newCoordinator.isLocalHost( ) && !this.coordinator ) {//wasn't coordinator, still am not AND someone else is ==> restart to resync data
+ LOG.error( "PARTITION RESTART: Possibility for stale data copy detected for Host: " + Hosts.localHost() );
+ LOG.error( "PARTITION RESTART: " + printMap( ) );
+ Databases.Locks.PARTITIONED.create( logPrefix( localView ) + " found different coordinator " + newCoordinator );
+ Databases.Locks.PARTITIONED.failStop( );
+// SystemBootstrapper.restart( );
+ } else if ( newCoordinator.isLocalHost( ) && !this.coordinator ) {//wasn't coordinator, but somehow am now (wtf?) ==> pretty sure this is badness ==> failstop
+ LOG.error( "PARTITION FAIL-STOP: Possibility for inconsistency detected for Host: " + Hosts.localHost() );
+ LOG.error( "PARTITION FAIL-STOP: " + printMap( ) );
+ Databases.Locks.PARTITIONED.create( logPrefix( localView ) + " found partitioned database in subgroup views: " + Joiner.on( ", " ).join( dbViews ) );
Databases.Locks.PARTITIONED.failStop( );
- } else if ( Hosts.isCoordinator( ) ) {
- LOG.fatal( "PARTITION FAIL-STOP: Possibility for inconsistency detected for hosts in the following views: " + Joiner.on( ", " ).join( dbViews ) );
}
}
}
+
};
Threads.newThread( mergeViews ).start( );
}
@@ -303,6 +303,8 @@ public boolean stop( ) throws Exception {
return true;
}
+ public static void restart( ) { System.exit( 123 ); }
+
private static native void shutdown( boolean reload );
public static native void hello( );
@@ -737,7 +737,7 @@ public void rollback( ) throws RecoverablePersistenceException {
this.txState = null;
} catch ( final RuntimeException ex ) {
Logs.extreme( ).error( ex );
- throw PersistenceExceptions.throwFiltered( ex );
+// throw PersistenceExceptions.throwFiltered( ex );
}
} else {
Logs.extreme( ).debug( "Duplicate call to rollback( )" );
@@ -932,8 +932,7 @@ public void rollback( ) {
try {
this.transaction.rollback( );
} catch ( final RuntimeException ex ) {
- LOG.warn( ex );
- Logs.extreme( ).warn( ex, ex );
+ LOG.error( ex, ex );
throw ex;
}
}
@@ -30,12 +30,14 @@
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.sql.Timestamp;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.Callable;
+import javax.annotation.Nullable;
import org.apache.log4j.Logger;
import org.hibernate.ejb.Ejb3Configuration;
import org.hibernate.tool.hbm2ddl.SchemaUpdate;
@@ -159,7 +161,8 @@ public String getValue( ) {
v3_1_0,
v3_1_1,
v3_1_2,
- v3_2_0;
+ v3_2_0,
+ v3_2_1;
public String getVersion( ) {
return this.name( ).substring( 1 ).replace( "_", "." );
@@ -176,6 +179,22 @@ public static Version getNewVersion( ) {
public static Version getCurrentVersion( ) {
return Version.valueOf( "v" + Arguments.CURRENT_VERSION.getValue( ).replace( ".", "_" ) );
}
+
+ /**
+ * Filter {@link Version#values()} to include only those {@link Version}s which are in the
+ * current upgrade path (if any).
+ *
+ * @return Iterable<Version> which are in the upgrade path.
+ */
+ public static Iterable<Version> upgradePath( ) {
+ return Iterables.filter( Arrays.asList( Version.values( ) ), new Predicate<Version>( ) {
+
+ @Override
+ public boolean apply( @Nullable Version input ) {
+ return getOldVersion( ).ordinal( ) < input.ordinal( ) && getNewVersion( ).ordinal( ) >= input.ordinal( );
+ }
+ } );
+ }
}
public static class UpgradeDiscovery extends ServiceJarDiscovery {
@@ -736,7 +755,7 @@ public Boolean call( ) {
@Override
public Boolean call( ) {
for ( ComponentId c : ComponentIds.list( ) ) {
- for ( Version v : Version.values( ) ) {
+ for ( Version v : Version.upgradePath( ) ) {
ComponentUpgradeInfo upgradeInfo = ComponentUpgradeInfo.get( v, c.getClass( ) );
for ( Callable<Boolean> p : upgradeInfo.getPreUpgrades( ) ) {
boolean result = false;
@@ -761,7 +780,7 @@ public Boolean call( ) {
@Override
public Boolean call( ) {
for ( ComponentId c : ComponentIds.list( ) ) {
- for ( Version v : Version.values( ) ) {
+ for ( Version v : Version.upgradePath( ) ) {
ComponentUpgradeInfo upgradeInfo = ComponentUpgradeInfo.get( v, c.getClass( ) );
for ( Entry<Class, Predicate> p : upgradeInfo.getEntityUpgrades( ).entries( ) ) {
Boolean result = false;
@@ -787,7 +806,7 @@ public Boolean call( ) {
@Override
public Boolean call( ) {
for ( ComponentId c : ComponentIds.list( ) ) {
- for ( Version v : Version.values( ) ) {
+ for ( Version v : Version.upgradePath( ) ) {
ComponentUpgradeInfo upgradeInfo = ComponentUpgradeInfo.get( v, c.getClass( ) );
for ( Callable<Boolean> p : upgradeInfo.getPostUpgrades( ) ) {
boolean result = false;
@@ -98,9 +98,10 @@
@ConfigurableField( description = "Domain name to use for DNS." )
@Column( name = "dns_domain" )
private String dnsDomain;
- @ConfigurableField( description = "Nameserver address." )
+ @ConfigurableField( description = "Nameserver hostname." )
@Column( name = "nameserver" )
private String nameserver;
+ @ConfigurableField( description = "Nameserver ip address." )
@Column( name = "ns_address" )
private String nameserverAddress;
@@ -67,11 +67,9 @@ public void createOrUpdateAccount(String id, String name)
return;
} else if (oldAccount!=null) {
updateInDb(account);
- ReportingAccountDao.getInstance().putCache(account);
} else {
try {
addToDb(account);
- ReportingAccountDao.getInstance().putCache(account);
} catch (RuntimeException e) {
LOG.error(e);
}
Oops, something went wrong.

0 comments on commit f6a7ec6

Please sign in to comment.