Permalink
Browse files

merged + finished gzip support for Walrus transfers, compiled in if z…

…lib is available
  • Loading branch information...
2 parents 49a4c56 + d61a535 commit c6c771ad181982541e70029c8d1d0ad12115e619 Dmitrii Zagorodnov committed Mar 13, 2009
Showing with 631 additions and 276 deletions.
  1. +2 −2 clc/modules/axis2-transport/src/edu/ucsb/eucalyptus/transport/Axis2InOutMessageReceiver.java
  2. +4 −0 clc/modules/axis2-transport/src/edu/ucsb/eucalyptus/transport/client/BasicClient.java
  3. +36 −10 clc/modules/cluster-manager/src/edu/ucsb/eucalyptus/cloud/cluster/ClusterAllocator.java
  4. +19 −2 clc/modules/cluster-manager/src/edu/ucsb/eucalyptus/cloud/cluster/UnassignAddressCallback.java
  5. +10 −5 clc/modules/cluster-manager/src/edu/ucsb/eucalyptus/cloud/cluster/VmRunCallback.java
  6. +7 −1 clc/modules/cluster-manager/src/edu/ucsb/eucalyptus/cloud/cluster/VolumeDetachCallback.java
  7. +71 −7 clc/modules/cluster-manager/src/edu/ucsb/eucalyptus/cloud/ws/AddressManager.java
  8. +1 −1 clc/modules/cluster-manager/src/edu/ucsb/eucalyptus/cloud/ws/SystemState.java
  9. +22 −8 clc/modules/cluster-manager/src/edu/ucsb/eucalyptus/cloud/ws/VmAdmissionControl.java
  10. +3 −1 clc/modules/core/src/edu/ucsb/eucalyptus/util/WalrusProperties.java
  11. +6 −3 clc/modules/image-manager/src/edu/ucsb/eucalyptus/cloud/ws/VolumeManager.java
  12. +6 −6 clc/modules/msgs/resources/s3-2006-03-01-binding.xml
  13. +2 −0 clc/modules/msgs/src/edu/ucsb/eucalyptus/cloud/Extra.groovy
  14. +14 −0 clc/modules/msgs/src/edu/ucsb/eucalyptus/msgs/VmAddresses.groovy
  15. +1 −1 clc/modules/storage-controller/src/edu/ucsb/eucalyptus/storage/LVM2Manager.java
  16. +12 −8 clc/modules/storage-manager/src/edu/ucsb/eucalyptus/cloud/ws/Bukkit.java
  17. +6 −1 cluster/handlers.c
  18. +1 −1 cluster/server-marshal.c
  19. +73 −1 configure
  20. +2 −1 configure.ac
  21. +3 −0 net/vnetwork.c
  22. +29 −13 node/handlers_kvm.c
  23. +18 −3 node/handlers_xen.c
  24. +35 −25 storage/storage.c
  25. +132 −104 storage/walrus.c
  26. +1 −0 tools/Makefile
  27. +26 −16 tools/euca_conf
  28. +87 −56 tools/partition2disk
  29. +2 −0 util/config.h.in
@@ -124,7 +124,7 @@ public void invokeBusinessLogic( MessageContext msgContext, MessageContext newMs
newMsgContext.setProperty( Axis2HttpWorker.REAL_HTTP_REQUEST, msgContext.getProperty( Axis2HttpWorker.REAL_HTTP_REQUEST ) );
newMsgContext.setProperty( Axis2HttpWorker.REAL_HTTP_RESPONSE, msgContext.getProperty( Axis2HttpWorker.REAL_HTTP_RESPONSE ) );
- LOG.info("Returning reply: " + message.getPayload());
+ LOG.info("Returning reply: " + message.getPayload());
if(message.getPayload() instanceof WalrusErrorMessageType) {
WalrusErrorMessageType errorMessage = (WalrusErrorMessageType) message.getPayload();
@@ -213,7 +213,7 @@ private SOAPEnvelope generateMessage( final String methodName, final SOAPFactory
SOAPEnvelope envelope = null;
LOG.info( "[" + serviceClass.getSimpleName() + ":" + methodName + "] Got return type " + response.getClass().getSimpleName() );
if( response instanceof AddClusterResponseType )
- bindingName = "msgs_eucalyptus_ucsb_edu";
+ bindingName = "msgs_eucalyptus_ucsb_edu";
try
{
/** construct the response **/
@@ -152,6 +152,10 @@ public OMElement sync( OMElement omMsg ) throws AxisFault {
omResponse = this.serviceClient.sendReceive( omMsg );
break;
}
+ catch ( RuntimeException e ) {
+ LOG.error( String.format( EucalyptusProperties.DEBUG_FSTRING, omMsg.getLocalName()+":"+i, e.getMessage() ) );
+ ex = e;
+ }
catch ( Exception e ) {
LOG.error( String.format( EucalyptusProperties.DEBUG_FSTRING, omMsg.getLocalName()+":"+i, e.getMessage() ) );
ex = e;
@@ -1,11 +1,27 @@
package edu.ucsb.eucalyptus.cloud.cluster;
-import com.google.common.collect.*;
-import edu.ucsb.eucalyptus.cloud.*;
-import edu.ucsb.eucalyptus.msgs.*;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+import edu.ucsb.eucalyptus.cloud.Network;
+import edu.ucsb.eucalyptus.cloud.NetworkToken;
+import edu.ucsb.eucalyptus.cloud.ResourceToken;
+import edu.ucsb.eucalyptus.cloud.VmAllocationInfo;
+import edu.ucsb.eucalyptus.cloud.VmImageInfo;
+import edu.ucsb.eucalyptus.cloud.VmKeyInfo;
+import edu.ucsb.eucalyptus.cloud.VmRunType;
+import edu.ucsb.eucalyptus.cloud.ws.AddressManager;
+import edu.ucsb.eucalyptus.msgs.AssociateAddressType;
+import edu.ucsb.eucalyptus.msgs.ConfigureNetworkType;
+import edu.ucsb.eucalyptus.msgs.RunInstancesType;
+import edu.ucsb.eucalyptus.msgs.StartNetworkType;
+import edu.ucsb.eucalyptus.msgs.VmTypeInfo;
+import edu.ucsb.eucalyptus.util.Admin;
import org.apache.log4j.Logger;
+import org.apache.axis2.AxisFault;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -32,6 +48,17 @@ public ClusterAllocator( ResourceToken vmToken, VmAllocationInfo vmAllocInfo ) {
this.setupVmMessages( vmToken );
}
+ public void setupAddressMessages( ResourceToken token ) {
+ for( String addr : token.getAddresses() ) {
+ String vmId = token.getInstanceIds().get( token.getAddresses().indexOf( addr ) );
+ try {
+ new AddressManager().AssociateAddress( Admin.makeMsg( AssociateAddressType.class, addr, vmId ) );
+ } catch ( AxisFault axisFault ) {
+ LOG.error( axisFault );
+ }
+ }
+ }
+
public void setupNetworkMessages( NetworkToken networkToken ) {
if ( networkToken != null ) {
StartNetworkType msg = new StartNetworkType( this.vmAllocInfo.getRequest(), networkToken.getVlan(), networkToken.getNetworkName() );
@@ -86,25 +113,23 @@ else if ( this.rollback.get() && State.ROLLBACK.equals( this.state ) )
public void run() {
this.state = State.CREATE_NETWORK;
while ( !this.state.equals( State.FINISHED ) ) {
+ this.queueEvents();
switch ( this.state ) {
case CREATE_NETWORK:
- this.queueEvents();
this.setState( State.CREATE_NETWORK_RULES );
break;
case CREATE_NETWORK_RULES:
- this.queueEvents();
this.setState( State.CREATE_VMS );
break;
case CREATE_VMS:
- this.queueEvents();
+ this.setState( State.ASSIGN_ADDRESSES );
+ break;
+ case ASSIGN_ADDRESSES:
this.setState( State.FINISHED );
break;
case ROLLBACK:
- this.queueEvents();
this.setState( State.FINISHED );
break;
- case FINISHED:
- break;
}
this.clearQueue();
}
@@ -133,6 +158,7 @@ public AtomicBoolean getRollback() {
CREATE_NETWORK,
CREATE_NETWORK_RULES,
CREATE_VMS,
+ ASSIGN_ADDRESSES,
FINISHED,
ROLLBACK
}
@@ -1,8 +1,15 @@
package edu.ucsb.eucalyptus.cloud.cluster;
import edu.ucsb.eucalyptus.cloud.entities.Address;
-import edu.ucsb.eucalyptus.msgs.*;
+import edu.ucsb.eucalyptus.cloud.net.Addresses;
+import edu.ucsb.eucalyptus.cloud.ws.AddressManager;
+import edu.ucsb.eucalyptus.cloud.EucalyptusCloudException;
+import edu.ucsb.eucalyptus.msgs.ReleaseAddressType;
+import edu.ucsb.eucalyptus.msgs.UnassignAddressResponseType;
+import edu.ucsb.eucalyptus.msgs.UnassignAddressType;
import edu.ucsb.eucalyptus.transport.client.Client;
+import edu.ucsb.eucalyptus.util.Admin;
+import edu.ucsb.eucalyptus.util.EucalyptusProperties;
import org.apache.log4j.Logger;
import java.util.NoSuchElementException;
@@ -23,13 +30,23 @@ public UnassignAddressCallback( final Address parent ) {
public void process( final Client clusterClient, final UnassignAddressType msg ) throws Exception {
UnassignAddressResponseType reply = ( UnassignAddressResponseType ) clusterClient.send( msg );
-
VmInstance vm = null;
try {
vm = VmInstances.getInstance().lookup( vmId );
LOG.debug( "Unassign [" + pubIp + "] clearing VM " + vmId + ":" + vmIp );
vm.getNetworkConfig().setIgnoredPublicIp( VmInstance.DEFAULT_IP );
} catch ( NoSuchElementException e1 ) {}
+ String addr = msg.getSource();
+ try {
+ Address a = Addresses.getInstance().lookup( addr );
+ if( EucalyptusProperties.NAME.equals( a.getUserId() ) ) {
+ new AddressManager().ReleaseAddress( Admin.makeMsg( ReleaseAddressType.class, addr ) );
+ }
+ } catch ( NoSuchElementException e1 ) {
+ LOG.error( e1 );
+ } catch ( EucalyptusCloudException e1 ) {
+ LOG.error( e1 );
+ }
}
}
@@ -1,11 +1,14 @@
package edu.ucsb.eucalyptus.cloud.cluster;
-import edu.ucsb.eucalyptus.cloud.*;
+import edu.ucsb.eucalyptus.cloud.ResourceToken;
+import edu.ucsb.eucalyptus.cloud.VmInfo;
+import edu.ucsb.eucalyptus.cloud.VmRunResponseType;
+import edu.ucsb.eucalyptus.cloud.VmRunType;
+import edu.ucsb.eucalyptus.msgs.TerminateInstancesType;
import edu.ucsb.eucalyptus.transport.client.Client;
-import edu.ucsb.eucalyptus.msgs.*;
import edu.ucsb.eucalyptus.util.EucalyptusProperties;
-import org.apache.log4j.Logger;
import org.apache.axis2.AxisFault;
+import org.apache.log4j.Logger;
class VmRunCallback extends QueuedEventCallback<VmRunType> {
@@ -34,9 +37,11 @@ public void process( final Client clusterClient, final VmRunType msg ) throws Ex
LOG.info( String.format( EucalyptusProperties.DEBUG_FSTRING, EucalyptusProperties.TokenState.redeemed, token ) );
if ( reply.get_return() ) {
for ( VmInfo vmInfo : reply.getVms() ) {
- VmInstances.getInstance().lookup( vmInfo.getInstanceId() ).getNetworkConfig().setIpAddress( vmInfo.getNetParams().getIpAddress() );
- VmInstances.getInstance().lookup( vmInfo.getInstanceId() ).getNetworkConfig().setIgnoredPublicIp( vmInfo.getNetParams().getIgnoredPublicIp() );
+ VmInstance vm = VmInstances.getInstance().lookup( vmInfo.getInstanceId() );
+ vm.getNetworkConfig().setIpAddress( vmInfo.getNetParams().getIpAddress() );
+ vm.getNetworkConfig().setIgnoredPublicIp( vmInfo.getNetParams().getIgnoredPublicIp() );
}
+ this.parent.setupAddressMessages( this.token );
} else {
this.parent.getRollback().lazySet( true );
}
@@ -1,5 +1,7 @@
package edu.ucsb.eucalyptus.cloud.cluster;
+import edu.ucsb.eucalyptus.msgs.AttachedVolume;
+import edu.ucsb.eucalyptus.msgs.DetachVolumeResponseType;
import edu.ucsb.eucalyptus.msgs.DetachVolumeType;
import edu.ucsb.eucalyptus.transport.client.Client;
import org.apache.log4j.Logger;
@@ -12,7 +14,11 @@ public VolumeDetachCallback( ){}
public void process( final Client cluster, final DetachVolumeType msg ) throws Exception
{
- cluster.send( msg );
+ DetachVolumeResponseType reply = (DetachVolumeResponseType) cluster.send( msg );
+ if( reply.get_return() ) {
+ VmInstance vm = VmInstances.getInstance().lookup( msg.getInstanceId() );
+ vm.getVolumes().remove( new AttachedVolume( msg.getVolumeId() ) );
+ }
}
}
@@ -34,19 +34,45 @@
package edu.ucsb.eucalyptus.cloud.ws;
-import edu.ucsb.eucalyptus.cloud.*;
-import edu.ucsb.eucalyptus.cloud.entities.*;
-import edu.ucsb.eucalyptus.cloud.cluster.*;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import edu.ucsb.eucalyptus.cloud.EucalyptusCloudException;
+import edu.ucsb.eucalyptus.cloud.cluster.AssignAddressCallback;
+import edu.ucsb.eucalyptus.cloud.cluster.ClusterEnvelope;
+import edu.ucsb.eucalyptus.cloud.cluster.NotEnoughResourcesAvailable;
+import edu.ucsb.eucalyptus.cloud.cluster.QueuedEvent;
+import edu.ucsb.eucalyptus.cloud.cluster.UnassignAddressCallback;
+import edu.ucsb.eucalyptus.cloud.cluster.VmInstance;
+import edu.ucsb.eucalyptus.cloud.cluster.VmInstances;
+import edu.ucsb.eucalyptus.cloud.entities.Address;
+import edu.ucsb.eucalyptus.cloud.entities.EntityWrapper;
import edu.ucsb.eucalyptus.cloud.exceptions.ExceptionList;
-import edu.ucsb.eucalyptus.cloud.net.*;
-import edu.ucsb.eucalyptus.msgs.*;
-import edu.ucsb.eucalyptus.util.*;
+import edu.ucsb.eucalyptus.cloud.net.Addresses;
+import edu.ucsb.eucalyptus.msgs.AllocateAddressResponseType;
+import edu.ucsb.eucalyptus.msgs.AllocateAddressType;
+import edu.ucsb.eucalyptus.msgs.AssignAddressType;
+import edu.ucsb.eucalyptus.msgs.AssociateAddressResponseType;
+import edu.ucsb.eucalyptus.msgs.AssociateAddressType;
+import edu.ucsb.eucalyptus.msgs.DescribeAddressesResponseItemType;
+import edu.ucsb.eucalyptus.msgs.DescribeAddressesResponseType;
+import edu.ucsb.eucalyptus.msgs.DescribeAddressesType;
+import edu.ucsb.eucalyptus.msgs.DisassociateAddressResponseType;
+import edu.ucsb.eucalyptus.msgs.DisassociateAddressType;
+import edu.ucsb.eucalyptus.msgs.ReleaseAddressResponseType;
+import edu.ucsb.eucalyptus.msgs.ReleaseAddressType;
+import edu.ucsb.eucalyptus.msgs.UnassignAddressType;
+import edu.ucsb.eucalyptus.util.Admin;
+import edu.ucsb.eucalyptus.util.EucalyptusProperties;
+import edu.ucsb.eucalyptus.util.Messaging;
import org.apache.axis2.AxisFault;
import org.apache.log4j.Logger;
import org.mule.api.MuleException;
import org.mule.api.lifecycle.Startable;
-import java.util.*;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.NoSuchElementException;
import java.util.concurrent.ConcurrentNavigableMap;
public class AddressManager implements Startable {
@@ -65,6 +91,44 @@ public void start() throws MuleException {
db.commit();
}
+ public static NavigableSet<String> allocateAddresses( int count ) throws NotEnoughResourcesAvailable {
+ ConcurrentNavigableMap<String, Address> unusedAddresses = Addresses.getInstance().getDisabledMap();
+ //:: try to fail fast if needed :://
+ if ( unusedAddresses.size() < count ) throw new NotEnoughResourcesAvailable( );
+ List<Map.Entry<String, Address>> addressList = Lists.newArrayList();
+ for ( int i = 0; i < count; i++ ) {
+ Map.Entry<String, Address> addressEntry = unusedAddresses.pollFirstEntry();
+ if ( addressEntry != null ) {
+ addressList.add( addressEntry );
+ } else {
+ for ( Map.Entry<String, Address> a : addressList ) {
+ unusedAddresses.putIfAbsent( a.getKey(), a.getValue() );
+ }
+ throw new NotEnoughResourcesAvailable( );
+ }
+ }
+ NavigableSet<String> ipList = Sets.newTreeSet();
+ for ( Map.Entry<String, Address> addressEntry : addressList ) {
+ Address address = addressEntry.getValue();
+ address.allocate( EucalyptusProperties.NAME );
+ EntityWrapper<Address> db = new EntityWrapper<Address>();
+ try {
+ Address addr = db.getUnique( new Address( address.getName() ) );
+ addr.allocate( EucalyptusProperties.NAME );
+ } catch ( EucalyptusCloudException e ) {
+ db.merge( address );
+ }
+ db.commit();
+ ipList.add( address.getName() );
+ try {
+ Addresses.getInstance().register( address );
+ } catch ( Exception e ) {
+ }
+ }
+ return ipList;
+ }
+
+
public AllocateAddressResponseType AllocateAddress( AllocateAddressType request ) throws EucalyptusCloudException {
int addrCount = 0;
for( Address a : Addresses.getInstance().listValues() ) {
@@ -84,7 +84,7 @@ public static void handle( VmDescribeResponseType request ) {
private static void cleanUp( final VmInstance vm ) {
for ( Address address : Addresses.getInstance().listValues() ) {
- if ( vm.getNetworkConfig().getIpAddress().equals( address.getInstanceAddress() ) && vm.getInstanceId().equals( address.getInstanceId() ) ) {
+ if ( vm.getInstanceId().equals( address.getInstanceId() ) ) {
SystemState.dispatch( vm.getPlacement(), new UnassignAddressCallback( address ), Admin.makeMsg( UnassignAddressType.class, address.getName(), address.getInstanceAddress() ) );
}
}
@@ -1,13 +1,19 @@
package edu.ucsb.eucalyptus.cloud.ws;
-import edu.ucsb.eucalyptus.cloud.*;
-import edu.ucsb.eucalyptus.cloud.cluster.*;
-import edu.ucsb.eucalyptus.cloud.entities.*;
+import edu.ucsb.eucalyptus.cloud.EucalyptusCloudException;
+import edu.ucsb.eucalyptus.cloud.FailScriptFailException;
+import edu.ucsb.eucalyptus.cloud.ResourceToken;
+import edu.ucsb.eucalyptus.cloud.SLAs;
+import edu.ucsb.eucalyptus.cloud.VmAllocationInfo;
+import edu.ucsb.eucalyptus.cloud.cluster.Clusters;
+import edu.ucsb.eucalyptus.cloud.cluster.NotEnoughResourcesAvailable;
+import edu.ucsb.eucalyptus.cloud.entities.Counters;
import edu.ucsb.eucalyptus.msgs.RunInstancesType;
import org.apache.log4j.Logger;
import org.bouncycastle.util.encoders.Base64;
import java.util.List;
+import java.util.NavigableSet;
public class VmAdmissionControl {
@@ -34,12 +40,20 @@ public VmAllocationInfo evaluate( VmAllocationInfo vmAllocInfo ) throws Eucalypt
boolean failed = false;
try {
allocTokeList = sla.doVmAllocation( vmAllocInfo );
- vmAllocInfo.getAllocationTokens().addAll( allocTokeList );
- try {
- sla.doNetworkAllocation( vmAllocInfo.getRequest().getUserId(), vmAllocInfo.getAllocationTokens(), vmAllocInfo.getNetworks() );
- } catch ( NotEnoughResourcesAvailable notEnoughResourcesAvailable ) {
- failed = true;
+ int addrCount = 0;
+ for ( ResourceToken token : allocTokeList ) {
+ addrCount += token.getAmount();
+ }
+ if ( "public".equals( vmAllocInfo.getRequest().getAddressingType() ) ) {
+ NavigableSet<String> addresses = AddressManager.allocateAddresses( addrCount );
+ for ( ResourceToken token : allocTokeList ) {
+ for ( int i = 0; i < token.getAmount(); i++ ) {
+ token.getAddresses().add( addresses.pollFirst() );
+ }
+ }
}
+ vmAllocInfo.getAllocationTokens().addAll( allocTokeList );
+ sla.doNetworkAllocation( vmAllocInfo.getRequest().getUserId(), vmAllocInfo.getAllocationTokens(), vmAllocInfo.getNetworks() );
} catch ( FailScriptFailException e ) {
failed = true;
} catch ( NotEnoughResourcesAvailable notEnoughResourcesAvailable ) {
@@ -52,9 +52,11 @@
public static long IMAGE_CACHE_SIZE = 30 * G;
public static String WALRUS_URL;
public static int MAX_TOTAL_SNAPSHOT_SIZE = 50;
-
+ public static int MAX_KEYS = 1000;
+
public static final int IO_CHUNK_SIZE = 102400;
+
public static void update() {
try {
SystemConfiguration systemConfiguration = EucalyptusProperties.getSystemConfiguration();
Oops, something went wrong. Retry.

0 comments on commit c6c771a

Please sign in to comment.