- fixes to RT: #5498 #5502 #5506 #5364 #4893 #4256 #5303 #5301 #5447 …

…#5455 #5396 #5055 #5500 #5416 #5361

- fix rt5361: describe-bundle-tasks showing duplicate tasks
- deleteOnTerminate and related BFE fixes RT: #5055
- make sure to setup local detachment state before sending request to SC or NC
- fixes to handling of volume attaching/detaching/etc states
- fixes to conflicting instance state updates, esp. term/stop 
- fix to overrunning topology queue 
- fix to interminable sync after ha-jdbc/jmx failure
- adjust id generation to be time independent
- fix address state restoration on restart
- fix to resource id collision rate
- remove uses of broken Threads.enqueue(Runnable)
- fix error handling for metadata.  lots of accidental whitespace. RT: #5500
- fix service redirect url construction, RT: #5416

clc/modules/authentication/src/main/java/com/eucalyptus/auth/crypto/DefaultCryptoProvider.java

@@ -26,6 +26,7 @@
 import com.eucalyptus.crypto.Signatures;
 import com.eucalyptus.records.EventRecord;
 import com.eucalyptus.records.EventType;
+import com.google.common.primitives.Longs;
 
 public class DefaultCryptoProvider implements CryptoProvider, CertificateProvider, HmacProvider {
   public static String  KEY_ALGORITHM         = "RSA";
@@ -192,12 +193,18 @@ public String generateSystemToken( byte[] data ) {
   }
 
   @Override
-  public String generateId( final String userId, final String prefix ) {
+  public String generateId( final String seed, final String prefix ) {
     Adler32 hash = new Adler32( );
-    String key = userId + (System.currentTimeMillis( ) * Math.random( ));
+    String key = seed;
     hash.update( key.getBytes( ) );
-    String imageId = String.format( "%s-%08X", prefix, hash.getValue( ) );
-    return imageId;
+    /**
+     * @see http://tools.ietf.org/html/rfc3309
+     */
+    for ( int i = key.length( ); i < 128; i += 8 ) {
+      hash.update( Longs.toByteArray( Double.doubleToRawLongBits( Math.random( ) ) ) );
+    }
+    String id = String.format( "%s-%08X", prefix, hash.getValue( ) );
+    return id;
   }
 
   @Override

clc/modules/cluster-manager/src/main/java/com/eucalyptus/address/AbstractSystemAddressManager.java

@@ -268,19 +268,29 @@ private static void clearVmState( final ClusterAddressInfo addrInfo ) {
 
     private static VmInstance maybeFindVm( final String instanceId, final String publicIp, final String privateIp ) {
       VmInstance vm = null;
-      try {
-        vm = VmInstances.lookup( instanceId );
-      } catch ( NoSuchElementException ex ) {
+      if ( instanceId != null ) {
+        try {
+          vm = VmInstances.lookup( instanceId );
+        } catch ( NoSuchElementException ex ) {
+          Logs.extreme( ).error( ex );
+        }
+      } else if ( privateIp != null ) {
         try {
           vm = VmInstances.lookupByPrivateIp( privateIp );
-        } catch ( final NoSuchElementException e ) {
-          Logs.exhaust( ).error( e );
+        } catch ( NoSuchElementException ex ) {
+          Logs.extreme( ).error( ex );
+        }
+      } else if ( publicIp != null ) {
+        try {
+          vm = VmInstances.lookupByPublicIp( publicIp );
+        } catch ( NoSuchElementException ex ) {
+          Logs.extreme( ).error( ex );
         }
       }
       if ( vm != null ) {
-        LOG.trace( "Candidate vm which claims this address: " + vm.getInstanceId( ) + " " + vm.getState( ) + " " + publicIp );
+        Logs.extreme( ).debug( "Candidate vm which claims this address: " + vm.getInstanceId( ) + " " + vm.getState( ) + " " + publicIp );
         if ( publicIp.equals( vm.getPublicAddress( ) ) ) {
-          LOG.trace( "Found vm which claims this address: " + vm.getInstanceId( ) + " " + vm.getState( ) + " " + publicIp );
+          Logs.extreme( ).debug( "Found vm which claims this address: " + vm.getInstanceId( ) + " " + vm.getState( ) + " " + publicIp );
         }
       }
       return vm;

clc/modules/cluster-manager/src/main/java/com/eucalyptus/address/Addresses.java

@@ -256,7 +256,7 @@ public void fire( ) {
       } else {
         addr.release( );
       }
-    } catch ( final IllegalStateException e ) {
+    } catch ( final Exception e ) {
       LOG.debug( e, e );
     }
   }

clc/modules/cluster-manager/src/main/java/com/eucalyptus/blockstorage/StorageUtil.java

@@ -1,63 +1,63 @@
 /*******************************************************************************
-*Copyright (c) 2009  Eucalyptus Systems, Inc.
-* 
-*  This program is free software: you can redistribute it and/or modify
-*  it under the terms of the GNU General Public License as published by
-*  the Free Software Foundation, only version 3 of the License.
-* 
-* 
-*  This file is distributed in the hope that it will be useful, but WITHOUT
-*  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-*  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-*  for more details.
-* 
-*  You should have received a copy of the GNU General Public License along
-*  with this program.  If not, see <http://www.gnu.org/licenses/>.
-* 
-*  Please contact Eucalyptus Systems, Inc., 130 Castilian
-*  Dr., Goleta, CA 93101 USA or visit <http://www.eucalyptus.com/licenses/>
-*  if you need additional information or have any questions.
-* 
-*  This file may incorporate work covered under the following copyright and
-*  permission notice:
-* 
-*    Software License Agreement (BSD License)
-* 
-*    Copyright (c) 2008, Regents of the University of California
-*    All rights reserved.
-* 
-*    Redistribution and use of this software in source and binary forms, with
-*    or without modification, are permitted provided that the following
-*    conditions are met:
-* 
-*      Redistributions of source code must retain the above copyright notice,
-*      this list of conditions and the following disclaimer.
-* 
-*      Redistributions in binary form must reproduce the above copyright
-*      notice, this list of conditions and the following disclaimer in the
-*      documentation and/or other materials provided with the distribution.
-* 
-*    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
-*    IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
-*    TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
-*    PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
-*    OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-*    EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-*    PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-*    PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-*    LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-*    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-*    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. USERS OF
-*    THIS SOFTWARE ACKNOWLEDGE THE POSSIBLE PRESENCE OF OTHER OPEN SOURCE
-*    LICENSED MATERIAL, COPYRIGHTED MATERIAL OR PATENTED MATERIAL IN THIS
-*    SOFTWARE, AND IF ANY SUCH MATERIAL IS DISCOVERED THE PARTY DISCOVERING
-*    IT MAY INFORM DR. RICH WOLSKI AT THE UNIVERSITY OF CALIFORNIA, SANTA
-*    BARBARA WHO WILL THEN ASCERTAIN THE MOST APPROPRIATE REMEDY, WHICH IN
-*    THE REGENTS' DISCRETION MAY INCLUDE, WITHOUT LIMITATION, REPLACEMENT
-*    OF THE CODE SO IDENTIFIED, LICENSING OF THE CODE SO IDENTIFIED, OR
-*    WITHDRAWAL OF THE CODE CAPABILITY TO THE EXTENT NEEDED TO COMPLY WITH
-*    ANY SUCH LICENSES OR RIGHTS.
-*******************************************************************************/
+ *Copyright (c) 2009  Eucalyptus Systems, Inc.
+ * 
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, only version 3 of the License.
+ * 
+ * 
+ *  This file is distributed in the hope that it will be useful, but WITHOUT
+ *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ *  for more details.
+ * 
+ *  You should have received a copy of the GNU General Public License along
+ *  with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * 
+ *  Please contact Eucalyptus Systems, Inc., 130 Castilian
+ *  Dr., Goleta, CA 93101 USA or visit <http://www.eucalyptus.com/licenses/>
+ *  if you need additional information or have any questions.
+ * 
+ *  This file may incorporate work covered under the following copyright and
+ *  permission notice:
+ * 
+ *    Software License Agreement (BSD License)
+ * 
+ *    Copyright (c) 2008, Regents of the University of California
+ *    All rights reserved.
+ * 
+ *    Redistribution and use of this software in source and binary forms, with
+ *    or without modification, are permitted provided that the following
+ *    conditions are met:
+ * 
+ *      Redistributions of source code must retain the above copyright notice,
+ *      this list of conditions and the following disclaimer.
+ * 
+ *      Redistributions in binary form must reproduce the above copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ * 
+ *    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+ *    IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ *    TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ *    PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+ *    OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ *    EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ *    PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ *    PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ *    LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ *    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ *    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. USERS OF
+ *    THIS SOFTWARE ACKNOWLEDGE THE POSSIBLE PRESENCE OF OTHER OPEN SOURCE
+ *    LICENSED MATERIAL, COPYRIGHTED MATERIAL OR PATENTED MATERIAL IN THIS
+ *    SOFTWARE, AND IF ANY SUCH MATERIAL IS DISCOVERED THE PARTY DISCOVERING
+ *    IT MAY INFORM DR. RICH WOLSKI AT THE UNIVERSITY OF CALIFORNIA, SANTA
+ *    BARBARA WHO WILL THEN ASCERTAIN THE MOST APPROPRIATE REMEDY, WHICH IN
+ *    THE REGENTS' DISCRETION MAY INCLUDE, WITHOUT LIMITATION, REPLACEMENT
+ *    OF THE CODE SO IDENTIFIED, LICENSING OF THE CODE SO IDENTIFIED, OR
+ *    WITHDRAWAL OF THE CODE CAPABILITY TO THE EXTENT NEEDED TO COMPLY WITH
+ *    ANY SUCH LICENSES OR RIGHTS.
+ *******************************************************************************/
 /*
  * Author: chris grzegorczyk <grze@eucalyptus.com>
  */
@@ -81,7 +81,11 @@
 import com.eucalyptus.component.Topology;
 import com.eucalyptus.component.id.Storage;
 import com.eucalyptus.entities.EntityWrapper;
+import com.eucalyptus.records.Logs;
 import com.eucalyptus.util.EucalyptusCloudException;
+import com.eucalyptus.vm.VmInstance;
+import com.eucalyptus.vm.VmInstances;
+import com.eucalyptus.vm.VmVolumeAttachment;
 import com.eucalyptus.ws.client.ServiceDispatcher;
 import com.google.common.base.Function;
 import com.google.common.collect.HashMultimap;
@@ -97,77 +101,99 @@
 
 public class StorageUtil {
   private static Logger LOG = Logger.getLogger( StorageUtil.class );
-
-  public static ArrayList<edu.ucsb.eucalyptus.msgs.Volume> getVolumeReply( Map<String, AttachedVolume> attachedVolumes, List<Volume> volumes ) throws EucalyptusCloudException {
-    Multimap<String,Volume> partitionVolumeMap = HashMultimap.create( );
-    Map<String,StorageVolume> idStorageVolumeMap = Maps.newHashMap( );
-    for( Volume v : volumes ) {
+
+  public static ArrayList<edu.ucsb.eucalyptus.msgs.Volume> getVolumeReply( List<Volume> volumes ) {
+    Multimap<String, Volume> partitionVolumeMap = HashMultimap.create( );
+    for ( Volume v : volumes ) {
       partitionVolumeMap.put( v.getPartition( ), v );
     }
     ArrayList<edu.ucsb.eucalyptus.msgs.Volume> reply = Lists.newArrayList( );
-    for( String partition : partitionVolumeMap.keySet( ) ) {
+    for ( String partition : partitionVolumeMap.keySet( ) ) {
       try {
-        ServiceConfiguration scConfig = Topology.lookup( Storage.class, Partitions.lookupByName( partition ) );
-        Iterator<String> volumeNames = Iterators.transform( partitionVolumeMap.get( partition ).iterator( ), new Function<Volume,String>() {
-          @Override
-          public String apply( Volume arg0 ) {
-            return arg0.getDisplayName( );
+        Map<String, StorageVolume> idStorageVolumeMap = updateVolumesInPartition( partitionVolumeMap, partition );
+        for ( Volume v : partitionVolumeMap.get( partition ) ) {
+          try {
+            reply.add( volumeStateTransform( idStorageVolumeMap, v ) );
+          } catch ( Exception ex ) {
+            Logs.extreme( ).error( ex );
+            reply.add( v.morph( new edu.ucsb.eucalyptus.msgs.Volume( ) ) );
           }
-        } );
-        DescribeStorageVolumesType descVols = new DescribeStorageVolumesType( Lists.newArrayList( volumeNames ) );
-        Dispatcher sc = ServiceDispatcher.lookup( scConfig );
-        DescribeStorageVolumesResponseType volState = sc.send( descVols );    
-        for ( StorageVolume vol : volState.getVolumeSet( ) ) {
-          idStorageVolumeMap.put( vol.getVolumeId( ), vol );
         }
-        for( Volume v : volumes ) {
-          if( !partition.equals( v.getPartition( ) ) ) continue;
-          String status = null;
-          Integer size = 0;
-          String actualDeviceName = "unknown";
-          if( idStorageVolumeMap.containsKey( v.getDisplayName( ) ) ) {
-            StorageVolume vol = idStorageVolumeMap.get( v.getDisplayName( ) );
-            status = vol.getStatus( );
-            size = Integer.parseInt( vol.getSize( ) );
-            actualDeviceName = vol.getActualDeviceName( );
-          } else {
-            v.setState( State.ANNIHILATED );
-          }
-          if ( attachedVolumes.containsKey( v.getDisplayName() ) ) {
-            v.setState( State.BUSY );
-          } else if( status != null ) {
-            v.setMappedState( status );
-          }
-          if( v.getSize() <= 0 ) {
-            v.setSize( new Integer( size ) );
-          }
-          if( "invalid".equals ( v.getRemoteDevice( ) ) || "unknown".equals( v.getRemoteDevice( ) ) || v.getRemoteDevice( ) == null ) {
-            v.setRemoteDevice( actualDeviceName );
-          }
-          edu.ucsb.eucalyptus.msgs.Volume aVolume = v.morph( new edu.ucsb.eucalyptus.msgs.Volume() );
-          if ( attachedVolumes.containsKey( v.getDisplayName() ) ) {
-            aVolume.setStatus( v.mapState( ) );
-            aVolume.getAttachmentSet().add( attachedVolumes.get( aVolume.getVolumeId() ) );
-
-            for ( AttachedVolume attachedVolume : aVolume.getAttachmentSet( ) ) {
-        	if(!attachedVolume.getDevice( ).startsWith("/dev/") ) {
-        	  attachedVolume.setDevice("/dev/" + attachedVolume.getDevice( ) );
-        	}
-            }	
-
-          }
-          if ( "invalid".equals( v.getRemoteDevice( ) ) && !State.FAIL.equals( v.getState( ) ) ) {
-            aVolume.setStatus( "creating" );
-          }
-          reply.add( aVolume );
+      } catch ( EucalyptusCloudException ex ) {
+        LOG.error( ex );
+        Logs.extreme( ).error( ex, ex );
+        for ( Volume v : partitionVolumeMap.get( partition ) ) {
+          reply.add( v.morph( new edu.ucsb.eucalyptus.msgs.Volume( ) ) );
         }
-      } catch ( NoSuchElementException ex ) {
-        LOG.error( ex , ex );
-      } catch ( NumberFormatException ex ) {
-        LOG.error( ex , ex );
       }
     }
     return reply;
   }
 
+  public static edu.ucsb.eucalyptus.msgs.Volume volumeStateTransform( Map<String, StorageVolume> idStorageVolumeMap, Volume v ) {
+    String status = null;
+    Integer size = 0;
+    String actualDeviceName = "unknown";
+    if ( idStorageVolumeMap.containsKey( v.getDisplayName( ) ) ) {
+      StorageVolume vol = idStorageVolumeMap.get( v.getDisplayName( ) );
+      status = vol.getStatus( );
+      size = Integer.parseInt( vol.getSize( ) );
+      actualDeviceName = vol.getActualDeviceName( );
+    } else {
+      v.setState( State.ANNIHILATED );
+    }
+    edu.ucsb.eucalyptus.msgs.Volume aVolume = transformVolume( v, status, size, actualDeviceName );
+    return aVolume;
+  }
+
+  public static edu.ucsb.eucalyptus.msgs.Volume transformVolume( Volume v, String status, Integer size, String actualDeviceName ) {
+    VmVolumeAttachment vmAttachedVol = null;
+    VmInstance vm = null;
+    try {
+      vmAttachedVol = VmInstances.lookupVolumeAttachment( v.getDisplayName( ) );
+      vm = vmAttachedVol.getVmInstance( );
+      v.setState( State.BUSY );
+    } catch ( NoSuchElementException ex ) {
+      v.setMappedState( status );
+    }
+    if ( v.getSize( ) <= 0 ) {
+      v.setSize( new Integer( size ) );
+    }
+    if ( "invalid".equals( v.getRemoteDevice( ) ) || "unknown".equals( v.getRemoteDevice( ) ) || v.getRemoteDevice( ) == null ) {
+      v.setRemoteDevice( actualDeviceName );
+    }
+    edu.ucsb.eucalyptus.msgs.Volume aVolume = v.morph( new edu.ucsb.eucalyptus.msgs.Volume( ) );
+    if ( vmAttachedVol != null ) {
+      aVolume.setStatus( v.mapState( ) );
+      aVolume.getAttachmentSet( ).add( VmVolumeAttachment.asAttachedVolume( vm ).apply( vmAttachedVol ) );
+      for ( AttachedVolume attachedVolume : aVolume.getAttachmentSet( ) ) {
+        if ( !attachedVolume.getDevice( ).startsWith( "/dev/" ) ) {
+          attachedVolume.setDevice( "/dev/" + attachedVolume.getDevice( ) );
+        }
+      }
+    }
+    if ( "invalid".equals( v.getRemoteDevice( ) ) && !State.FAIL.equals( v.getState( ) ) ) {
+      aVolume.setStatus( "creating" );
+    }
+    return aVolume;
+  }
+
+  private static Map<String, StorageVolume> updateVolumesInPartition( Multimap<String, Volume> partitionVolumeMap, String partition ) throws EucalyptusCloudException {
+    Map<String, StorageVolume> idStorageVolumeMap = Maps.newHashMap( );
+    ServiceConfiguration scConfig = Topology.lookup( Storage.class, Partitions.lookupByName( partition ) );
+    Iterator<String> volumeNames = Iterators.transform( partitionVolumeMap.get( partition ).iterator( ), new Function<Volume, String>( ) {
+      @Override
+      public String apply( Volume arg0 ) {
+        return arg0.getDisplayName( );
+      }
+    } );
+    DescribeStorageVolumesType descVols = new DescribeStorageVolumesType( Lists.newArrayList( volumeNames ) );
+    Dispatcher sc = ServiceDispatcher.lookup( scConfig );
+    DescribeStorageVolumesResponseType volState = sc.send( descVols );
+    for ( StorageVolume vol : volState.getVolumeSet( ) ) {
+      idStorageVolumeMap.put( vol.getVolumeId( ), vol );
+    }
+    return idStorageVolumeMap;
+  }
+
 }