diff --git a/clc/modules/loadbalancing-backend/src/main/java/com/eucalyptus/loadbalancing/activities/EventHandlerChainDelete.java b/clc/modules/loadbalancing-backend/src/main/java/com/eucalyptus/loadbalancing/activities/EventHandlerChainDelete.java index f078242ba4b..86c9430bb16 100644 --- a/clc/modules/loadbalancing-backend/src/main/java/com/eucalyptus/loadbalancing/activities/EventHandlerChainDelete.java +++ b/clc/modules/loadbalancing-backend/src/main/java/com/eucalyptus/loadbalancing/activities/EventHandlerChainDelete.java @@ -34,6 +34,7 @@ import com.eucalyptus.component.Topology; import com.eucalyptus.component.id.Eucalyptus; import com.eucalyptus.entities.Entities; +import com.eucalyptus.entities.TransactionResource; import com.eucalyptus.event.ClockTick; import com.eucalyptus.event.EventListener; import com.eucalyptus.event.Listeners; @@ -112,7 +113,20 @@ public void apply(DeleteLoadbalancerEvent evt) if(address==null || address.length()<=0) continue; try{ - EucalyptusActivityTasks.getInstance().removeARecord(dns.getZone(), dns.getName(), address); + EucalyptusActivityTasks.getInstance().removeARecord(dns.getZone(), dns.getName(), address); + try ( final TransactionResource db = + Entities.transactionFor(LoadBalancerServoInstance.class)){ + try{ + final LoadBalancerServoInstance entity = + Entities.uniqueResult(LoadBalancerServoInstance.named(instance.getInstanceId())); + entity.setDnsState(LoadBalancerServoInstance.DNS_STATE.Deregistered); + Entities.persist(entity); + db.commit(); + }catch(final Exception ex){ + LOG.error(String.format("failed to set servo instance(%s)'s dns state to deregistered", + instance.getInstanceId()), ex); + } + } }catch(Exception ex){ LOG.error(String.format("failed to remove dns a record (zone=%s, name=%s, address=%s)", dns.getZone(), dns.getName(), address), ex); @@ -448,51 +462,56 @@ public void fireEvent(ClockTick event) { if(retired == null || retired.size()<=0) return; - - /// for each: + final List retiredAndDnsClean = Lists.newArrayList(); + for(final LoadBalancerServoInstance instance: retired){ + /// make sure DNS is deregistered + if(LoadBalancerServoInstance.DNS_STATE.Deregistered.equals(instance.getDnsState())) + retiredAndDnsClean.add(instance); + } + /// for each: // describe instances - final List param = Lists.newArrayList(); - final Map latestState = Maps.newHashMap(); - for(final LoadBalancerServoInstance instance : retired){ - /// call describe instance - String instanceId = instance.getInstanceId(); - if(instanceId == null) - continue; - param.clear(); - param.add(instanceId); - String instanceState = null; - try{ - final List result = - EucalyptusActivityTasks.getInstance().describeSystemInstances(param); - if (result.isEmpty()) - instanceState= "terminated"; - else - instanceState = result.get(0).getStateName(); - }catch(final Exception ex){ - LOG.warn("failed to query instances", ex); - continue; - } - latestState.put(instanceId, instanceState); - } - + final List param = Lists.newArrayList(); + final Map latestState = Maps.newHashMap(); + for(final LoadBalancerServoInstance instance : retiredAndDnsClean){ + /// call describe instance + String instanceId = instance.getInstanceId(); + if(instanceId == null) + continue; + param.clear(); + param.add(instanceId); + String instanceState = null; + try{ + final List result = + EucalyptusActivityTasks.getInstance().describeSystemInstances(param); + if (result.isEmpty()) + instanceState= "terminated"; + else + instanceState = result.get(0).getStateName(); + }catch(final Exception ex){ + LOG.warn("failed to query instances", ex); + continue; + } + latestState.put(instanceId, instanceState); + } + // if state==terminated or describe instances return no result, // delete the database record - for(String instanceId : latestState.keySet()){ - String state = latestState.get(instanceId); - if(state.equals("terminated")){ - final EntityTransaction db2 = Entities.get( LoadBalancerServoInstance.class ); - try{ - LoadBalancerServoInstance toDelete = Entities.uniqueResult(LoadBalancerServoInstance.named(instanceId)); - Entities.delete(toDelete); - db2.commit(); - }catch(Exception ex){ - db2.rollback(); - }finally { - if(db2.isActive()) - db2.rollback(); - } - } - } + for(String instanceId : latestState.keySet()){ + String state = latestState.get(instanceId); + if(state.equals("terminated")){ + final EntityTransaction db2 = Entities.get( LoadBalancerServoInstance.class ); + try{ + LoadBalancerServoInstance toDelete = Entities.uniqueResult(LoadBalancerServoInstance.named(instanceId)); + Entities.delete(toDelete); + db2.commit(); + }catch(Exception ex){ + db2.rollback(); + }finally { + if(db2.isActive()) + db2.rollback(); + } + } + } } } } diff --git a/clc/modules/msgs/src/main/java/com/eucalyptus/configurable/PropertyChangeListeners.java b/clc/modules/msgs/src/main/java/com/eucalyptus/configurable/PropertyChangeListeners.java index 084fce03597..a69df3f09f1 100644 --- a/clc/modules/msgs/src/main/java/com/eucalyptus/configurable/PropertyChangeListeners.java +++ b/clc/modules/msgs/src/main/java/com/eucalyptus/configurable/PropertyChangeListeners.java @@ -64,9 +64,12 @@ import com.eucalyptus.configurable.PropertyDirectory.NoopEventListener; import com.google.common.collect.Constraint; +import org.apache.log4j.Logger; + public class PropertyChangeListeners { - + + public static final Logger LOG = Logger.getLogger(PropertyChangeListener.class); public static void applyConstraint( final Object newValue, final Constraint... constraints ) throws ConfigurablePropertyException { for ( final Constraint testNewValue : constraints ) { try { @@ -118,6 +121,8 @@ public static PropertyChangeListener getListenerFromClass( Class { + public static class RestartWebServicesListener implements PropertyChangeListener { @Override - public void fireChange( ConfigurableProperty t, Integer newValue ) throws ConfigurablePropertyException { - WebServices.restart( ); + public void fireChange( ConfigurableProperty t, Object newValue ) throws ConfigurablePropertyException { +// Calling this here has issues... A periodic poller was added WebServicePropertiesChangedEventListener +// WebServices.restart( ); } } - + + public static class CheckNonNegativeIntegerAndRestartWebServicesListener implements PropertyChangeListener { + @Override + public void fireChange( ConfigurableProperty t, Object newValue ) throws ConfigurablePropertyException { + int value = -1; + try { + value = Integer.parseInt((String) newValue); + } catch (Exception ex) { + throw new ConfigurablePropertyException("Invalid value " + newValue); + } + if (value < 0) { + throw new ConfigurablePropertyException("Invalid value " + newValue); + } + new RestartWebServicesListener().fireChange(t, newValue); + } + } + + public static class CheckNonNegativeLongAndRestartWebServicesListener implements PropertyChangeListener { + @Override + public void fireChange( ConfigurableProperty t, Object newValue ) throws ConfigurablePropertyException { + long value = -1; + try { + value = Long.parseLong((String) newValue); + } catch (Exception ex) { + throw new ConfigurablePropertyException("Invalid value " + newValue); + } + if (value < 0) { + throw new ConfigurablePropertyException("Invalid value " + newValue); + } + new RestartWebServicesListener().fireChange(t, newValue); + } + } + + public static class CheckBooleanAndRestartWebServicesListener implements PropertyChangeListener { + @Override + public void fireChange( ConfigurableProperty t, Object newValue ) throws ConfigurablePropertyException { + if ((newValue == null) || (!((String) newValue).equalsIgnoreCase("true") && !((String) newValue).equalsIgnoreCase("false"))) { + throw new ConfigurablePropertyException("Invalid value " + newValue); + } + new RestartWebServicesListener().fireChange(t, newValue); + } + } + + private static Logger LOG = Logger.getLogger( WebServices.class ); private static Executor clientWorkerThreadPool; private static NioClientSocketChannelFactory nioClientSocketChannelFactory; @@ -236,7 +284,6 @@ public ChannelPipeline getPipeline( ) throws Exception { final Channel serverChannel = bootstrap.bind( new InetSocketAddress( StackConfiguration.PORT ) ); serverChannelGroup.add( serverChannel ); } - serverChannelGroup.add( bootstrap.bind( new InetSocketAddress( 443 ) ) );//GRZE:HACKHACK: always bind 443 try { final Channel serverChannel = bootstrap.bind( new InetSocketAddress( StackConfiguration.INTERNAL_PORT ) ); serverChannelGroup.add( serverChannel ); @@ -257,7 +304,148 @@ public void run( ) { } } - + + public static class WebServicePropertiesChangedEventListener implements EventListener { + // These are all the properties in StackConfiguration that have the RestartWebServicesListener. + private Integer CHANNEL_CONNECT_TIMEOUT = 500; + private Boolean SERVER_CHANNEL_REUSE_ADDRESS = true; + private Boolean SERVER_CHANNEL_NODELAY = true; + private boolean CHANNEL_REUSE_ADDRESS = true; + private Boolean CHANNEL_KEEP_ALIVE = true; + private Boolean CHANNEL_NODELAY = true; + private Integer SERVER_POOL_MAX_THREADS = 128; + private Long SERVER_POOL_MAX_MEM_PER_CONN = 0L; + private Long SERVER_POOL_TOTAL_MEM = 0L; + private Long SERVER_POOL_TIMEOUT_MILLIS = 500L; + private Integer SERVER_BOSS_POOL_MAX_THREADS = 128; + private Long SERVER_BOSS_POOL_MAX_MEM_PER_CONN = 0L; + private Long SERVER_BOSS_POOL_TOTAL_MEM = 0L; + private Long SERVER_BOSS_POOL_TIMEOUT_MILLIS = 500L; + private Integer PORT = 8773; + private AtomicBoolean isRunning = new AtomicBoolean(false); + + public static void register( ) { + Listeners.register(Hertz.class, new WebServicePropertiesChangedEventListener()); + } + + @Override + public void fireEvent( final Hertz event ) { + if (Bootstrap.isOperational() && event.isAsserted( 60 ) && isRunning.compareAndSet(false, true)) { + LOG.trace("Checking for updates to bootstrap.webservices properties"); + boolean different = false; + // temp vars so only look at StackConfiguration.* once (in case they change in the meantime) + Integer NEW_CHANNEL_CONNECT_TIMEOUT = StackConfiguration.CHANNEL_CONNECT_TIMEOUT; + Boolean NEW_SERVER_CHANNEL_REUSE_ADDRESS = StackConfiguration.SERVER_CHANNEL_REUSE_ADDRESS; + Boolean NEW_SERVER_CHANNEL_NODELAY = StackConfiguration.SERVER_CHANNEL_NODELAY; + boolean NEW_CHANNEL_REUSE_ADDRESS = StackConfiguration.CHANNEL_REUSE_ADDRESS; + Boolean NEW_CHANNEL_KEEP_ALIVE = StackConfiguration.CHANNEL_KEEP_ALIVE; + Boolean NEW_CHANNEL_NODELAY = StackConfiguration.CHANNEL_NODELAY; + Integer NEW_SERVER_POOL_MAX_THREADS = StackConfiguration.SERVER_POOL_MAX_THREADS; + Long NEW_SERVER_POOL_MAX_MEM_PER_CONN = StackConfiguration.SERVER_POOL_MAX_MEM_PER_CONN; + Long NEW_SERVER_POOL_TOTAL_MEM = StackConfiguration.SERVER_POOL_TOTAL_MEM; + Long NEW_SERVER_POOL_TIMEOUT_MILLIS = StackConfiguration.SERVER_POOL_TIMEOUT_MILLIS; + Integer NEW_SERVER_BOSS_POOL_MAX_THREADS = StackConfiguration.SERVER_BOSS_POOL_MAX_THREADS; + Long NEW_SERVER_BOSS_POOL_MAX_MEM_PER_CONN = StackConfiguration.SERVER_BOSS_POOL_MAX_MEM_PER_CONN; + Long NEW_SERVER_BOSS_POOL_TOTAL_MEM = StackConfiguration.SERVER_BOSS_POOL_TOTAL_MEM; + Long NEW_SERVER_BOSS_POOL_TIMEOUT_MILLIS = StackConfiguration.SERVER_BOSS_POOL_TIMEOUT_MILLIS; + Integer NEW_PORT = StackConfiguration.PORT; + if (!CHANNEL_CONNECT_TIMEOUT.equals(NEW_CHANNEL_CONNECT_TIMEOUT)) { + LOG.info("bootstrap.webservices.channel_connect_timeout has changed: oldValue = " + CHANNEL_CONNECT_TIMEOUT + ", newValue = " + NEW_CHANNEL_CONNECT_TIMEOUT); + CHANNEL_CONNECT_TIMEOUT = NEW_CHANNEL_CONNECT_TIMEOUT; + different = true; + } + if (SERVER_CHANNEL_REUSE_ADDRESS != NEW_SERVER_CHANNEL_REUSE_ADDRESS) { + LOG.info("bootstrap.webservices.server_channel_reuse_address has changed: oldValue = " + SERVER_CHANNEL_REUSE_ADDRESS + ", newValue = " + NEW_SERVER_CHANNEL_REUSE_ADDRESS); + SERVER_CHANNEL_REUSE_ADDRESS = NEW_SERVER_CHANNEL_REUSE_ADDRESS; + different = true; + } + if (SERVER_CHANNEL_NODELAY != NEW_SERVER_CHANNEL_NODELAY) { + LOG.info("bootstrap.webservices.server_channel_nodelay has changed: oldValue = " + SERVER_CHANNEL_NODELAY + ", newValue = " + NEW_SERVER_CHANNEL_NODELAY); + SERVER_CHANNEL_NODELAY = NEW_SERVER_CHANNEL_NODELAY; + different = true; + } + if (CHANNEL_REUSE_ADDRESS != NEW_CHANNEL_REUSE_ADDRESS) { + LOG.info("bootstrap.webservices.channel_reuse_address has changed: oldValue = " + CHANNEL_REUSE_ADDRESS + ", newValue = " + NEW_CHANNEL_REUSE_ADDRESS); + CHANNEL_REUSE_ADDRESS = NEW_CHANNEL_REUSE_ADDRESS; + different = true; + } + if (CHANNEL_KEEP_ALIVE != NEW_CHANNEL_KEEP_ALIVE) { + LOG.info("bootstrap.webservices.channel_keep_alive has changed: oldValue = " + CHANNEL_KEEP_ALIVE + ", newValue = " + NEW_CHANNEL_KEEP_ALIVE); + CHANNEL_KEEP_ALIVE = NEW_CHANNEL_KEEP_ALIVE; + different = true; + } + if (CHANNEL_NODELAY != NEW_CHANNEL_NODELAY) { + LOG.info("bootstrap.webservices.channel_nodelay has changed: oldValue = " + CHANNEL_NODELAY + ", newValue = " + NEW_CHANNEL_NODELAY); + CHANNEL_NODELAY = NEW_CHANNEL_NODELAY; + different = true; + } + if (!SERVER_POOL_MAX_THREADS.equals(NEW_SERVER_POOL_MAX_THREADS)) { + LOG.info("bootstrap.webservices.server_pool_max_threads has changed: oldValue = " + SERVER_POOL_MAX_THREADS + ", newValue = " + NEW_SERVER_POOL_MAX_THREADS); + SERVER_POOL_MAX_THREADS = NEW_SERVER_POOL_MAX_THREADS; + different = true; + } + if (!SERVER_POOL_MAX_MEM_PER_CONN.equals(NEW_SERVER_POOL_MAX_MEM_PER_CONN)) { + LOG.info("bootstrap.webservices.server_pool_max_mem_per_conn has changed: oldValue = " + SERVER_POOL_MAX_MEM_PER_CONN + ", newValue = " + NEW_SERVER_POOL_MAX_MEM_PER_CONN); + SERVER_POOL_MAX_MEM_PER_CONN = NEW_SERVER_POOL_MAX_MEM_PER_CONN; + different = true; + } + if (!SERVER_POOL_TOTAL_MEM.equals(NEW_SERVER_POOL_TOTAL_MEM)) { + LOG.info("bootstrap.webservices.server_pool_total_mem has changed: oldValue = " + SERVER_POOL_TOTAL_MEM + ", newValue = " + NEW_SERVER_POOL_TOTAL_MEM); + SERVER_POOL_TOTAL_MEM = NEW_SERVER_POOL_TOTAL_MEM; + different = true; + } + if (!SERVER_POOL_TIMEOUT_MILLIS.equals(NEW_SERVER_POOL_TIMEOUT_MILLIS)) { + LOG.info("bootstrap.webservices.server_pool_timeout_millis has changed: oldValue = " + SERVER_POOL_TIMEOUT_MILLIS + ", newValue = " + NEW_SERVER_POOL_TIMEOUT_MILLIS); + SERVER_POOL_TIMEOUT_MILLIS = NEW_SERVER_POOL_TIMEOUT_MILLIS; + different = true; + } + if (!SERVER_BOSS_POOL_MAX_THREADS.equals(NEW_SERVER_BOSS_POOL_MAX_THREADS)) { + LOG.info("bootstrap.webservices.server_boss_pool_max_threads has changed: oldValue = " + SERVER_BOSS_POOL_MAX_THREADS + ", newValue = " + NEW_SERVER_BOSS_POOL_MAX_THREADS); + SERVER_BOSS_POOL_MAX_THREADS = NEW_SERVER_BOSS_POOL_MAX_THREADS; + different = true; + } + if (!SERVER_BOSS_POOL_MAX_MEM_PER_CONN.equals(NEW_SERVER_BOSS_POOL_MAX_MEM_PER_CONN)) { + LOG.info("bootstrap.webservices.server_boss_pool_max_mem_per_conn has changed: oldValue = " + SERVER_BOSS_POOL_MAX_MEM_PER_CONN + ", newValue = " + NEW_SERVER_BOSS_POOL_MAX_MEM_PER_CONN); + SERVER_BOSS_POOL_MAX_MEM_PER_CONN = NEW_SERVER_BOSS_POOL_MAX_MEM_PER_CONN; + different = true; + } + if (!SERVER_BOSS_POOL_TOTAL_MEM.equals(NEW_SERVER_BOSS_POOL_TOTAL_MEM)) { + LOG.info("bootstrap.webservices.server_boss_pool_total_mem has changed: oldValue = " + SERVER_BOSS_POOL_TOTAL_MEM + ", newValue = " + NEW_SERVER_BOSS_POOL_TOTAL_MEM); + SERVER_BOSS_POOL_TOTAL_MEM = NEW_SERVER_BOSS_POOL_TOTAL_MEM; + different = true; + } + if (!SERVER_BOSS_POOL_TIMEOUT_MILLIS.equals(NEW_SERVER_BOSS_POOL_TIMEOUT_MILLIS)) { + LOG.info("bootstrap.webservices.server_boss_pool_timeout_millis has changed: oldValue = " + SERVER_BOSS_POOL_TIMEOUT_MILLIS + ", newValue = " + NEW_SERVER_BOSS_POOL_TIMEOUT_MILLIS); + SERVER_BOSS_POOL_TIMEOUT_MILLIS = NEW_SERVER_BOSS_POOL_TIMEOUT_MILLIS; + different = true; + } + if (!PORT.equals(NEW_PORT)) { + LOG.info("bootstrap.webservices.port has changed: oldValue = " + PORT + ", newValue = " + NEW_PORT); + PORT = NEW_PORT; + different = true; + } + if (different) { + LOG.info("One or more bootstrap.webservices properties have changed, calling WebServices.restart() [May change ports]"); + new Thread() { + public void run() { + try { + restart(); + LOG.info("WebServices.restart() complete"); + } catch (Exception ex) { + LOG.error(ex, ex); + } finally { + isRunning.set(false); + } + } + }.start(); + } else { + isRunning.set(false); + LOG.trace("No updates found to webserver properties"); + } + } + } + } + private static DefaultChannelGroup channelGroup( ) { return new DefaultChannelGroup( Empyrean.INSTANCE.getFullName( ) + ":" + WebServices.class.getSimpleName( ) @@ -311,4 +499,5 @@ private static Executor workerPool( ) { TimeUnit.MILLISECONDS ); return workerPool; } + } diff --git a/clc/modules/object-storage-common/src/main/java/com/eucalyptus/objectstorage/entities/Bucket.java b/clc/modules/object-storage-common/src/main/java/com/eucalyptus/objectstorage/entities/Bucket.java index 14dbf526af5..ecf9deaa546 100644 --- a/clc/modules/object-storage-common/src/main/java/com/eucalyptus/objectstorage/entities/Bucket.java +++ b/clc/modules/object-storage-common/src/main/java/com/eucalyptus/objectstorage/entities/Bucket.java @@ -24,6 +24,7 @@ import com.eucalyptus.objectstorage.util.OSGUtil; import com.eucalyptus.objectstorage.util.ObjectStorageProperties; import com.eucalyptus.objectstorage.util.ObjectStorageProperties.VersioningStatus; +import com.eucalyptus.storage.common.DateFormatter; import com.eucalyptus.storage.msgs.s3.AccessControlPolicy; import com.eucalyptus.storage.msgs.s3.BucketListEntry; import org.hibernate.annotations.Cache; @@ -344,7 +345,7 @@ public String toString() { } public BucketListEntry toBucketListEntry() { - return new BucketListEntry(this.getBucketName(), OSGUtil.dateToListingFormattedString(this.getCreationTimestamp())); + return new BucketListEntry(this.getBucketName(), DateFormatter.dateToListingFormattedString(this.getCreationTimestamp())); } @Override diff --git a/clc/modules/object-storage-common/src/main/java/com/eucalyptus/objectstorage/entities/ObjectEntity.java b/clc/modules/object-storage-common/src/main/java/com/eucalyptus/objectstorage/entities/ObjectEntity.java index 33690102c81..99ee4296b35 100644 --- a/clc/modules/object-storage-common/src/main/java/com/eucalyptus/objectstorage/entities/ObjectEntity.java +++ b/clc/modules/object-storage-common/src/main/java/com/eucalyptus/objectstorage/entities/ObjectEntity.java @@ -26,6 +26,7 @@ import com.eucalyptus.objectstorage.exceptions.s3.AccountProblemException; import com.eucalyptus.objectstorage.util.OSGUtil; import com.eucalyptus.objectstorage.util.ObjectStorageProperties; +import com.eucalyptus.storage.common.DateFormatter; import com.eucalyptus.storage.msgs.s3.CanonicalUser; import com.eucalyptus.storage.msgs.s3.DeleteMarkerEntry; import com.eucalyptus.storage.msgs.s3.KeyEntry; @@ -433,7 +434,7 @@ public ListEntry toListEntry() { ListEntry e = new ListEntry(); e.setEtag("\"" + this.geteTag() + "\""); e.setKey(this.getObjectKey()); - e.setLastModified(OSGUtil.dateToListingFormattedString(this.getObjectModifiedTimestamp())); + e.setLastModified(DateFormatter.dateToListingFormattedString(this.getObjectModifiedTimestamp())); e.setSize(this.getSize()); e.setStorageClass(this.getStorageClass()); e.setOwner(new CanonicalUser(this.getOwnerCanonicalId(), this.getOwnerDisplayName())); @@ -451,7 +452,7 @@ public KeyEntry toVersionEntry() { e.setEtag("\"" + this.geteTag() + "\""); e.setKey(this.getObjectKey()); e.setVersionId(this.getVersionId()); - e.setLastModified(OSGUtil.dateToListingFormattedString(this.getObjectModifiedTimestamp())); + e.setLastModified(DateFormatter.dateToListingFormattedString(this.getObjectModifiedTimestamp())); e.setSize(this.getSize()); e.setIsLatest(this.isLatest); e.setStorageClass(this.getStorageClass()); @@ -461,7 +462,7 @@ public KeyEntry toVersionEntry() { DeleteMarkerEntry e = new DeleteMarkerEntry(); e.setKey(this.getObjectKey()); e.setVersionId(this.getVersionId()); - e.setLastModified(OSGUtil.dateToListingFormattedString(this.getObjectModifiedTimestamp())); + e.setLastModified(DateFormatter.dateToListingFormattedString(this.getObjectModifiedTimestamp())); e.setIsLatest(this.isLatest); e.setOwner(new CanonicalUser(this.getOwnerCanonicalId(), this.getOwnerDisplayName())); return e; diff --git a/clc/modules/object-storage-common/src/main/java/com/eucalyptus/objectstorage/util/OSGUtil.java b/clc/modules/object-storage-common/src/main/java/com/eucalyptus/objectstorage/util/OSGUtil.java index 7773ae3867f..a48305e99cf 100644 --- a/clc/modules/object-storage-common/src/main/java/com/eucalyptus/objectstorage/util/OSGUtil.java +++ b/clc/modules/object-storage-common/src/main/java/com/eucalyptus/objectstorage/util/OSGUtil.java @@ -119,76 +119,4 @@ public static String[] getTarget(String operationPath) { operationPath = operationPath.substring(1); return operationPath.split("/"); } - - /** - * Helper to do the ISO8601 formatting as found in object/bucket lists - * - * @param d - * @return - */ - public static String dateToListingFormattedString(Date d) { - if (d == null) { - return null; - } else { - try { - return DateUtils.format(d.getTime(), DateUtils.ALT_ISO8601_DATE_PATTERN); - } catch (Exception e) { - return null; - } - } - } - - /** - * Helper to parse the ISO8601 date, as found in listings - * - * @param header - * @return - */ - public static Date dateFromListingFormattedString(String header) { - if (header == null) { - return null; - } else { - try { - return DateUtils.parseIso8601DateTimeOrDate(header); - } catch (Exception e) { - return null; - } - } - } - - /** - * Parses an RFC-822 formated date, as found in headers - * - * @param dateStr - * @return - */ - public static Date dateFromHeaderFormattedString(String dateStr) { - if (dateStr == null) { - return null; - } else { - try { - return DateUtils.parseRfc822DateTime(dateStr); - } catch (Exception e) { - return null; - } - } - } - - /** - * Helper to do the RFC822 formatting for placement in HTTP headers - * - * @param d - * @return - */ - public static String dateToHeaderFormattedString(Date d) { - if (d == null) { - return null; - } else { - try { - return DateUtils.format(d.getTime(), DateUtils.RFC822_DATETIME_PATTERN); - } catch (Exception e) { - return null; - } - } - } } diff --git a/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/ObjectFactoryImpl.java b/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/ObjectFactoryImpl.java index c37e30e8d4a..222e0f2416e 100644 --- a/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/ObjectFactoryImpl.java +++ b/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/ObjectFactoryImpl.java @@ -67,6 +67,7 @@ import com.eucalyptus.objectstorage.providers.ObjectStorageProviderClient; import com.eucalyptus.objectstorage.util.AclUtils; import com.eucalyptus.objectstorage.util.ObjectStorageProperties; +import com.eucalyptus.storage.common.DateFormatter; import com.eucalyptus.storage.msgs.s3.AccessControlPolicy; import com.eucalyptus.storage.msgs.s3.MetaDataEntry; import com.eucalyptus.storage.msgs.s3.Part; @@ -114,6 +115,7 @@ public ObjectEntity copyObject(@Nonnull final ObjectStorageProviderClient provid } final String etag; + Date lastModified; CopyObjectResponseType response; try { @@ -161,7 +163,8 @@ public Object call() throws Exception { } }; response = waitForCompletion(putTask, uploadingObject.getObjectUuid(), updateTimeout, failTime, checkIntervalSec); - // lastModified = response.getLastModified(); // TODO should CopyObjectResponseType.getLastModified return a Date? + // Last modified date in copy response is in ISO8601 format as per S3 spec + lastModified = DateFormatter.dateFromListingFormattedString(response.getLastModified()); etag = response.getEtag(); } catch(Exception e) { @@ -181,7 +184,7 @@ public Object call() throws Exception { try { //fireRepairTask(bucket, savedEntity.getObjectKey()); //Update metadata to "extant". Retry as necessary - return ObjectMetadataManagers.getInstance().finalizeCreation(entity, new Date(), etag); + return ObjectMetadataManagers.getInstance().finalizeCreation(entity, lastModified, etag); } catch(Exception e) { LOG.warn("Failed to update object metadata for finalization. Failing PUT operation", e); throw new InternalErrorException(entity.getResourceFullName()); @@ -235,6 +238,8 @@ else if (metadataDirective == null || "".equals(metadataDirective) || "COPY".equ response.setStatusMessage(port.getStatusMessage()); response.setEtag(port.getEtag()); response.setMetaData(port.getMetaData()); + // Last modified date in copy response is in ISO8601 format as per S3 API + response.setLastModified(DateFormatter.dateToListingFormattedString(port.getLastModified())); return response; } diff --git a/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/ObjectStorageGateway.java b/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/ObjectStorageGateway.java index d2ebe055bce..9b814937282 100644 --- a/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/ObjectStorageGateway.java +++ b/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/ObjectStorageGateway.java @@ -140,9 +140,9 @@ import com.eucalyptus.objectstorage.providers.ObjectStorageProviderClient; import com.eucalyptus.objectstorage.providers.ObjectStorageProviders; import com.eucalyptus.objectstorage.util.AclUtils; -import com.eucalyptus.objectstorage.util.OSGUtil; import com.eucalyptus.objectstorage.util.ObjectStorageProperties; import com.eucalyptus.reporting.event.S3ObjectEvent; +import com.eucalyptus.storage.common.DateFormatter; import com.eucalyptus.storage.msgs.s3.AccessControlList; import com.eucalyptus.storage.msgs.s3.AccessControlPolicy; import com.eucalyptus.storage.msgs.s3.BucketListEntry; @@ -163,7 +163,6 @@ import edu.ucsb.eucalyptus.msgs.ComponentProperty; import edu.ucsb.eucalyptus.util.SystemUtil; import org.apache.log4j.Logger; -import org.apache.tools.ant.util.DateUtils; import org.jboss.netty.handler.codec.http.DefaultHttpResponse; import org.jboss.netty.handler.codec.http.HttpHeaders; import org.jboss.netty.handler.codec.http.HttpResponseStatus; @@ -1017,7 +1016,7 @@ protected DefaultHttpResponse createHttpResponse(ObjectStorageDataGetResponseTyp httpResponse.addHeader( HttpHeaders.Names.CONTENT_TYPE, contentType != null ? contentType : "binary/octet-stream" ); if(etag != null) httpResponse.addHeader(HttpHeaders.Names.ETAG, "\"" + etag + "\""); //etag in quotes, per s3-spec. - httpResponse.addHeader(HttpHeaders.Names.LAST_MODIFIED, OSGUtil.dateToHeaderFormattedString(lastModified)); + httpResponse.addHeader(HttpHeaders.Names.LAST_MODIFIED, DateFormatter.dateToHeaderFormattedString(lastModified)); if(contentDisposition != null) { httpResponse.addHeader("Content-Disposition", contentDisposition); } @@ -1026,7 +1025,7 @@ protected DefaultHttpResponse createHttpResponse(ObjectStorageDataGetResponseTyp if(versionId != null) { httpResponse.addHeader(ObjectStorageProperties.X_AMZ_VERSION_ID, versionId); } - httpResponse.setHeader(HttpHeaders.Names.DATE, OSGUtil.dateToHeaderFormattedString(new Date())); + httpResponse.setHeader(HttpHeaders.Names.DATE, DateFormatter.dateToHeaderFormattedString(new Date())); //write extra headers if(reply.getByteRangeEnd() != null) { @@ -1290,8 +1289,6 @@ public CopyObjectResponseType copyObject(CopyObjectType request) throws S3Except destObject.setObjectModifiedTimestamp(srcLastMod); destObject.setIsLatest(Boolean.TRUE); - reply.setEtag(etag); - reply.setLastModified(DateUtils.format(srcLastMod.getTime(), DateUtils.ALT_ISO8601_DATE_PATTERN)); if (destBucket.getVersioning() == ObjectStorageProperties.VersioningStatus.Enabled ) { reply.setCopySourceVersionId(versionId); reply.setVersionId(versionId); @@ -1307,7 +1304,9 @@ public CopyObjectResponseType copyObject(CopyObjectType request) throws S3Except request.setDestinationObject(destObjUuid); request.setDestinationBucket(destBckUuid); try { - OsgObjectFactory.getFactory().copyObject(ospClient, destObject, request, requestUser, metadataDirective); + ObjectEntity objectEntity = OsgObjectFactory.getFactory().copyObject(ospClient, destObject, request, requestUser, metadataDirective); + reply.setLastModified(DateFormatter.dateToListingFormattedString(objectEntity.getObjectModifiedTimestamp())); + reply.setEtag(objectEntity.geteTag()); try { fireObjectCreationEvent(destBucket.getBucketName(), destObject.getObjectKey(), versionId, requestUser.getUserId(), destObject.getSize(), origDestObjectSize); diff --git a/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/pipeline/binding/ObjectStorageRESTBinding.java b/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/pipeline/binding/ObjectStorageRESTBinding.java index 515be7a2c80..155cd23a173 100644 --- a/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/pipeline/binding/ObjectStorageRESTBinding.java +++ b/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/pipeline/binding/ObjectStorageRESTBinding.java @@ -88,6 +88,7 @@ import com.eucalyptus.objectstorage.pipeline.handlers.ObjectStorageAuthenticationHandler; import com.eucalyptus.objectstorage.util.OSGUtil; import com.eucalyptus.objectstorage.util.ObjectStorageProperties; +import com.eucalyptus.storage.common.DateFormatter; import com.eucalyptus.storage.msgs.BucketLogData; import com.eucalyptus.storage.msgs.s3.AccessControlList; import com.eucalyptus.storage.msgs.s3.AccessControlPolicy; @@ -232,7 +233,7 @@ public void outgoingMessage( ChannelHandlerContext ctx, MessageEvent event ) thr ChannelBuffer buffer = ChannelBuffers.wrappedBuffer( req ); httpResponse.setHeader( HttpHeaders.Names.CONTENT_LENGTH, String.valueOf(buffer.readableBytes() ) ); httpResponse.setHeader( HttpHeaders.Names.CONTENT_TYPE, "application/xml" ); - httpResponse.setHeader( HttpHeaders.Names.DATE, OSGUtil.dateToHeaderFormattedString(new Date())); + httpResponse.setHeader( HttpHeaders.Names.DATE, DateFormatter.dateToHeaderFormattedString(new Date())); httpResponse.setHeader( "x-amz-request-id", msg.getCorrelationId()); httpResponse.setContent( buffer ); } diff --git a/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/pipeline/handlers/ObjectStorageGETOutboundHandler.java b/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/pipeline/handlers/ObjectStorageGETOutboundHandler.java index 0eba963bff7..ab225a78ff6 100644 --- a/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/pipeline/handlers/ObjectStorageGETOutboundHandler.java +++ b/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/pipeline/handlers/ObjectStorageGETOutboundHandler.java @@ -68,6 +68,7 @@ import com.eucalyptus.objectstorage.util.OSGUtil; import com.eucalyptus.objectstorage.util.ObjectStorageProperties; import com.eucalyptus.storage.common.ChunkedDataStream; +import com.eucalyptus.storage.common.DateFormatter; import com.eucalyptus.storage.msgs.s3.MetaDataEntry; import com.eucalyptus.ws.WebServicesException; import com.eucalyptus.ws.server.Statistics; @@ -198,7 +199,7 @@ protected DefaultHttpResponse createHttpResponse(ObjectStorageDataGetResponseTyp if (etag != null) { httpResponse.addHeader(HttpHeaders.Names.ETAG, "\"" + etag + "\""); //etag in quotes, per s3-spec. } - httpResponse.addHeader(HttpHeaders.Names.LAST_MODIFIED, OSGUtil.dateToHeaderFormattedString(reply.getLastModified())); + httpResponse.addHeader(HttpHeaders.Names.LAST_MODIFIED, DateFormatter.dateToHeaderFormattedString(reply.getLastModified())); if (contentDisposition != null) { httpResponse.addHeader("Content-Disposition", contentDisposition); @@ -208,7 +209,7 @@ protected DefaultHttpResponse createHttpResponse(ObjectStorageDataGetResponseTyp if (versionId != null && !ObjectStorageProperties.NULL_VERSION_ID.equals(versionId)) { httpResponse.addHeader(ObjectStorageProperties.X_AMZ_VERSION_ID, versionId); } - httpResponse.setHeader(HttpHeaders.Names.DATE, OSGUtil.dateToHeaderFormattedString(new Date())); + httpResponse.setHeader(HttpHeaders.Names.DATE, DateFormatter.dateToHeaderFormattedString(new Date())); //Add user metadata for (MetaDataEntry m : reply.getMetaData()) { diff --git a/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/pipeline/handlers/ObjectStorageHEADOutboundHandler.java b/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/pipeline/handlers/ObjectStorageHEADOutboundHandler.java index df1b682dd36..58ad4ee3c7f 100644 --- a/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/pipeline/handlers/ObjectStorageHEADOutboundHandler.java +++ b/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/pipeline/handlers/ObjectStorageHEADOutboundHandler.java @@ -66,6 +66,7 @@ import com.eucalyptus.objectstorage.msgs.ObjectStorageDataResponseType; import com.eucalyptus.objectstorage.util.OSGUtil; import com.eucalyptus.objectstorage.util.ObjectStorageProperties; +import com.eucalyptus.storage.common.DateFormatter; import com.eucalyptus.storage.msgs.s3.MetaDataEntry; import com.eucalyptus.ws.handlers.MessageStackHandler; import com.google.common.base.Strings; @@ -87,7 +88,7 @@ public void outgoingMessage(ChannelHandlerContext ctx, MessageEvent event) throw if (event.getMessage() instanceof MappingHttpResponse) { MappingHttpResponse httpResponse = (MappingHttpResponse) event.getMessage(); BaseMessage msg = (BaseMessage) httpResponse.getMessage(); - httpResponse.setHeader("Date", OSGUtil.dateToHeaderFormattedString(new Date())); + httpResponse.setHeader("Date", DateFormatter.dateToHeaderFormattedString(new Date())); httpResponse.setHeader("x-amz-request-id", msg.getCorrelationId()); if (msg instanceof ObjectStorageDataResponseType) { httpResponse.addHeader(HttpHeaders.Names.ETAG, "\"" + ((ObjectStorageDataResponseType) msg).getEtag() + "\""); //etag in quotes, per s3-spec. diff --git a/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/pipeline/handlers/ObjectStorageOutboundHandler.java b/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/pipeline/handlers/ObjectStorageOutboundHandler.java index d3d94481b7a..4b86eafe356 100644 --- a/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/pipeline/handlers/ObjectStorageOutboundHandler.java +++ b/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/pipeline/handlers/ObjectStorageOutboundHandler.java @@ -70,6 +70,7 @@ import com.eucalyptus.objectstorage.msgs.PutObjectResponseType; import com.eucalyptus.objectstorage.util.OSGUtil; import com.eucalyptus.objectstorage.util.ObjectStorageProperties; +import com.eucalyptus.storage.common.DateFormatter; import com.eucalyptus.ws.handlers.MessageStackHandler; import edu.ucsb.eucalyptus.msgs.BaseMessage; import org.apache.log4j.Logger; @@ -121,7 +122,7 @@ public void outgoingMessage(ChannelHandlerContext ctx, MessageEvent event) throw PutObjectResponseType putObjectResponse = (PutObjectResponseType) msg; httpResponse.setHeader(HttpHeaders.Names.ETAG, '\"' + putObjectResponse.getEtag() + '\"'); if (putObjectResponse.getLastModified() != null) { - httpResponse.setHeader(HttpHeaders.Names.LAST_MODIFIED, OSGUtil.dateToHeaderFormattedString(putObjectResponse.getLastModified())); + httpResponse.setHeader(HttpHeaders.Names.LAST_MODIFIED, DateFormatter.dateToHeaderFormattedString(putObjectResponse.getLastModified())); } if (putObjectResponse.getVersionId() != null) { httpResponse.setHeader(ObjectStorageProperties.X_AMZ_VERSION_ID, putObjectResponse.getVersionId()); @@ -132,7 +133,7 @@ public void outgoingMessage(ChannelHandlerContext ctx, MessageEvent event) throw httpResponse.addHeader(HttpHeaders.Names.ETAG, '\"' + response.getEtag() + '\"'); } if (response.getLastModified() != null) { - httpResponse.addHeader(HttpHeaders.Names.LAST_MODIFIED, OSGUtil.dateToHeaderFormattedString(response.getLastModified())); + httpResponse.addHeader(HttpHeaders.Names.LAST_MODIFIED, DateFormatter.dateToHeaderFormattedString(response.getLastModified())); } if (response.getVersionId() != null) { httpResponse.addHeader(ObjectStorageProperties.X_AMZ_VERSION_ID, response.getVersionId()); diff --git a/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/providers/s3/S3ProviderClient.java b/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/providers/s3/S3ProviderClient.java index 2342ec198b7..45f3c18cf01 100644 --- a/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/providers/s3/S3ProviderClient.java +++ b/clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/providers/s3/S3ProviderClient.java @@ -135,6 +135,7 @@ import com.eucalyptus.objectstorage.util.AclUtils; import com.eucalyptus.objectstorage.util.OSGUtil; import com.eucalyptus.objectstorage.client.OsgInternalS3Client; +import com.eucalyptus.storage.common.DateFormatter; import com.eucalyptus.storage.msgs.s3.AccessControlList; import com.eucalyptus.storage.msgs.s3.AccessControlPolicy; import com.eucalyptus.storage.msgs.s3.BucketListEntry; @@ -382,7 +383,7 @@ public ListAllMyBucketsResponseType listAllMyBuckets(ListAllMyBucketsType reques //Map s3 client result to euca response message List result = s3Client.listBuckets(listRequest); for(Bucket b : result) { - myBucketList.getBuckets().add(new BucketListEntry(b.getName(), OSGUtil.dateToHeaderFormattedString(b.getCreationDate()))); + myBucketList.getBuckets().add(new BucketListEntry(b.getName(), DateFormatter.dateToHeaderFormattedString(b.getCreationDate()))); } reply.setBucketList(myBucketList); @@ -576,7 +577,7 @@ public ListBucketResponseType listBucket(ListBucketType request) throws S3Except //Add entry, note that the canonical user is set based on requesting user, not returned user reply.getContents().add(new ListEntry( obj.getKey(), - OSGUtil.dateToHeaderFormattedString(obj.getLastModified()), + DateFormatter.dateToHeaderFormattedString(obj.getLastModified()), obj.getETag(), obj.getSize(), getCanonicalUser(requestUser), @@ -771,7 +772,7 @@ public CopyObjectResponseType copyObject(CopyObjectType request) AmazonS3Client s3Client = getS3Client(requestUser, requestUser.getUserId()); CopyObjectResult result = s3Client.copyObject(copyRequest); reply.setEtag(result.getETag()); - reply.setLastModified(OSGUtil.dateToHeaderFormattedString(result.getLastModifiedDate())); + reply.setLastModified(DateFormatter.dateToListingFormattedString(result.getLastModifiedDate())); String destinationVersionId = result.getVersionId(); if (destinationVersionId != null) { reply.setCopySourceVersionId(sourceVersionId); @@ -930,7 +931,7 @@ public ListVersionsResponseType listVersions(ListVersionsType request) throws S3 v = new VersionEntry(); v.setKey(summary.getKey()); v.setVersionId(summary.getVersionId()); - v.setLastModified(OSGUtil.dateToHeaderFormattedString(summary.getLastModified())); + v.setLastModified(DateFormatter.dateToHeaderFormattedString(summary.getLastModified())); v.setEtag(summary.getETag()); v.setIsLatest(summary.isLatest()); v.setOwner(owner); @@ -940,7 +941,7 @@ public ListVersionsResponseType listVersions(ListVersionsType request) throws S3 d = new DeleteMarkerEntry(); d.setIsLatest(summary.isLatest()); d.setKey(summary.getKey()); - d.setLastModified(OSGUtil.dateToHeaderFormattedString(summary.getLastModified())); + d.setLastModified(DateFormatter.dateToHeaderFormattedString(summary.getLastModified())); d.setOwner(owner); d.setVersionId(summary.getVersionId()); versions.add(d); @@ -1081,6 +1082,7 @@ public CompleteMultipartUploadResponseType completeMultipartUpload( reply.setBucket(bucketName); reply.setKey(key); reply.setLocation(result.getLocation()); + reply.setLastModified(new Date()); } catch(AmazonServiceException e) { LOG.debug("Error from backend", e); throw S3ExceptionMapper.fromAWSJavaSDK(e); diff --git a/clc/modules/object-storage/src/test/java/com/eucalyptus/objectstorage/providers/InMemoryProvider.java b/clc/modules/object-storage/src/test/java/com/eucalyptus/objectstorage/providers/InMemoryProvider.java index 7e7758fc0c2..50eb1acf6c4 100644 --- a/clc/modules/object-storage/src/test/java/com/eucalyptus/objectstorage/providers/InMemoryProvider.java +++ b/clc/modules/object-storage/src/test/java/com/eucalyptus/objectstorage/providers/InMemoryProvider.java @@ -70,6 +70,7 @@ import com.eucalyptus.objectstorage.msgs.UploadPartType; import com.eucalyptus.objectstorage.util.OSGUtil; import com.eucalyptus.objectstorage.util.ObjectStorageProperties; +import com.eucalyptus.storage.common.DateFormatter; import com.eucalyptus.storage.msgs.s3.AccessControlList; import com.eucalyptus.storage.msgs.s3.AccessControlPolicy; import com.eucalyptus.storage.msgs.s3.BucketListEntry; @@ -166,7 +167,7 @@ private class MemoryBucket { TreeMap uploads = new TreeMap<>(); BucketListEntry toBucketListEntry() { - return new BucketListEntry(this.name, OSGUtil.dateToListingFormattedString(this.createdDate)); + return new BucketListEntry(this.name, DateFormatter.dateToListingFormattedString(this.createdDate)); } } diff --git a/clc/modules/storage-common/src/main/java/com/eucalyptus/storage/common/DateFormatter.java b/clc/modules/storage-common/src/main/java/com/eucalyptus/storage/common/DateFormatter.java new file mode 100644 index 00000000000..61c7720ef33 --- /dev/null +++ b/clc/modules/storage-common/src/main/java/com/eucalyptus/storage/common/DateFormatter.java @@ -0,0 +1,141 @@ +/************************************************************************* + * Copyright 2009-2014 Eucalyptus Systems, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 3 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see http://www.gnu.org/licenses/. + * + * Please contact Eucalyptus Systems, Inc., 6755 Hollister Ave., Goleta + * CA 93117, USA or visit http://www.eucalyptus.com/licenses/ if you need + * additional information or have any questions. + * + * This file may incorporate work covered under the following copyright + * and permission notice: + * + * Software License Agreement (BSD License) + * + * Copyright (c) 2008, Regents of the University of California + * All rights reserved. + * + * Redistribution and use of this software in source and binary forms, + * with or without modification, are permitted provided that the + * following conditions are met: + * + * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. USERS OF THIS SOFTWARE ACKNOWLEDGE + * THE POSSIBLE PRESENCE OF OTHER OPEN SOURCE LICENSED MATERIAL, + * COPYRIGHTED MATERIAL OR PATENTED MATERIAL IN THIS SOFTWARE, + * AND IF ANY SUCH MATERIAL IS DISCOVERED THE PARTY DISCOVERING + * IT MAY INFORM DR. RICH WOLSKI AT THE UNIVERSITY OF CALIFORNIA, + * SANTA BARBARA WHO WILL THEN ASCERTAIN THE MOST APPROPRIATE REMEDY, + * WHICH IN THE REGENTS' DISCRETION MAY INCLUDE, WITHOUT LIMITATION, + * REPLACEMENT OF THE CODE SO IDENTIFIED, LICENSING OF THE CODE SO + * IDENTIFIED, OR WITHDRAWAL OF THE CODE CAPABILITY TO THE EXTENT + * NEEDED TO COMPLY WITH ANY SUCH LICENSES OR RIGHTS. + ************************************************************************/ + +package com.eucalyptus.storage.common; + +import java.util.Date; + +import org.apache.tools.ant.util.DateUtils; + +public class DateFormatter { + /** + * Helper to do the ISO8601 formatting as found in object/bucket lists + * + * @param d + * @return + */ + public static String dateToListingFormattedString(Date d) { + if (d == null) { + return null; + } else { + try { + return DateUtils.format(d.getTime(), DateUtils.ALT_ISO8601_DATE_PATTERN); + } catch (Exception e) { + return null; + } + } + } + + /** + * Helper to parse the ISO8601 date, as found in listings + * + * @param header + * @return + */ + public static Date dateFromListingFormattedString(String header) { + if (header == null) { + return null; + } else { + try { + return DateUtils.parseIso8601DateTimeOrDate(header); + } catch (Exception e) { + return null; + } + } + } + + /** + * Parses an RFC-822 formated date, as found in headers + * + * @param dateStr + * @return + */ + public static Date dateFromHeaderFormattedString(String dateStr) { + if (dateStr == null) { + return null; + } else { + try { + return DateUtils.parseRfc822DateTime(dateStr); + } catch (Exception e) { + return null; + } + } + } + + /** + * Helper to do the RFC822 formatting for placement in HTTP headers + * + * @param d + * @return + */ + public static String dateToHeaderFormattedString(Date d) { + if (d == null) { + return null; + } else { + try { + return DateUtils.format(d.getTime(), DateUtils.RFC822_DATETIME_PATTERN); + } catch (Exception e) { + return null; + } + } + } +} diff --git a/clc/modules/walrus-common/src/main/java/com/eucalyptus/walrus/entities/DRBDInfo.java b/clc/modules/walrus-common/src/main/java/com/eucalyptus/walrus/entities/DRBDInfo.java index 3d9f4628972..44a71b6bdc9 100644 --- a/clc/modules/walrus-common/src/main/java/com/eucalyptus/walrus/entities/DRBDInfo.java +++ b/clc/modules/walrus-common/src/main/java/com/eucalyptus/walrus/entities/DRBDInfo.java @@ -81,7 +81,7 @@ @PersistenceContext(name="eucalyptus_walrus") @Table( name = "drbd_info" ) @Cache( usage = CacheConcurrencyStrategy.TRANSACTIONAL ) -@ConfigurableClass(root = "walrus", alias = "drbd", description = "DRBD configuration.", deferred = true) +@ConfigurableClass(root = "walrusbackend", alias = "drbd", description = "DRBD configuration.", deferred = true) public class DRBDInfo extends AbstractPersistent { @Column(name = "walrus_name", unique=true) private String name; diff --git a/clc/modules/walrus-common/src/main/java/com/eucalyptus/walrus/exceptions/InternalErrorException.java b/clc/modules/walrus-common/src/main/java/com/eucalyptus/walrus/exceptions/InternalErrorException.java index b7dd2857df1..2566b9a2672 100644 --- a/clc/modules/walrus-common/src/main/java/com/eucalyptus/walrus/exceptions/InternalErrorException.java +++ b/clc/modules/walrus-common/src/main/java/com/eucalyptus/walrus/exceptions/InternalErrorException.java @@ -69,12 +69,12 @@ public class InternalErrorException extends WalrusException { public InternalErrorException() { - super( "Not Found" ); + super( "Internal Error" ); } - public InternalErrorException(String bucket) + public InternalErrorException(String resource) { - super("NoSuchBucket", "The specified bucket was not found", "Bucket", bucket, HttpResponseStatus.NOT_FOUND); + super("InternalError", "Server reported Internal Error", "Resource", resource, HttpResponseStatus.INTERNAL_SERVER_ERROR); } diff --git a/clc/modules/walrus-common/src/main/java/com/eucalyptus/walrus/exceptions/InvalidPartException.java b/clc/modules/walrus-common/src/main/java/com/eucalyptus/walrus/exceptions/InvalidPartException.java index f7a7d0caef3..17048ee10d8 100644 --- a/clc/modules/walrus-common/src/main/java/com/eucalyptus/walrus/exceptions/InvalidPartException.java +++ b/clc/modules/walrus-common/src/main/java/com/eucalyptus/walrus/exceptions/InvalidPartException.java @@ -69,18 +69,18 @@ public class InvalidPartException extends WalrusException { public InvalidPartException() { - super( "NoSuchUpload" ); + super( "InvalidPart" ); } public InvalidPartException(String resource) { - super("NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.", "upload id: ", resource, HttpResponseStatus.NOT_FOUND); + super("InvalidPart", "One or more of the specified parts could not be found. The part might not have been uploaded, or the specified entity tag might not have matched the part's entity tag.", "Resource: ", resource, HttpResponseStatus.BAD_REQUEST); } public InvalidPartException(Throwable ex) { - super("NoSuchUpload", ex); + super("InvalidPart", ex); } public InvalidPartException(String message, Throwable ex) diff --git a/clc/modules/walrus-common/src/main/java/com/eucalyptus/walrus/exceptions/NoSuchUploadException.java b/clc/modules/walrus-common/src/main/java/com/eucalyptus/walrus/exceptions/NoSuchUploadException.java index 214947b4d91..9af8101f251 100644 --- a/clc/modules/walrus-common/src/main/java/com/eucalyptus/walrus/exceptions/NoSuchUploadException.java +++ b/clc/modules/walrus-common/src/main/java/com/eucalyptus/walrus/exceptions/NoSuchUploadException.java @@ -69,18 +69,18 @@ public class NoSuchUploadException extends WalrusException { public NoSuchUploadException() { - super( "Not Found" ); + super( "NoSuchUpload" ); } - public NoSuchUploadException(String bucket) + public NoSuchUploadException(String resource) { - super("NoSuchBucket", "The specified bucket was not found", "Bucket", bucket, HttpResponseStatus.NOT_FOUND); + super("NoSuchUpload", "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.", "upload id: ", resource, HttpResponseStatus.NOT_FOUND); } public NoSuchUploadException(Throwable ex) { - super("Not Found", ex); + super("NoSuchUpload", ex); } public NoSuchUploadException(String message, Throwable ex) diff --git a/clc/modules/walrus-common/src/main/java/com/eucalyptus/walrus/msgs/Walrus.groovy b/clc/modules/walrus-common/src/main/java/com/eucalyptus/walrus/msgs/Walrus.groovy index a42424ceb9a..0c22de13c7c 100644 --- a/clc/modules/walrus-common/src/main/java/com/eucalyptus/walrus/msgs/Walrus.groovy +++ b/clc/modules/walrus-common/src/main/java/com/eucalyptus/walrus/msgs/Walrus.groovy @@ -362,7 +362,7 @@ public class WalrusDataRequestType extends WalrusRequestType { public class WalrusDataResponseType extends WalrusStreamingResponseType { String etag; - String lastModified; + Date lastModified; Long size; ArrayList metaData = new ArrayList(); Integer errorCode; diff --git a/clc/modules/walrus/src/main/java/com/eucalyptus/walrus/WalrusFSManager.java b/clc/modules/walrus/src/main/java/com/eucalyptus/walrus/WalrusFSManager.java index 6dca23908f4..4f15bc51b86 100644 --- a/clc/modules/walrus/src/main/java/com/eucalyptus/walrus/WalrusFSManager.java +++ b/clc/modules/walrus/src/main/java/com/eucalyptus/walrus/WalrusFSManager.java @@ -85,6 +85,9 @@ import com.eucalyptus.walrus.entities.PartInfo; +import com.eucalyptus.walrus.exceptions.InternalErrorException; +import com.eucalyptus.walrus.exceptions.InvalidPartException; +import com.eucalyptus.walrus.exceptions.NoSuchUploadException; import com.eucalyptus.walrus.msgs.GetLifecycleResponseType; import com.eucalyptus.walrus.msgs.GetLifecycleType; import com.eucalyptus.walrus.msgs.LifecycleConfigurationType; @@ -99,7 +102,6 @@ import com.eucalyptus.walrus.exceptions.NoSuchLifecycleConfigurationException; import org.apache.log4j.Logger; -import org.apache.tools.ant.util.DateUtils; import org.bouncycastle.util.encoders.Base64; import org.hibernate.Criteria; import org.hibernate.criterion.Example; @@ -126,6 +128,7 @@ import com.eucalyptus.entities.Entities; import com.eucalyptus.entities.EntityWrapper; import com.eucalyptus.entities.TransactionException; +import com.eucalyptus.storage.common.DateFormatter; import com.eucalyptus.storage.common.fs.FileIO; import com.eucalyptus.storage.msgs.BucketLogData; import com.eucalyptus.storage.msgs.s3.AccessControlList; @@ -274,11 +277,11 @@ public void check() throws EucalyptusCloudException { if (!bukkits.exists()) { if (!bukkits.mkdirs()) { LOG.fatal("Unable to make bucket root directory: " + bukkitDir); - throw new EucalyptusCloudException("Invalid bucket root directory"); + throw new InternalErrorException("Invalid bucket root directory"); } } else if (!bukkits.canWrite()) { LOG.fatal("Cannot write to bucket root directory: " + bukkitDir); - throw new EucalyptusCloudException("Invalid bucket root directory"); + throw new InternalErrorException("Invalid bucket root directory"); } try { SystemUtil.setEucaReadWriteOnly(bukkitDir); @@ -325,7 +328,7 @@ private boolean bucketHasSnapshots(String bucketName) throws Exception { @Override public ListAllMyBucketsResponseType listAllMyBuckets( - ListAllMyBucketsType request) throws EucalyptusCloudException { + ListAllMyBucketsType request) throws WalrusException { LOG.info("Handling ListAllBuckets request"); ListAllMyBucketsResponseType reply = (ListAllMyBucketsResponseType) request.getReply(); @@ -353,8 +356,7 @@ public ListAllMyBucketsResponseType listAllMyBuckets( if (bucketHasSnapshots(bucketInfo.getBucketName())) { continue; } else { - buckets.add(new BucketListEntry(bucketInfo.getBucketName(), DateUtils.format(bucketInfo.getCreationDate().getTime(), - DateUtils.ALT_ISO8601_DATE_PATTERN))); + buckets.add(new BucketListEntry(bucketInfo.getBucketName(), DateFormatter.dateToListingFormattedString(bucketInfo.getCreationDate()))); } } catch (Exception e) { LOG.debug(e, e); @@ -375,7 +377,7 @@ public ListAllMyBucketsResponseType listAllMyBuckets( } } catch (EucalyptusCloudException e) { db.rollback(); - throw e; + throw new InternalErrorException(e); } catch (Exception e) { LOG.debug(e, e); db.rollback(); @@ -391,7 +393,7 @@ public ListAllMyBucketsResponseType listAllMyBuckets( * @throws EucalyptusCloudException */ @Override - public HeadBucketResponseType headBucket(HeadBucketType request) throws EucalyptusCloudException { + public HeadBucketResponseType headBucket(HeadBucketType request) throws WalrusException { HeadBucketResponseType reply = (HeadBucketResponseType) request.getReply(); Context ctx = Contexts.lookup(); Account account = ctx.getAccount(); @@ -406,7 +408,7 @@ public HeadBucketResponseType headBucket(HeadBucketType request) throws Eucalypt } catch (TransactionException e) { LOG.error("DB transaction error looking up bucket " + bucketName + ": " + e.getMessage()); LOG.debug("DB tranction exception looking up bucket " + bucketName, e); - throw new EucalyptusCloudException("Internal error doing db lookup for " + bucketName, e); + throw new HeadNoSuchBucketException("Internal error doing db lookup for " + bucketName, e); } finally { // Nothing to commit, always rollback. db.rollback(); @@ -415,7 +417,7 @@ public HeadBucketResponseType headBucket(HeadBucketType request) throws Eucalypt @Override public CreateBucketResponseType createBucket(CreateBucketType request) - throws EucalyptusCloudException { + throws WalrusException { CreateBucketResponseType reply = (CreateBucketResponseType) request .getReply(); Context ctx = Contexts.lookup(); @@ -480,7 +482,7 @@ public CreateBucketResponseType createBucket(CreateBucketType request) if (Exceptions.isCausedBy(ex, ConstraintViolationException.class)) { throw new BucketAlreadyExistsException(bucketName); } else { - throw new EucalyptusCloudException("Unable to create bucket: " + bucketName); + throw new InternalErrorException("Unable to create bucket: " + bucketName); } } @@ -540,7 +542,7 @@ private boolean checkDNSNaming(String bucketName) { } @Override - public DeleteBucketResponseType deleteBucket(DeleteBucketType request) throws EucalyptusCloudException { + public DeleteBucketResponseType deleteBucket(DeleteBucketType request) throws WalrusException { DeleteBucketResponseType reply = (DeleteBucketResponseType) request.getReply(); String bucketName = request.getBucket(); Context ctx = Contexts.lookup(); @@ -599,7 +601,7 @@ public DeleteBucketResponseType deleteBucket(DeleteBucketType request) throws Eu @Override public GetBucketAccessControlPolicyResponseType getBucketAccessControlPolicy( GetBucketAccessControlPolicyType request) - throws EucalyptusCloudException { + throws WalrusException { GetBucketAccessControlPolicyResponseType reply = (GetBucketAccessControlPolicyResponseType) request .getReply(); @@ -730,7 +732,7 @@ private static void addPermission(ArrayList grants, GrantInfo grantInfo) } } - public PutLifecycleResponseType putLifecycle( PutLifecycleType request) throws EucalyptusCloudException { + public PutLifecycleResponseType putLifecycle( PutLifecycleType request) throws WalrusException { PutLifecycleResponseType response = request.getReply(); String bucketName = request.getBucket(); @@ -864,7 +866,7 @@ else if (ex instanceof NoSuchBucketException) { return bucket; } - public GetLifecycleResponseType getLifecycle( GetLifecycleType request) throws EucalyptusCloudException { + public GetLifecycleResponseType getLifecycle( GetLifecycleType request) throws WalrusException { GetLifecycleResponseType response = request.getReply(); String bucketName = request.getBucket(); @@ -978,7 +980,7 @@ private LifecycleRuleInfo convertLifecycleRule(LifecycleRule rule, String bucket } @Override - public PutObjectResponseType putObject(PutObjectType request) throws EucalyptusCloudException { + public PutObjectResponseType putObject(PutObjectType request) throws WalrusException { PutObjectResponseType reply = (PutObjectResponseType) request.getReply(); Context ctx = Contexts.lookup(); @@ -1125,7 +1127,7 @@ public PutObjectResponseType putObject(PutObjectType request) throws EucalyptusC fileIO = storageManager.prepareForWrite(bucketName, tempObjectName); } catch (Exception ex) { messenger.removeQueue(key, randomKey); - throw new EucalyptusCloudException(ex); + throw new AccessDeniedException(ex); } } else if (WalrusDataMessage.isEOF(dataMessage)) { if (digest != null) { @@ -1187,7 +1189,7 @@ public PutObjectResponseType putObject(PutObjectType request) throws EucalyptusC } catch (IOException ex) { LOG.error(ex); messenger.removeQueue(key, randomKey); - throw new EucalyptusCloudException(objectKey); + throw new AccessDeniedException(objectKey); } lastModified = new Date(); ObjectInfo searchObject = new ObjectInfo(bucketName, objectKey); @@ -1222,7 +1224,7 @@ public PutObjectResponseType putObject(PutObjectType request) throws EucalyptusC foundObject = objectInfo; } else { dbObject.rollback(); - throw new EucalyptusCloudException("Unable to update object: " + bucketName + "/" + objectKey); + throw new InternalErrorException("Unable to update object: " + bucketName + "/" + objectKey); } } foundObject.setVersionId(versionId); @@ -1304,7 +1306,7 @@ public PutObjectResponseType putObject(PutObjectType request) throws EucalyptusC } catch (InterruptedException ex) { LOG.error(ex, ex); messenger.removeQueue(key, randomKey); - throw new EucalyptusCloudException("Transfer interrupted: " + key + "." + randomKey); + throw new InternalErrorException("Transfer interrupted: " + key + "." + randomKey); } } else { db.rollback(); @@ -1313,7 +1315,7 @@ public PutObjectResponseType putObject(PutObjectType request) throws EucalyptusC } reply.setEtag(md5); - reply.setLastModified(DateUtils.format(lastModified.getTime(), DateUtils.RFC822_DATETIME_PATTERN)); + reply.setLastModified(lastModified); return reply; } @@ -1334,7 +1336,7 @@ private void cleanupTempObject(Context ctx, String bucketName, @Override public PostObjectResponseType postObject(PostObjectType request) - throws EucalyptusCloudException { + throws WalrusException { PostObjectResponseType reply = (PostObjectResponseType) request .getReply(); @@ -1400,7 +1402,7 @@ public PostObjectResponseType postObject(PostObjectType request) @Override public PutObjectInlineResponseType putObjectInline( - PutObjectInlineType request) throws EucalyptusCloudException { + PutObjectInlineType request) throws WalrusException { PutObjectInlineResponseType reply = (PutObjectInlineResponseType) request .getReply(); Context ctx = Contexts.lookup(); @@ -1508,7 +1510,7 @@ public PutObjectInlineResponseType putObjectInline( } } catch (Exception ex) { db.rollback(); - throw new EucalyptusCloudException(ex); + throw new AccessDeniedException(ex); } md5 = Hashes.getHexString(Digest.MD5.get().digest(base64Data)); foundObject.setEtag(md5); @@ -1531,7 +1533,7 @@ public PutObjectInlineResponseType putObjectInline( } catch (Exception ex) { LOG.error(ex); db.rollback(); - throw new EucalyptusCloudException(bucketName); + throw new InternalErrorException(bucketName); } } else { db.rollback(); @@ -1541,13 +1543,13 @@ public PutObjectInlineResponseType putObjectInline( db.commit(); reply.setEtag(md5); - reply.setLastModified(DateUtils.format(lastModified.getTime(), DateUtils.RFC822_DATETIME_PATTERN)); + reply.setLastModified(lastModified); return reply; } @Override public AddObjectResponseType addObject(AddObjectType request) - throws EucalyptusCloudException { + throws WalrusException { AddObjectResponseType reply = (AddObjectResponseType) request.getReply(); String bucketName = request.getBucket(); @@ -1575,7 +1577,7 @@ public AddObjectResponseType addObject(AddObjectType request) if (objectInfo.getObjectKey().equals(key)) { // key (object) exists. db.rollback(); - throw new EucalyptusCloudException("object already exists " + key); + throw new InternalErrorException("object already exists " + key); } } // write object to bucket @@ -1602,7 +1604,7 @@ public AddObjectResponseType addObject(AddObjectType request) @Override public DeleteObjectResponseType deleteObject(DeleteObjectType request) - throws EucalyptusCloudException { + throws WalrusException { DeleteObjectResponseType reply = (DeleteObjectResponseType) request .getReply(); String bucketName = request.getBucket(); @@ -1622,7 +1624,7 @@ public DeleteObjectResponseType deleteObject(DeleteObjectType request) bucketInfo = null; } catch(TransactionException e) { LOG.error("Transaction error looking up bucket: " + bucketName,e); - throw new EucalyptusCloudException(e); + throw new NoSuchBucketException(e); } if(bucketInfo != null) { @@ -1683,7 +1685,7 @@ public DeleteObjectResponseType deleteObject(DeleteObjectType request) if (objectInfos.size() > 1) { // This shouldn't happen, so bail if it does db.rollback(); - throw new EucalyptusCloudException("More than one object set to 'last' found"); + throw new InternalErrorException("More than one object set to 'last' found"); } ObjectInfo lastObject = objectInfos.get(0); if (lastObject.getVersionId().equals(WalrusProperties.NULL_VERSION_ID)) { @@ -1712,7 +1714,7 @@ public DeleteObjectResponseType deleteObject(DeleteObjectType request) lastObject.setLast(false); } else { db.rollback(); - throw new EucalyptusCloudException( + throw new InternalErrorException( "Non 'null' versioned object found in a versioning disabled bucket, not sure how to proceed with delete."); } } @@ -1749,7 +1751,7 @@ public DeleteObjectResponseType deleteObject(DeleteObjectType request) return reply; } catch(EucalyptusCloudException e) { LOG.error("DeleteObject operation for " + bucketName + "/" + objectKey + " failed with: " + e.getMessage()); - throw e; + throw new InternalErrorException(e); } finally { if(db != null && db.isActive()) { db.rollback(); @@ -1844,7 +1846,7 @@ public void run() { } @Override - public ListBucketResponseType listBucket(ListBucketType request) throws EucalyptusCloudException { + public ListBucketResponseType listBucket(ListBucketType request) throws WalrusException { ListBucketResponseType reply = (ListBucketResponseType) request.getReply(); EntityWrapper db = EntityWrapper.get(BucketInfo.class); @@ -1885,7 +1887,7 @@ public ListBucketResponseType listBucket(ListBucketType request) throws Eucalypt } } catch (Exception e) { db.rollback(); - throw new EucalyptusCloudException(e); + throw new InternalErrorException(e); } } @@ -1997,7 +1999,7 @@ public ListBucketResponseType listBucket(ListBucketType request) throws Eucalypt ListEntry listEntry = new ListEntry(); listEntry.setKey(objectKey); listEntry.setEtag(objectInfo.getEtag()); - listEntry.setLastModified(DateUtils.format(objectInfo.getLastModified().getTime(), DateUtils.ALT_ISO8601_DATE_PATTERN)); + listEntry.setLastModified(DateFormatter.dateToListingFormattedString(objectInfo.getLastModified())); listEntry.setStorageClass(objectInfo.getStorageClass()); listEntry.setSize(objectInfo.getSize()); listEntry.setStorageClass(objectInfo.getStorageClass()); @@ -2086,7 +2088,7 @@ private void addGrants(ArrayList grants, List grantInfos) { @Override public GetObjectAccessControlPolicyResponseType getObjectAccessControlPolicy( GetObjectAccessControlPolicyType request) - throws EucalyptusCloudException { + throws WalrusException { GetObjectAccessControlPolicyResponseType reply = (GetObjectAccessControlPolicyResponseType) request .getReply(); @@ -2207,7 +2209,7 @@ private void fixCanonicalIds(AccessControlList accessControlList, boolean isBuck } } - public SetBucketAccessControlPolicyResponseType setBucketAccessControlPolicy(SetBucketAccessControlPolicyType request) throws EucalyptusCloudException { + public SetBucketAccessControlPolicyResponseType setBucketAccessControlPolicy(SetBucketAccessControlPolicyType request) throws WalrusException { SetBucketAccessControlPolicyResponseType reply = (SetBucketAccessControlPolicyResponseType) request.getReply(); Context ctx = Contexts.lookup(); Account account = ctx.getAccount(); @@ -2252,7 +2254,7 @@ public SetBucketAccessControlPolicyResponseType setBucketAccessControlPolicy(Set @Override public SetRESTBucketAccessControlPolicyResponseType setRESTBucketAccessControlPolicy( SetRESTBucketAccessControlPolicyType request) - throws EucalyptusCloudException { + throws WalrusException { SetRESTBucketAccessControlPolicyResponseType reply = (SetRESTBucketAccessControlPolicyResponseType) request.getReply(); Context ctx = Contexts.lookup(); Account account = ctx.getAccount(); @@ -2301,7 +2303,7 @@ public SetRESTBucketAccessControlPolicyResponseType setRESTBucketAccessControlPo @Override public SetObjectAccessControlPolicyResponseType setObjectAccessControlPolicy( SetObjectAccessControlPolicyType request) - throws EucalyptusCloudException { + throws WalrusException { SetObjectAccessControlPolicyResponseType reply = (SetObjectAccessControlPolicyResponseType) request .getReply(); Context ctx = Contexts.lookup(); @@ -2383,7 +2385,7 @@ public SetObjectAccessControlPolicyResponseType setObjectAccessControlPolicy( @Override public SetRESTObjectAccessControlPolicyResponseType setRESTObjectAccessControlPolicy( SetRESTObjectAccessControlPolicyType request) - throws EucalyptusCloudException { + throws WalrusException { SetRESTObjectAccessControlPolicyResponseType reply = (SetRESTObjectAccessControlPolicyResponseType) request.getReply(); Context ctx = Contexts.lookup(); Account account = ctx.getAccount(); @@ -2467,7 +2469,7 @@ public SetRESTObjectAccessControlPolicyResponseType setRESTObjectAccessControlPo } @Override - public GetObjectResponseType getObject(GetObjectType request) throws EucalyptusCloudException { + public GetObjectResponseType getObject(GetObjectType request) throws WalrusException { GetObjectResponseType reply = (GetObjectResponseType) request.getReply(); // Must explicitly set to true for streaming large objects. reply.setHasStreamingData(false); @@ -2529,7 +2531,7 @@ public GetObjectResponseType getObject(GetObjectType request) throws EucalyptusC if (objectInfo.isGlobalRead()) { if (!WalrusProperties.enableTorrents) { LOG.warn("Bittorrent support has been disabled. Please check pre-requisites"); - throw new EucalyptusCloudException("Torrents disabled"); + throw new InternalErrorException("Torrents disabled"); } EntityWrapper dbTorrent = EntityWrapper.get(TorrentInfo.class); TorrentInfo torrentInfo = new TorrentInfo(bucketName, objectKey); @@ -2546,7 +2548,7 @@ public GetObjectResponseType getObject(GetObjectType request) throws EucalyptusC torrentCreator.create(); } catch (Exception e) { LOG.error(e); - throw new EucalyptusCloudException("could not create torrent file " + torrentFile); + throw new InternalErrorException("could not create torrent file " + torrentFile); } torrentInfo.setTorrentFile(torrentFile); dbTorrent.add(torrentInfo); @@ -2582,7 +2584,7 @@ public GetObjectResponseType getObject(GetObjectType request) throws EucalyptusC db.rollback(); String errorString = "Could not get torrent file " + torrentFilePath; LOG.error(errorString); - throw new EucalyptusCloudException(errorString); + throw new InternalErrorException(errorString); } } else { // No global object read permission @@ -2634,30 +2636,29 @@ public GetObjectResponseType getObject(GetObjectType request) throws EucalyptusC fileIO.finish(); } catch (Exception e) { LOG.error(e, e); - throw new EucalyptusCloudException(e); + throw new InternalErrorException(e); } reply.setBase64Data(Hashes.base64encode(base64Data)); } else { reply.setHasStreamingData(true); // support for large objects - storageManager.sendObject(request, httpResponse, bucketName, objectName, size, etag, - DateUtils.format(lastModified.getTime(), DateUtils.RFC822_DATETIME_PATTERN), contentType, contentDisposition, - request.getIsCompressed(), versionId, logData); + storageManager.sendObject(request, httpResponse, bucketName, objectName, size, etag, + DateFormatter.dateToHeaderFormattedString(lastModified), contentType, contentDisposition, request.getIsCompressed(), + versionId, logData); return null; } } } else { // Request is for headers/metadata only - storageManager.sendHeaders(request, httpResponse, size, etag, - DateUtils.format(lastModified.getTime(), DateUtils.RFC822_DATETIME_PATTERN), contentType, contentDisposition, versionId, - logData); + storageManager.sendHeaders(request, httpResponse, size, etag, DateFormatter.dateToHeaderFormattedString(lastModified), contentType, + contentDisposition, versionId, logData); return null; } reply.setEtag(etag); - reply.setLastModified(DateUtils.format(lastModified, DateUtils.RFC822_DATETIME_PATTERN)); + reply.setLastModified(lastModified); reply.setSize(size); reply.setContentType(contentType); reply.setContentDisposition(contentDisposition); @@ -2692,7 +2693,7 @@ public GetObjectResponseType getObject(GetObjectType request) throws EucalyptusC } @Override - public GetObjectExtendedResponseType getObjectExtended(GetObjectExtendedType request) throws EucalyptusCloudException { + public GetObjectExtendedResponseType getObjectExtended(GetObjectExtendedType request) throws WalrusException { GetObjectExtendedResponseType reply = (GetObjectExtendedResponseType) request.getReply(); Date ifModifiedSince = request.getIfModifiedSince(); Date ifUnmodifiedSince = request.getIfUnmodifiedSince(); @@ -2798,14 +2799,13 @@ public GetObjectExtendedResponseType getObjectExtended(GetObjectExtendedType req versionId = objectInfo.getVersionId() != null ? objectInfo.getVersionId() : WalrusProperties.NULL_VERSION_ID; } if (request.getGetData()) { - storageManager.sendObject(request, httpResponse, bucketName, objectName, byteRangeStart, byteRangeEnd + 1, size, etag, - DateUtils.format(lastModified.getTime(), DateUtils.RFC822_DATETIME_PATTERN), contentType, contentDisposition, - request.getIsCompressed(), versionId, logData); + storageManager.sendObject(request, httpResponse, bucketName, objectName, byteRangeStart, byteRangeEnd + 1, size, etag, + DateFormatter.dateToHeaderFormattedString(lastModified), contentType, contentDisposition, request.getIsCompressed(), versionId, + logData); return null; } else { - storageManager.sendHeaders(request, httpResponse, size, etag, - DateUtils.format(lastModified.getTime(), DateUtils.RFC822_DATETIME_PATTERN), contentType, contentDisposition, versionId, - logData); + storageManager.sendHeaders(request, httpResponse, size, etag, DateFormatter.dateToHeaderFormattedString(lastModified), contentType, + contentDisposition, versionId, logData); return null; } } else { @@ -2831,7 +2831,7 @@ public GetObjectExtendedResponseType getObjectExtended(GetObjectExtendedType req } } - private String getMultipartData(DefaultHttpResponse httpResponse, ObjectInfo objectInfo, GetObjectType request, GetObjectResponseType response) throws EucalyptusCloudException { + private String getMultipartData(DefaultHttpResponse httpResponse, ObjectInfo objectInfo, GetObjectType request, GetObjectResponseType response) throws WalrusException { //get all parts PartInfo searchPart = new PartInfo(request.getBucket(), request.getKey()); searchPart.setCleanup(false); @@ -2847,7 +2847,7 @@ private String getMultipartData(DefaultHttpResponse httpResponse, ObjectInfo obj parts = partCriteria.list(); if(parts.size() == 0) { - throw new EucalyptusCloudException("No parts found corresponding to uploadId: " + objectInfo.getUploadId()); + throw new InternalErrorException("No parts found corresponding to uploadId: " + objectInfo.getUploadId()); } } finally { db.rollback(); @@ -2874,16 +2874,16 @@ private String getMultipartData(DefaultHttpResponse httpResponse, ObjectInfo obj fileIO.finish(); } catch (Exception e) { LOG.error(e, e); - throw new EucalyptusCloudException(e); + throw new InternalErrorException(e); } } return Hashes.base64encode(base64Data); } else { response.setHasStreamingData(true); // support for large objects - storageManager.sendObject(request, httpResponse, parts, objectInfo.getSize(), objectInfo.getEtag(), - DateUtils.format(objectInfo.getLastModified().getTime(), DateUtils.RFC822_DATETIME_PATTERN), objectInfo.getContentType(), objectInfo.getContentDisposition(), - request.getIsCompressed(), objectInfo.getVersionId()); + storageManager.sendObject(request, httpResponse, parts, objectInfo.getSize(), objectInfo.getEtag(), + DateFormatter.dateToHeaderFormattedString(objectInfo.getLastModified()), objectInfo.getContentType(), objectInfo.getContentDisposition(), + request.getIsCompressed(), objectInfo.getVersionId()); return null; } @@ -2891,7 +2891,7 @@ private String getMultipartData(DefaultHttpResponse httpResponse, ObjectInfo obj @Override public GetBucketLocationResponseType getBucketLocation( - GetBucketLocationType request) throws EucalyptusCloudException { + GetBucketLocationType request) throws WalrusException { GetBucketLocationResponseType reply = (GetBucketLocationResponseType) request .getReply(); String bucketName = request.getBucket(); @@ -2925,7 +2925,7 @@ public GetBucketLocationResponseType getBucketLocation( @Override public CopyObjectResponseType copyObject(CopyObjectType request) - throws EucalyptusCloudException { + throws WalrusException { CopyObjectResponseType reply = (CopyObjectResponseType) request .getReply(); Context ctx = Contexts.lookup(); @@ -3064,13 +3064,14 @@ public CopyObjectResponseType copyObject(CopyObjectType request) } catch (Exception ex) { LOG.error(ex); db.rollback(); - throw new EucalyptusCloudException("Could not rename " + sourceObjectName + " to " + destinationObjectName); + throw new InternalErrorException("Could not rename " + sourceObjectName + " to " + destinationObjectName); } if (addNew) dbObject.add(destinationObjectInfo); reply.setEtag(etag); - reply.setLastModified(DateUtils.format(lastModified.getTime(), DateUtils.ALT_ISO8601_DATE_PATTERN)); + // Last modified date in copy response is in ISO8601 format as per S3 spec + reply.setLastModified(DateFormatter.dateToListingFormattedString(lastModified)); if (foundDestinationBucketInfo.isVersioningEnabled()) { reply.setCopySourceVersionId(sourceVersionId); @@ -3099,7 +3100,7 @@ public CopyObjectResponseType copyObject(CopyObjectType request) @Override public SetBucketLoggingStatusResponseType setBucketLoggingStatus( - SetBucketLoggingStatusType request) throws EucalyptusCloudException { + SetBucketLoggingStatusType request) throws WalrusException { SetBucketLoggingStatusResponseType reply = (SetBucketLoggingStatusResponseType) request .getReply(); String bucket = request.getBucket(); @@ -3185,7 +3186,7 @@ private String findInvalidGrant(List grants) { @Override public GetBucketLoggingStatusResponseType getBucketLoggingStatus( - GetBucketLoggingStatusType request) throws EucalyptusCloudException { + GetBucketLoggingStatusType request) throws WalrusException { GetBucketLoggingStatusResponseType reply = (GetBucketLoggingStatusResponseType) request .getReply(); String bucket = request.getBucket(); @@ -3232,7 +3233,7 @@ private void updateLogData(BucketInfo bucket, BucketLogData logData) { @Override public GetBucketVersioningStatusResponseType getBucketVersioningStatus(GetBucketVersioningStatusType request) - throws EucalyptusCloudException { + throws WalrusException { GetBucketVersioningStatusResponseType reply = (GetBucketVersioningStatusResponseType) request.getReply(); String bucket = request.getBucket(); Context ctx = Contexts.lookup(); @@ -3260,7 +3261,7 @@ public GetBucketVersioningStatusResponseType getBucketVersioningStatus(GetBucket @Override public SetBucketVersioningStatusResponseType setBucketVersioningStatus( SetBucketVersioningStatusType request) - throws EucalyptusCloudException { + throws WalrusException { SetBucketVersioningStatusResponseType reply = (SetBucketVersioningStatusResponseType) request .getReply(); String bucket = request.getBucket(); @@ -3307,7 +3308,7 @@ else if (WalrusProperties.VersioningStatus.Suspended.toString().equals(status) * Significantly re-done version of listVersions that is based on listBuckets and the old listVersions. */ @Override - public ListVersionsResponseType listVersions(ListVersionsType request) throws EucalyptusCloudException { + public ListVersionsResponseType listVersions(ListVersionsType request) throws WalrusException { ListVersionsResponseType reply = (ListVersionsResponseType) request.getReply(); EntityWrapper db = EntityWrapper.get(BucketInfo.class); @@ -3402,7 +3403,7 @@ public ListVersionsResponseType listVersions(ListVersionsType request) throws Eu } catch (TransactionException e) { LOG.error(e); dbObject.rollback(); - throw new EucalyptusCloudException("Next-Key-Marker or Next-Version-Id marker invalid"); + throw new InternalErrorException("Next-Key-Marker or Next-Version-Id marker invalid"); } // The result set should be exclusive of the key with the key-marker version-id-marker pair. Look for keys that lexicographically // follow the version-id-marker for a given key-marker and also the keys that follow the key-marker. @@ -3494,7 +3495,7 @@ public ListVersionsResponseType listVersions(ListVersionsType request) throws Eu keyEntry.setKey(objectKey); keyEntry.setVersionId(objectInfo.getVersionId()); keyEntry.setIsLatest(objectInfo.getLast()); - keyEntry.setLastModified(DateUtils.format(objectInfo.getLastModified().getTime(), DateUtils.ALT_ISO8601_DATE_PATTERN)); + keyEntry.setLastModified(DateFormatter.dateToListingFormattedString(objectInfo.getLastModified())); try { Account ownerAccount = Accounts.lookupAccountById(objectInfo.getOwnerId()); keyEntry.setOwner(new CanonicalUser(ownerAccount.getCanonicalId(), ownerAccount.getName())); @@ -3542,7 +3543,7 @@ public ListVersionsResponseType listVersions(ListVersionsType request) throws Eu } @Override - public DeleteVersionResponseType deleteVersion(DeleteVersionType request) throws EucalyptusCloudException { + public DeleteVersionResponseType deleteVersion(DeleteVersionType request) throws WalrusException { DeleteVersionResponseType reply = (DeleteVersionResponseType) request.getReply(); String bucketName = request.getBucket(); String objectKey = request.getKey(); @@ -3561,7 +3562,7 @@ public DeleteVersionResponseType deleteVersion(DeleteVersionType request) throws ObjectInfo searchObjectInfo = new ObjectInfo(bucketName, objectKey); if (request.getVersionid() == null) { db.rollback(); - throw new EucalyptusCloudException("versionId is null"); + throw new InternalErrorException("versionId is null"); } searchObjectInfo.setVersionId(request.getVersionid()); List objectInfos = dbObject.queryEscape(searchObjectInfo); @@ -3610,21 +3611,21 @@ public DeleteVersionResponseType deleteVersion(DeleteVersionType request) throws return reply; } - public static InetAddress getBucketIp(String bucket) throws EucalyptusCloudException { + public static InetAddress getBucketIp(String bucket) throws WalrusException { EntityWrapper db = EntityWrapper.get(BucketInfo.class); try { BucketInfo searchBucket = new BucketInfo(bucket); db.getUniqueEscape(searchBucket); return WalrusProperties.getWalrusAddress(); } catch (EucalyptusCloudException ex) { - throw ex; + throw new InternalErrorException(ex); } finally { db.rollback(); } } @Override - public void fastDeleteObject(DeleteObjectType request) throws EucalyptusCloudException { + public void fastDeleteObject(DeleteObjectType request) throws WalrusException { String bucketName = request.getBucket(); String objectKey = request.getKey(); EntityWrapper db = EntityWrapper.get(BucketInfo.class); @@ -3662,7 +3663,7 @@ public void fastDeleteObject(DeleteObjectType request) throws EucalyptusCloudExc } @Override - public void fastDeleteBucket(DeleteBucketType request) throws EucalyptusCloudException { + public void fastDeleteBucket(DeleteBucketType request) throws WalrusException { String bucketName = request.getBucket(); EntityWrapper db = EntityWrapper.get(BucketInfo.class); BucketInfo searchBucket = new BucketInfo(bucketName); @@ -3695,7 +3696,7 @@ public void fastDeleteBucket(DeleteBucketType request) throws EucalyptusCloudExc db.commit(); } - public InitiateMultipartUploadResponseType initiateMultipartUpload(InitiateMultipartUploadType request) throws EucalyptusCloudException { + public InitiateMultipartUploadResponseType initiateMultipartUpload(InitiateMultipartUploadType request) throws WalrusException { InitiateMultipartUploadResponseType reply = (InitiateMultipartUploadResponseType) request.getReply(); Context ctx = Contexts.lookup(); @@ -3745,7 +3746,7 @@ public InitiateMultipartUploadResponseType initiateMultipartUpload(InitiateMulti return reply; } - public UploadPartResponseType uploadPart(UploadPartType request) throws EucalyptusCloudException { + public UploadPartResponseType uploadPart(UploadPartType request) throws WalrusException { UploadPartResponseType reply = (UploadPartResponseType) request.getReply(); @@ -3776,7 +3777,7 @@ public UploadPartResponseType uploadPart(UploadPartType request) throws Eucalypt } catch (NumberFormatException e) { LOG.error("Invalid content length " + request.getContentLength()); //TODO: FIXME This should be a MissingContentLengthException - throw new EucalyptusCloudException("Missing Content-Length"); + throw new InternalErrorException("Missing Content-Length"); } String objectName = null; @@ -3795,11 +3796,11 @@ public UploadPartResponseType uploadPart(UploadPartType request) throws Eucalypt List found = partCriteria.list(); if (found.size() == 0) { db.rollback(); - throw new EucalyptusCloudException("Multipart upload ID is invalid. Intitiate a multipart upload request before uploading the parts"); + throw new InternalErrorException("Multipart upload ID is invalid. Intitiate a multipart upload request before uploading the parts"); } if (found.size() > 1) { db.rollback(); - throw new EucalyptusCloudException("Multiple manifests found for same uploadId"); + throw new InternalErrorException("Multiple manifests found for same uploadId"); } PartInfo foundManifest = found.get(0); @@ -3812,7 +3813,7 @@ public UploadPartResponseType uploadPart(UploadPartType request) throws Eucalypt dbPart.commit(); } catch (Exception ex) { - throw new EucalyptusCloudException(ex); + throw new InternalErrorException(ex); } // writes are unconditional @@ -3861,7 +3862,7 @@ public UploadPartResponseType uploadPart(UploadPartType request) throws Eucalypt fileIO = storageManager.prepareForWrite(bucketName, tempObjectName); } catch (Exception ex) { messenger.removeQueue(key, randomKey); - throw new EucalyptusCloudException(ex); + throw new InternalErrorException(ex); } } else if (WalrusDataMessage.isEOF(dataMessage)) { if (digest != null) { @@ -3899,7 +3900,7 @@ public UploadPartResponseType uploadPart(UploadPartType request) throws Eucalypt } catch (IOException ex) { LOG.error(ex); messenger.removeQueue(key, randomKey); - throw new EucalyptusCloudException(objectKey); + throw new InternalErrorException(objectKey); } lastModified = new Date(); PartInfo searchPart = new PartInfo(bucketName, objectKey); @@ -3913,7 +3914,7 @@ public UploadPartResponseType uploadPart(UploadPartType request) throws Eucalypt foundPart = dbPart.getUniqueEscape(searchPart); } catch (EucalyptusCloudException ex) { dbPart.rollback(); - throw new EucalyptusCloudException("Unable to update part: " + bucketName + "/" + objectKey + " uploadId: " + uploadId + " partNumber: " + partNumber); + throw new InternalErrorException("Unable to update part: " + bucketName + "/" + objectKey + " uploadId: " + uploadId + " partNumber: " + partNumber); } foundPart.setEtag(md5); foundPart.setSize(size); @@ -3962,7 +3963,7 @@ public UploadPartResponseType uploadPart(UploadPartType request) throws Eucalypt LOG.error(ex, ex); db.rollback(); messenger.removeQueue(key, randomKey); - throw new EucalyptusCloudException("Transfer interrupted: " + key + "." + randomKey); + throw new InternalErrorException("Transfer interrupted: " + key + "." + randomKey); } } else { @@ -3972,11 +3973,11 @@ public UploadPartResponseType uploadPart(UploadPartType request) throws Eucalypt } reply.setEtag(md5); - reply.setLastModified(DateUtils.format(lastModified.getTime(), DateUtils.RFC822_DATETIME_PATTERN)); + reply.setLastModified(lastModified); return reply; } - public CompleteMultipartUploadResponseType completeMultipartUpload(CompleteMultipartUploadType request) throws EucalyptusCloudException { + public CompleteMultipartUploadResponseType completeMultipartUpload(CompleteMultipartUploadType request) throws WalrusException { CompleteMultipartUploadResponseType reply = (CompleteMultipartUploadResponseType) request.getReply(); Context ctx = Contexts.lookup(); @@ -4006,11 +4007,11 @@ public CompleteMultipartUploadResponseType completeMultipartUpload(CompleteMulti List found = partCriteria.list(); if (found.size() == 0) { db.rollback(); - throw new EucalyptusCloudException("Multipart upload ID is invalid. Intitiate a multipart upload request before uploading the parts"); + throw new NoSuchUploadException(request.getUploadId()); } if (found.size() > 1) { db.rollback(); - throw new EucalyptusCloudException("Multiple manifests found for same uploadId"); + throw new InternalErrorException("Multiple manifests found for same uploadId: " + request.getUploadId()); } PartInfo foundManifest = found.get(0); @@ -4050,6 +4051,8 @@ public CompleteMultipartUploadResponseType completeMultipartUpload(CompleteMulti throw ex; } catch (EucalyptusCloudException ex) { // No existing object found + //empty catch? We need to throw or catch a more specific exception here. EucalyptusCloudException is not + //specific enough and could be raised in a variety of cases. } } @@ -4067,7 +4070,7 @@ public CompleteMultipartUploadResponseType completeMultipartUpload(CompleteMulti long size = 0; if(foundParts != null && foundParts.size() > 0) { if(requestParts != null && requestParts.size() > foundParts.size()) { - throw new EucalyptusCloudException("One or more parts has not been uploaded yet. Either upload the part or fix the manifest"); + throw new InternalErrorException("One or more parts has not been uploaded yet. Either upload the part or fix the manifest. Upload Id: " + request.getUploadId()); } else { // Create a hashmap Map partsMap = new HashMap(foundParts.size()); @@ -4083,8 +4086,7 @@ public CompleteMultipartUploadResponseType completeMultipartUpload(CompleteMulti size += lookupPart.getSize(); partsMap.remove(lookupPart.getPartNumber()); } else { - //TODO: This needs to be an InvalidPart exception - throw new EucalyptusCloudException("Part not found"); + throw new InvalidPartException("Part Number: " + requestPart.getPartNumber() + " upload id: " + request.getUploadId()); } } MessageDigest digest = Digest.MD5.get(); @@ -4103,6 +4105,7 @@ public CompleteMultipartUploadResponseType completeMultipartUpload(CompleteMulti } ObjectInfo objectInfo = new ObjectInfo(bucketName, objectKey); + objectInfo.setOwnerId(account.getAccountNumber()); objectInfo.setCleanup(false); objectInfo.setEtag(eTag); objectInfo.setUploadId(foundManifest.getUploadId()); @@ -4116,6 +4119,7 @@ public CompleteMultipartUploadResponseType completeMultipartUpload(CompleteMulti objectInfo.setLast(true); objectInfo.setDeleted(false); objectInfo.updateGrants(foundManifest); + objectInfo.setLastModified(new Date()); EntityWrapper dbOject = db.recast(ObjectInfo.class); dbOject.add(objectInfo); @@ -4130,19 +4134,20 @@ public CompleteMultipartUploadResponseType completeMultipartUpload(CompleteMulti reply.setLocation("WalrusBackend" + foundManifest.getBucketName() + "/" + foundManifest.getObjectKey()); reply.setBucket(foundManifest.getBucketName()); reply.setKey(foundManifest.getObjectKey()); + reply.setLastModified(objectInfo.getLastModified()); //fire cleanup firePartsCleanupTask(foundManifest.getBucketName(), request.getKey(), request.getUploadId()); } } else { - throw new EucalyptusCloudException("No parts found in the database"); + throw new InvalidPartException("No parts found for: " + request.getUploadId()); } } else { - throw new EucalyptusCloudException("Multipart upload ID is invalid."); + throw new NoSuchUploadException(request.getUploadId()); } } catch (Exception ex) { db.rollback(); - throw new EucalyptusCloudException(ex); + throw new InternalErrorException(ex); } } else { db.rollback(); @@ -4154,7 +4159,7 @@ public CompleteMultipartUploadResponseType completeMultipartUpload(CompleteMulti return reply; } - public AbortMultipartUploadResponseType abortMultipartUpload(AbortMultipartUploadType request) throws EucalyptusCloudException { + public AbortMultipartUploadResponseType abortMultipartUpload(AbortMultipartUploadType request) throws WalrusException { AbortMultipartUploadResponseType reply = (AbortMultipartUploadResponseType) request.getReply(); Context ctx = Contexts.lookup(); Account account = ctx.getAccount(); @@ -4183,11 +4188,11 @@ public AbortMultipartUploadResponseType abortMultipartUpload(AbortMultipartUploa List found = partCriteria.list(); if (found.size() == 0) { db.rollback(); - throw new EucalyptusCloudException("Multipart upload ID is invalid. Intitiate a multipart upload request before uploading the parts"); + throw new InternalErrorException("Multipart upload ID is invalid. Intitiate a multipart upload request before uploading the parts"); } if (found.size() > 1) { db.rollback(); - throw new EucalyptusCloudException("Multiple manifests found for same uploadId"); + throw new InternalErrorException("Multiple manifests found for same uploadId"); } PartInfo foundManifest = found.get(0); @@ -4204,18 +4209,18 @@ public AbortMultipartUploadResponseType abortMultipartUpload(AbortMultipartUploa part.markForCleanup(); } } else { - throw new EucalyptusCloudException("No parts found for upload: " + request.getUploadId()); + throw new InternalErrorException("No parts found for upload: " + request.getUploadId()); } db.commit(); //fire cleanup firePartsCleanupTask(foundManifest.getBucketName(), foundManifest.getObjectKey(), request.getUploadId()); } else { - throw new EucalyptusCloudException("Multipart upload ID is invalid."); + throw new InternalErrorException("Multipart upload ID is invalid."); } } catch (Exception ex) { db.rollback(); - throw new EucalyptusCloudException(ex); + throw new InternalErrorException(ex); } } else { db.rollback(); diff --git a/clc/modules/walrus/src/main/java/com/eucalyptus/walrus/pipeline/WalrusOutboundHandler.java b/clc/modules/walrus/src/main/java/com/eucalyptus/walrus/pipeline/WalrusOutboundHandler.java index e84e5aab303..3add73ba3fc 100644 --- a/clc/modules/walrus/src/main/java/com/eucalyptus/walrus/pipeline/WalrusOutboundHandler.java +++ b/clc/modules/walrus/src/main/java/com/eucalyptus/walrus/pipeline/WalrusOutboundHandler.java @@ -62,6 +62,7 @@ package com.eucalyptus.walrus.pipeline; +import com.eucalyptus.storage.common.DateFormatter; import com.eucalyptus.walrus.msgs.WalrusDataGetResponseType; import com.eucalyptus.walrus.msgs.WalrusDataResponseType; import org.apache.log4j.Logger; @@ -110,14 +111,14 @@ public void outgoingMessage( ChannelHandlerContext ctx, MessageEvent event ) thr if(msg instanceof PutObjectResponseType) { PutObjectResponseType putObjectResponse = (PutObjectResponseType) msg; httpResponse.addHeader(HttpHeaders.Names.ETAG, '\"' + putObjectResponse.getEtag() + '\"'); - httpResponse.addHeader(HttpHeaders.Names.LAST_MODIFIED, putObjectResponse.getLastModified()); + httpResponse.addHeader(HttpHeaders.Names.LAST_MODIFIED, DateFormatter.dateToHeaderFormattedString(putObjectResponse.getLastModified())); if(putObjectResponse.getVersionId() != null) { httpResponse.addHeader(WalrusProperties.X_AMZ_VERSION_ID, putObjectResponse.getVersionId()); } } else if(msg instanceof WalrusDataResponseType) { WalrusDataResponseType response = (WalrusDataResponseType) msg; httpResponse.addHeader(HttpHeaders.Names.ETAG, '\"' + response.getEtag() + '\"'); - httpResponse.addHeader(HttpHeaders.Names.LAST_MODIFIED, response.getLastModified()); + httpResponse.addHeader(HttpHeaders.Names.LAST_MODIFIED, DateFormatter.dateToHeaderFormattedString(response.getLastModified())); if(response.getVersionId() != null) { httpResponse.addHeader(WalrusProperties.X_AMZ_VERSION_ID, response.getVersionId()); } diff --git a/tools/imaging/eucatoolkit/stages/__init__.py b/tools/imaging/eucatoolkit/stages/__init__.py index b2912b6b13e..8eff6e7abcc 100644 --- a/tools/imaging/eucatoolkit/stages/__init__.py +++ b/tools/imaging/eucatoolkit/stages/__init__.py @@ -29,6 +29,7 @@ import sys _chunk_size = 8192 +_max_part_buffer_size = 11534336 _logger_name = 'DownloadImage' diff --git a/tools/imaging/eucatoolkit/stages/cryptoutils.py b/tools/imaging/eucatoolkit/stages/cryptoutils.py index c3d2f7e58c0..235739b7c88 100644 --- a/tools/imaging/eucatoolkit/stages/cryptoutils.py +++ b/tools/imaging/eucatoolkit/stages/cryptoutils.py @@ -156,3 +156,20 @@ def _decrypt_hex_key(hex_encrypted_key, key_filename): + '", and keyfile:"' + str(key_filename) + '".' + VE.message] raise VE + +def _calc_digest_for_fileobj(file_obj, algorithm, chunk_size=None): + ''' + Calculated and return the digest for the fileobj provided using the + hashlib 'alogrithm' provided. + :param file_obj: file like obj to read compute digest for + :param algorithm: string representing hashlib type(sha1, md5, etc) + :param chunksize: # of bytes to read/write per read()/write() + ''' + chunk_size = chunk_size or 8192 + digest = _get_digest_algorithm_from_string(algorithm) + while True: + chunk = file_obj.read(chunk_size) + if not chunk: + break + digest.update(chunk) + return digest.hexdigest() diff --git a/tools/imaging/eucatoolkit/stages/downloadimage.py b/tools/imaging/eucatoolkit/stages/downloadimage.py index d7ad5d372cb..56033d5c723 100644 --- a/tools/imaging/eucatoolkit/stages/downloadimage.py +++ b/tools/imaging/eucatoolkit/stages/downloadimage.py @@ -46,13 +46,14 @@ def __init__(self, dest_file=None, **kwargs): parser.add_argument('-m', '--manifest', dest='manifest', required=True, help='''Path to 'download-manifest. Use '-' to read manifest from stdin''') - parser.add_argument('-d', '--dest', dest='destination', required=False, + parser.add_argument('-d', '--dest', dest='destination', help='''Destination path to write image to. Use '-' for stdout.''') parser.add_argument('-k', '--privatekey', dest='privatekey', help='''file containing the private key to decrypt the bundle with.''') parser.add_argument('-c', '--cloudcert', dest='cloudcert', + required=True, help='''file containing the cloud cert used to verify manifest signature.''') parser.add_argument('-x', '--xsd', dest='xsd', default=None, @@ -63,9 +64,6 @@ def __init__(self, dest_file=None, **kwargs): parser.add_argument('--maxbytes', dest='maxbytes', default=0, help='''Maximum bytes allowed to be written to the destination.''') - parser.add_argument('--euca_prefix', dest='euca_prefix', default=None, - help='''Path to eucalyptus install. - ie: /opt/eucalyptus/''') parser.add_argument('--debug', dest='debug', default=False, action='store_true', help='''Enable debug to a log file''') @@ -97,8 +95,8 @@ def __init__(self, dest_file=None, **kwargs): arg_value.append(kwargs[kwarg]) arg_list.extend(arg_value) self.args = parser.parse_args(arg_list) - if dest_file != None: - self.args.destination = dest_file + if dest_file is not None: + self.args.destination = dest_file if self.args.destination == "-": force_stderr = True else: @@ -126,24 +124,6 @@ def _setup(self): if not isinstance(xsd_file, file): xsd_file = os.path.expanduser(os.path.abspath(xsd_file)) self.args.xsd = xsd_file - if self.args.euca_prefix is None: - self.args.euca_prefix = "" - for path in ['/opt/eucalyptus/', '/']: - if os.path.exists(path): - self.args.euca_prefix = path - break - #if private key was not provided try to find it - if not self.args.privatekey: - keypath = os.path.join(self.args.euca_prefix, - 'var/lib/eucalyptus/keys/node-pk.pem') - if os.path.isfile(keypath): - self.args.privatekey = keypath - #if mandatory cloudcert was not provided try to find it - if not self.args.cloudcert: - certpath = os.path.join(self.args.euca_prefix, - 'var/lib/eucalyptus/keys/cloud-cert.pem') - if os.path.isfile(certpath): - self.args.cloudcert = certpath if not self.args.cloudcert: raise argparse.ArgumentError(self.args.cloudcert, "Cloud cert must be provided to " @@ -231,8 +211,8 @@ def _get_download_manifest_obj(self, manifest_input=None): if not parsed_url or not parsed_url.scheme: self.args.manifest = self._read_manifest_from_file() else: - self.log.debug('Reading from remote manifest from url') - #For now limit urls to http(s)... + # Reading from remote manifest from url + # For now limit urls to http(s)... if not parsed_url.scheme in ['http', 'https']: raise ArgumentTypeError('Manifest url only ' 'supports http, https at ' @@ -255,7 +235,7 @@ def _download_parts_to_fileobj(self, manifest, dest_fileobj): bytes = 0 for part_index in xrange(0, manifest.part_count): part = manifest.get_part_by_index(part_index) - self.log.debug('Downloading part:' + str(part.get_url)) + self.log.debug('Downloading part#:' + str(part.part_index)) bytes += part.download(dest_fileobj=dest_fileobj) or 0 self.log.debug('Wrote bytes:' + str(bytes) + "/" + str(manifest.download_image_size) + ", digest:" @@ -282,7 +262,7 @@ def _download_to_unbundlestream(self, dest_fileobj, manifest=None, tools_path=None, - inactivity_timeout=30): + inactivity_timeout=120): ''' Attempts to iterate through all parts contained in 'manifest' and download and concatenate each part to euca2ools unbundle stream. @@ -385,7 +365,7 @@ def _validate_written_image_size(self, expected_size, filepath): ''' self.log.debug('Validating size:"{0}", for file:{1}:' .format(expected_size, filepath)) - #Check size raise os.error if file is not accessible... + # Check size raise os.error if file is not accessible... file_size = os.path.getsize(filepath) if file_size != expected_size: raise ValueError('Written Image size:{0} does not equal expected ' @@ -397,7 +377,6 @@ def main(self): if self.args.dumpmanifest: print str(manifest) os.sys.exit(0) - self.log.debug('\nMANIFEST:' + str(manifest)) dest_file = self.args.destination dest_file_name = self.args.destination bytes = 0 @@ -419,8 +398,8 @@ def main(self): if not self.args.privatekey: raise ArgumentError(self.args.privatekey, 'Bundle type needs privatekey -k') - bytes = self._download_to_unbundlestream(dest_fileobj=dest_fileobj, - manifest=manifest) + bytes = self._download_to_unbundlestream( + dest_fileobj=dest_fileobj, manifest=manifest) else: with dest_fileobj: bytes = self._download_parts_to_fileobj( diff --git a/tools/imaging/eucatoolkit/stages/downloadmanifest.py b/tools/imaging/eucatoolkit/stages/downloadmanifest.py index 67d81631542..4c9b105da91 100644 --- a/tools/imaging/eucatoolkit/stages/downloadmanifest.py +++ b/tools/imaging/eucatoolkit/stages/downloadmanifest.py @@ -136,24 +136,6 @@ def read_from_url(cls, key_filename=key_filename, sig_key_filename=sig_key_filename) - @classmethod - def _calc_digest_for_fileobj(cls, file_obj, algorithm, chunk_size=None): - ''' - Calculated and return the digest for the fileobj provided using the - hashlib 'alogrithm' provided. - :param file_obj: file like obj to read compute digest for - :param algorithm: string representing hashlib type(sha1, md5, etc) - :param chunksize: # of bytes to read/write per read()/write() - ''' - chunk_size = chunk_size or stages._chunk_size - digest = cls._get_digest_algorithm_from_string(algorithm) - while True: - chunk = file_obj.read(chunk_size) - if not chunk: - break - digest.update(chunk) - return digest.hexdigest() - @classmethod def _read_from_fileobj(cls, manifest_fileobj, @@ -199,7 +181,7 @@ def _read_from_fileobj(cls, signature=manifest.signature, data=data_to_sign, algorithm=manifest.signature_algorithm) - #IF this manifest contains a bundle get and decrypt keys,iv... + #If this manifest contains a bundle get and decrypt keys,iv... if manifest.file_format == 'BUNDLE': bundle = xml.bundle manifest.unbundled_image_size = long(bundle.find('unbundled-size')) diff --git a/tools/imaging/eucatoolkit/stages/downloadpart.py b/tools/imaging/eucatoolkit/stages/downloadpart.py index 14b329db9bb..610017d877b 100644 --- a/tools/imaging/eucatoolkit/stages/downloadpart.py +++ b/tools/imaging/eucatoolkit/stages/downloadpart.py @@ -25,8 +25,9 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import requests +import time from requests.exceptions import HTTPError, Timeout, ConnectionError -import hashlib +from cryptoutils import _get_digest_algorithm_from_string from eucatoolkit import stages @@ -38,13 +39,16 @@ def __init__(self, chunk_size=None, digest_algorithm='MD5', digest=None, + max_buffer_size=None, logger=None): self.get_url = str(get_url) + self.debug_url = str(self.get_url).split('?')[0] # rem sensitive info self.part_index = part_index self.chunk_size = chunk_size or stages._chunk_size self.digest_algorithm = digest_algorithm self.digest = digest self.log = logger or stages.get_logger() + self.max_buffer_size = max_buffer_size or stages._max_part_buffer_size self.written_digest = None self.bytes_written = 0 @@ -52,7 +56,8 @@ def download(self, dest_fileobj, chunk_size=None, connection_timeout=30, - max_attempts=2): + max_attempts=2, + max_buffer_size=None): ''' Attempts to download/get() self.get_url and write to 'dest_fileobj' Attempts to validate the downloaded content against self.digest using @@ -64,88 +69,165 @@ def download(self, :param chunk_size: the size to read/write per iteration :param connection_timeout: int timeout for establishing connection :param max_attempts: attempts to connect upon failure + :param max_buffer_size: size in bytes returns bytes downloaded/written ''' - bytes = 0 attempt = 0 response = None - self.log.debug('Downloading part:{0}'.format(self.get_url)) + max_buffer_size = max_buffer_size or self.max_buffer_size + self.log.debug('Downloading part:{0}'.format(self.debug_url)) chunk_size = chunk_size or self.chunk_size - if self.digest_algorithm: - if self.digest_algorithm.upper() == 'SHA1': - digest = hashlib.sha1() - elif self.digest_algorithm.upper() == 'MD5': - digest = hashlib.md5() + digest = _get_digest_algorithm_from_string(self.digest_algorithm) #Attempt get request to get_url... - while not response and attempt < max_attempts: + while attempt < max_attempts: attempt += 1 + part_buffer = "" + content_length = None + bytes = 0 + self.written_digest = None + response = self._get_respsonse( + url=self.get_url, + max_attempts=max_attempts, + connection_timeout=connection_timeout) try: - response = requests.get(self.get_url, - stream=True, - timeout=connection_timeout) - except (Timeout, ConnectionError) as CE: - #If there was a connection error retry up max_attempts - if attempt >= max_attempts: - err_msg = str("{0}, Connection failure. Attempts:{1}" - .format(str(self), str(attempt))) - self.log(err_msg) - CE.args = [err_msg + "\n" + str(CE.message)] - raise CE - try: - response.raise_for_status() - except HTTPError as HE: - HE.args = [str(self) + "\n" + str(HE.message)] - raise HE - #Begin writing to destination fileobj - for chunk in response.iter_content(chunk_size): - dest_fileobj.write(chunk) - digest.update(chunk) - bytes += len(chunk) - self.bytes_written = bytes - dest_fileobj.flush() - #Download is complete record checksum - self.written_digest = str(digest.hexdigest()) - #Check the written/downloaded digest vs the original part digest... + content_length = response.headers.get('content-length', None) + if content_length is not None: + content_length = long(content_length) + # If the size of the part is less than the max buffer size, + # store the contents in a temp buffer to allow the download to be + # be retried in the case of a failure before potentially + # writing bad data to an unbundle pipeline + if content_length <= max_buffer_size: + for chunk in response.iter_content(chunk_size): + part_buffer += chunk + digest.update(chunk) + bytes += len(chunk) + else: + # The remote part is bigger than the allowed buffer size + # Begin writing directly to destination fileobj + for chunk in response.iter_content(chunk_size): + dest_fileobj.write(chunk) + digest.update(chunk) + bytes += len(chunk) + dest_fileobj.flush() + if bytes != content_length: + raise ValueError( + 'Part:"{0}". Expected Size:"{1}", Downloaded:"{2}"' + .format(self.part_index, content_length, bytes)) + self.bytes_written = bytes + # Download is complete validate digest + self.written_digest = self._validate_written_digest( + response=response, + digest=digest) + # If checksum was valid, write part to destination + if part_buffer: + dest_fileobj.write(part_buffer) + dest_fileobj.flush() + except ValueError as VE: + if attempt <= max_attempts: + raise + else: + self.log.warn( + 'Attempt:{0}/{1}, {2}' + .format(attempt, max_attempts, str(VE))) + # Back off and retry + time.sleep(10) + else: + return bytes + + def _validate_written_digest(self, response, digest): + # Record the hex digest string + written_digest = str(digest.hexdigest()) + # Check the written/downloaded digest vs the original part digest... if self.digest: - #The digest was provided at __init__ ... - if self.digest != self.written_digest: + # The digest was provided at __init__ ... + if self.digest != written_digest: raise ValueError( 'Part:"{0}". Expected Digest:"{1}" Written Digest:{2}' - .format(self.part_index, self.digest, self.written_digest)) - #If the digest algorithm is md5 try to use the header content... - elif (digest.name == 'md5' and hasattr(response, 'headers')): - #Attempt to get the digest from the http headers... - if 'Content-MD5' in response.headers: - cont_md5 = str(response.headers.get('content_md5')).strip('"') - cont_md5 = cont_md5.strip() - if (self.written_digest and - (str(self.written_digest) != str(cont_md5))): + .format(self.part_index, self.digest, written_digest)) + # If the digest algorithm is md5 try to use the header content... + elif digest.name == 'md5' and hasattr(response, 'headers'): + # Attempt to get the digest from the http headers... + content_md5 = response.headers.get('content_md5', None) + if content_md5: + content_md5 = str(content_md5).strip('"').strip() + if (written_digest and + (str(written_digest) != str(content_md5))): raise ValueError( 'Part:"{0}". Expected Digest:"{1}" != ' 'Written Digest:{2}' .format(self.part_index, - cont_md5, - self.written_digest)) + content_md5, + written_digest)) else: + # Digest was verified log for debugging purposes self.log.debug( 'Part:"{0}". Expected Digest:"{1}" Written Digest:{2}' .format(self.part_index, - cont_md5, - self.written_digest)) + content_md5, + written_digest)) elif 'etag' in response.headers: - #Try to compare with etag - #Since etag is not a reliable md5 checksum, just warn here + # Try to compare with etag + # Since etag is not a reliable md5 checksum, just warn here etag = str(response.headers.get('etag')).strip('"').strip() etag_msg = ('Part:"{0}". etag:"{1}" vs Written Digest:"{2}"' .format(self.part_index, etag, - self.written_digest)) - if self.written_digest != etag: - self.log.warn(etag_msg + ", etag != digest") + written_digest)) + if written_digest != etag: + err_msg = etag_msg + ", etag does not equal written digest" + # If the etag was 32 hex chars assume this is the md5 sum + # and raise the error, if not just log a warning + try: + int(etag, 32) + except ValueError: + self.log.warn(err_msg) + else: + raise ValueError(err_msg) else: + # Digest was verified log for debugging purposes self.log.debug(etag_msg + ", etag == digest") - return bytes + return written_digest + + def _get_respsonse(self, url, max_attempts, connection_timeout): + attempt = 0 + response = None + # Attempt get request to get_url... + while not response and attempt < max_attempts: + attempt += 1 + try: + response = requests.get(self.get_url, + stream=True, + timeout=connection_timeout) + try: + response.raise_for_status() + except HTTPError as HE: + http_err = str("{0}, get() err, attempt:{1}/{2}, err:\n{3}" + .format(str(self), + str(attempt), + str(max_attempts), + str(HE.message))) + self.log.warn(http_err) + if attempt >= max_attempts: + HE.args = [http_err] + raise HE + response = None + continue + except (Timeout, ConnectionError) as CE: + # If there was a connection error retry up max_attempts + if attempt >= max_attempts: + err_msg = str("{0}, Connection failure. Attempts:{1}/{2}" + .format(str(self), + str(attempt), + str(max_attempts))) + self.log.warn(err_msg) + CE.args = [err_msg + "\n" + str(CE.message)] + raise CE + # Back off and retry + if not response: + time.sleep(5) + return response def __repr__(self): return 'DownloadPart({0}, {1} )'.format( - repr(self.part_index), repr(self.get_url)) + repr(self.part_index), repr(self.debug_url)) diff --git a/tools/imaging/eucatoolkit/stages/processutils.py b/tools/imaging/eucatoolkit/stages/processutils.py index 3d07aaac8af..3613d4e209d 100644 --- a/tools/imaging/eucatoolkit/stages/processutils.py +++ b/tools/imaging/eucatoolkit/stages/processutils.py @@ -69,7 +69,7 @@ def monitor_subprocess_io(infile, sub_stderr=None, chunk_size=None, log_method=None, - inactivity_timeout=30): + inactivity_timeout=120): ''' Monitors the io availability of 'infile'. Reads from infile and writes to outfile. If there is no activity on infile for a period of @@ -98,7 +98,8 @@ def monitor_subprocess_io(infile, last_read = time.time() done = False while not done: - reads, writes, errors = select(readfds, [], [], 30) + reads, writes, errors = select(readfds, [], [], + inactivity_timeout) if len(reads) > 0: for fd in reads: #check for each fds in read ready list diff --git a/tools/imaging/setup.py b/tools/imaging/setup.py index a675ebf7d92..b48f76b27d6 100644 --- a/tools/imaging/setup.py +++ b/tools/imaging/setup.py @@ -79,7 +79,7 @@ def add_paths_to_scripts(self): 'Topic :: Internet', ], install_requires=[ - "argparse", + "argparse", "lxml", "requests", ], data_files=[], cmdclass={'build_scripts': build_scripts_with_path_headers},