Merge commit '04096830d6f939c301a30337106eb7a765c3a799' into HEAD

clc/modules/object-storage-common/src/main/java/com/eucalyptus/objectstorage/entities/ObjectStorageGlobalConfiguration.java

@@ -62,6 +62,8 @@
 
 package com.eucalyptus.objectstorage.entities;
 
+import com.eucalyptus.upgrade.Upgrades.EntityUpgrade;
+import com.eucalyptus.upgrade.Upgrades.Version;
 import groovy.sql.GroovyRowResult;
 import groovy.sql.Sql;
 
@@ -70,11 +72,7 @@
 import java.util.Set;
 
 import javax.annotation.Nullable;
-import javax.persistence.Column;
-import javax.persistence.Entity;
-import javax.persistence.PersistenceContext;
-import javax.persistence.Table;
-import javax.persistence.Transient;
+import javax.persistence.*;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
@@ -118,6 +116,8 @@ public class ObjectStorageGlobalConfiguration extends AbstractPersistent impleme
   @Transient
   private static final int DEFAULT_MAX_BUCKETS_PER_ACCOUNT = 100;
   @Transient
+  private static final int DEFAULT_MAX_INBOUND_HTTP_CHUNK_SIZE = 1024 * 1024 * 10; // 10 MB
+  @Transient
   private static final int DEFAULT_PUT_TIMEOUT_HOURS = 168; // An upload not marked completed or deleted in 24 hours from record creation will be
                                                             // considered 'failed'
   @Transient
@@ -132,6 +132,10 @@ public ObjectStorageGlobalConfiguration getLatest() {
     return getConfiguration();
   }
 
+  @Column
+  @ConfigurableField(description = "Maximum allowed size of inbound http chunks", displayName = "Maximum inbound http chunk size")
+  protected Integer max_inbound_http_chunk_size;
+
   @Column
   @ConfigurableField(description = "Maximum number of buckets per account", displayName = "Maximum buckets per account")
   protected Integer max_buckets_per_account;
@@ -182,6 +186,7 @@ protected ObjectStorageGlobalConfiguration initializeDefaults() {
     this.setDoGetPutOnCopyFail(DEFAULT_COPY_UNSUPPORTED_STRATEGY);
     this.setFailed_put_timeout_hrs(DEFAULT_PUT_TIMEOUT_HOURS);
     this.setMax_buckets_per_account(DEFAULT_MAX_BUCKETS_PER_ACCOUNT);
+    this.setMax_inbound_http_chunk_size(DEFAULT_MAX_INBOUND_HTTP_CHUNK_SIZE);
     this.setMax_total_reporting_capacity_gb(Integer.MAX_VALUE);
     return this;
   }
@@ -201,6 +206,14 @@ public void fireChange(ConfigurableProperty t, Object newValue) throws Configura
 
   }
 
+  public Integer getMax_inbound_http_chunk_size() {
+    return max_inbound_http_chunk_size != null ? max_inbound_http_chunk_size : DEFAULT_MAX_INBOUND_HTTP_CHUNK_SIZE;
+  }
+
+  public void setMax_inbound_http_chunk_size(Integer max_inbound_http_chunk_size) {
+    this.max_inbound_http_chunk_size = max_inbound_http_chunk_size;
+  }
+
   public Integer getMax_buckets_per_account() {
     return max_buckets_per_account;
   }
@@ -265,6 +278,14 @@ public void setProviderClient(String providerClient) {
     this.providerClient = providerClient;
   }
 
+  @PrePersist
+  @PreUpdate
+  public void updateDefaults() {
+    if (max_inbound_http_chunk_size == null) {
+      max_inbound_http_chunk_size = DEFAULT_MAX_INBOUND_HTTP_CHUNK_SIZE;
+    }
+  }
+
   /**
    * Gets this config from the DB. May throw an exception on db failure
    * 
@@ -373,6 +394,36 @@ public boolean apply(@Nullable Class arg0) {
 
     }
 
+    /**
+     * Upgrade OSG configuration for 4.4.
+     *
+     * @throws Exception
+     */
+    @Upgrades.EntityUpgrade(entities = {ObjectStorageGlobalConfiguration.class}, since = Upgrades.Version.v4_4_0, value = ObjectStorage.class)
+    public static enum OSG44ConfigUpgrade implements Predicate<Class> {
+      INSTANCE;
+
+      @Override
+      public boolean apply(@Nullable Class arg0) {
+        try (TransactionResource trans = Entities.transactionFor(arg0)) {
+          ObjectStorageGlobalConfiguration config;
+          try {
+            config = Entities.uniqueResult(new ObjectStorageGlobalConfiguration());
+          } catch (NoSuchElementException e) {
+            config = new ObjectStorageGlobalConfiguration().initializeDefaults();
+          }
+          config.updateDefaults();
+          Entities.persist(config);
+          trans.commit();
+          return true;
+        } catch (Exception e) {
+          String msg = "Error saving OSG 4.4 configuration upgrade";
+          LOG.error(msg, e);
+          throw Exceptions.toUndeclared(msg, e);
+        }
+      }
+    }
+
     /*
      * Get the value of objectstorage.providerclient stored in config_static_property table. Had to use sql directly as StaticDatabasePropertyEntry
      * has a private constructor that restricts its usage for lookup/delete operations

clc/modules/object-storage-common/src/main/java/com/eucalyptus/objectstorage/exceptions/s3/S3Exceptions.groovy

@@ -712,6 +712,12 @@ class MaxMessageLengthExceededException extends S3Exception {
     this();
     this.resource = resource;
   }
+
+  def MaxMessageLengthExceededException(String resource, String message) {
+    this();
+    this.resource = resource;
+    this.message = message;
+  }
 }
 
 class MaxPostPreDataLengthExceededErrorException extends S3Exception {

clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/pipeline/handlers/ObjectStorageAuthenticationAggregator.java

@@ -66,11 +66,12 @@
 import com.eucalyptus.http.MappingHttpRequest;
 import com.eucalyptus.objectstorage.OSGChannelWriter;
 import com.eucalyptus.objectstorage.OSGMessageResponse;
-import com.eucalyptus.objectstorage.exceptions.s3.MissingContentLengthException;
+import com.eucalyptus.objectstorage.exceptions.s3.MaxMessageLengthExceededException;
 import com.eucalyptus.objectstorage.pipeline.auth.S3V4Authentication;
 import com.eucalyptus.objectstorage.pipeline.handlers.AwsChunkStream.StreamingHttpRequest;
 import com.google.common.base.Strings;
 import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.Channels;
 import org.jboss.netty.channel.MessageEvent;
 import org.jboss.netty.channel.UpstreamMessageEvent;
 import org.jboss.netty.handler.codec.http.HttpChunk;
@@ -83,7 +84,7 @@
  * Aggregates streaming and chunked data for V4 signed request authentication.
  */
 public class ObjectStorageAuthenticationAggregator extends HttpChunkAggregator {
-
+  private final int maxContentLength;
 
   // Http chunking required
   private boolean httpChunked;
@@ -94,6 +95,7 @@ public class ObjectStorageAuthenticationAggregator extends HttpChunkAggregator {
 
   public ObjectStorageAuthenticationAggregator(int maxContentLength) {
     super(maxContentLength);
+    this.maxContentLength = maxContentLength;
   }
 
   @Override
@@ -115,6 +117,11 @@ public void messageReceived(ChannelHandlerContext ctx, MessageEvent event) throw
         if (!Strings.isNullOrEmpty(authHeader) && authHeader.startsWith(S3V4Authentication.AWS_V4_AUTH_TYPE) && HttpHeaders
             .is100ContinueExpected(request)) {
           httpChunked = true;
+          long contentLength = HttpHeaders.getContentLength(request);
+          if (contentLength > maxContentLength) {
+            Channels.fireExceptionCaught(ctx, new MaxMessageLengthExceededException(null, "Max request content length exceeded."));
+            return;
+          }
 
           // Clear expect header and send continue
           HttpHeaders.set100ContinueExpected(request, false);
@@ -141,9 +148,9 @@ public void messageReceived(ChannelHandlerContext ctx, MessageEvent event) throw
     }
 
     // Aggregate continuable requests
-    if (httpChunked)
+    if (httpChunked) {
       super.messageReceived(ctx, event);
-    else
+    } else
       ctx.sendUpstream(event);
   }
 }

clc/modules/object-storage/src/main/java/com/eucalyptus/objectstorage/pipeline/stages/ObjectStorageAuthenticationAggregatorStage.java

@@ -62,7 +62,9 @@
 
 package com.eucalyptus.objectstorage.pipeline.stages;
 
+import com.eucalyptus.objectstorage.entities.ObjectStorageGlobalConfiguration;
 import com.eucalyptus.objectstorage.pipeline.handlers.ObjectStorageAuthenticationAggregator;
+import com.eucalyptus.storage.config.ConfigurationCache;
 import com.eucalyptus.ws.StackConfiguration;
 import com.eucalyptus.ws.stages.UnrollableStage;
 import org.jboss.netty.channel.ChannelPipeline;
@@ -75,7 +77,8 @@ public class ObjectStorageAuthenticationAggregatorStage implements UnrollableSta
 
   @Override
   public void unrollStage(ChannelPipeline pipeline) {
-    pipeline.addLast(NAME, new ObjectStorageAuthenticationAggregator(StackConfiguration.CLIENT_HTTP_CHUNK_BUFFER_MAX));
+    pipeline.addLast(NAME, new ObjectStorageAuthenticationAggregator(ConfigurationCache.getConfiguration(
+        ObjectStorageGlobalConfiguration.class).getMax_inbound_http_chunk_size()));
   }
 
   @Override