Fix upload quota check and provide better error message

.gitignore

@@ -12,6 +12,7 @@ target
 .metadata
 .project
 .idea
+artifactrepo
 
 # Windows image file caches
 Thumbs.db

hawkbit-core/src/main/java/org/eclipse/hawkbit/exception/SpServerError.java

@@ -137,6 +137,16 @@ public enum SpServerError {
      */
     SP_QUOTA_EXCEEDED("hawkbit.server.error.quota.tooManyEntries", "Too many entries have been inserted."),
 
+    /**
+     * error that describes that size of uploaded file exceeds size quota
+     */
+    SP_FILE_SIZE_QUOTA_EXCEEDED("hawkbit.server.error.quota.fileSizeExceeded", "File exceeds size quota."),
+
+    /**
+     * error that describes that size of uploaded file exceeds storage quota
+     */
+    SP_STORAGE_QUOTA_EXCEEDED("hawkbit.server.error.quota.storageExceeded", "Storage quota will be exceeded if file is uploaded."),
+
     /**
      * error message, which describes that the action can not be canceled cause
      * the action is inactive.

hawkbit-repository/hawkbit-repository-api/src/main/java/org/eclipse/hawkbit/repository/exception/QuotaExceededException.java

@@ -13,31 +13,75 @@
 import org.eclipse.hawkbit.repository.model.BaseEntity;
 
 /**
- * Thrown if too many entries are added to repository.
+ * Thrown if quota is exceeded (too many entries, file size, storage size).
  *
  */
 public final class QuotaExceededException extends AbstractServerRtException {
 
     private static final long serialVersionUID = 1L;
 
+    private static final String KB = "KB";
+    private static final String MB = "MB";
+
     private static final String ASSIGNMENT_QUOTA_EXCEEDED_MESSAGE = "Quota exceeded: Cannot assign %s more %s entities to %s '%s'. The maximum is %s.";
+    private static final String MAX_ARTIFACT_SIZE_EXCEEDED = "Maximum artifact size (%s) exceeded.";
+    private static final String MAX_ARTIFACT_SIZE_TOTAL_EXCEEDED = "Storage quota exceeded, %s left.";
+
+    private final QuotaType quotaType;
+
+    private final long exceededQuotaValue;
+
+    /**
+     * Enum that describes the types of quota that may be exceeded
+     */
+    public enum QuotaType {
+        STORAGE_QUOTA(SpServerError.SP_STORAGE_QUOTA_EXCEEDED, "message.upload.storageQuota"),
+        SIZE_QUOTA(SpServerError.SP_FILE_SIZE_QUOTA_EXCEEDED, "message.upload.fileSizeQuota"),
+        ASSIGNMENT_QUOTA(SpServerError.SP_QUOTA_EXCEEDED, "message.upload.artifactCountQuota");
+
+        private final SpServerError errorType;
+        public final String messageId;
+
+        QuotaType(final SpServerError errorType, final String messageId) {
+            this.errorType = errorType;
+            this.messageId = messageId;
+        }
+    }
 
     /**
      * Creates a new QuotaExceededException with
      * {@link SpServerError#SP_QUOTA_EXCEEDED} error.
      */
     public QuotaExceededException() {
-        super(SpServerError.SP_QUOTA_EXCEEDED);
+        super(QuotaType.ASSIGNMENT_QUOTA.errorType);
+        this.quotaType = QuotaType.ASSIGNMENT_QUOTA;
+        this.exceededQuotaValue = 0;
     }
 
     /**
      * Creates a new QuotaExceededException with a custom error message.
-     * 
+     *
      * @param message
      *            The custom error message.
      */
     public QuotaExceededException(final String message) {
-        super(message, SpServerError.SP_QUOTA_EXCEEDED);
+        super(message, QuotaType.ASSIGNMENT_QUOTA.errorType);
+        this.quotaType = QuotaType.ASSIGNMENT_QUOTA;
+        this.exceededQuotaValue = 0;
+    }
+
+    /**
+     * Creates a new QuotaExceededException with quota type and quota value.
+     *
+     * @param quotaType
+     *            {@link QuotaType} that will lead to the connected error type
+     * @param exceededQuotaValue
+     *            Value by how much the quota was exceeded
+     */
+    public QuotaExceededException(final QuotaType quotaType, final long exceededQuotaValue) {
+        super(createQuotaErrorMessage(quotaType, exceededQuotaValue), quotaType.errorType);
+        this.quotaType = quotaType;
+        this.exceededQuotaValue = exceededQuotaValue;
     }
 
     /**
@@ -50,7 +94,9 @@ public QuotaExceededException(final String message) {
      *            for the exception
      */
     public QuotaExceededException(final String message, final Throwable cause) {
-        super(message, SpServerError.SP_QUOTA_EXCEEDED, cause);
+        super(message, QuotaType.ASSIGNMENT_QUOTA.errorType, cause);
+        this.quotaType = QuotaType.ASSIGNMENT_QUOTA;
+        this.exceededQuotaValue = 0;
     }
 
     /**
@@ -76,7 +122,9 @@ public QuotaExceededException(final Class<? extends BaseEntity> type, final long
      */
     public QuotaExceededException(final String type, final long inserted, final int quota) {
         super("Request contains too many entries of {" + type + "}. {" + inserted + "} is beyond the permitted {"
-                + quota + "}.", SpServerError.SP_QUOTA_EXCEEDED);
+                + quota + "}.", QuotaType.ASSIGNMENT_QUOTA.errorType);
+        this.quotaType = QuotaType.ASSIGNMENT_QUOTA;
+        this.exceededQuotaValue = quota;
     }
 
     /**
@@ -121,10 +169,47 @@ public QuotaExceededException(final Class<?> type, final Class<?> parentType, fi
      *            parent entity.
      */
     public QuotaExceededException(final String type, final String parentType, final Object parentId,
-            final long requested,
-            final long quota) {
+            final long requested, final long quota) {
         super(String.format(ASSIGNMENT_QUOTA_EXCEEDED_MESSAGE, requested, type, parentType,
-                parentId != null ? String.valueOf(parentId) : "<new>", quota), SpServerError.SP_QUOTA_EXCEEDED);
+                parentId != null ? String.valueOf(parentId) : "<new>", quota), QuotaType.ASSIGNMENT_QUOTA.errorType);
+        this.quotaType = QuotaType.ASSIGNMENT_QUOTA;
+        this.exceededQuotaValue = quota;
+    }
+
+    public QuotaType getQuotaType() {
+        return quotaType;
+    }
+
+    public String getExceededQuotaValueString() {
+        switch (quotaType){
+            case STORAGE_QUOTA:
+            case SIZE_QUOTA:
+                return byteValueToReadableString(exceededQuotaValue);
+            default:
+                return String.valueOf(exceededQuotaValue);
+        }
+
+    }
+
+    private static String createQuotaErrorMessage(final QuotaType quotaType, final long exceededQuotaValue) {
+        if (quotaType == QuotaExceededException.QuotaType.STORAGE_QUOTA) {
+            return String.format(MAX_ARTIFACT_SIZE_TOTAL_EXCEEDED, byteValueToReadableString(exceededQuotaValue));
+        } else {
+            return String.format(MAX_ARTIFACT_SIZE_EXCEEDED, byteValueToReadableString(exceededQuotaValue));
+        }
     }
 
+    /**
+     * Convert byte values to human readable strings with units
+     */
+    private static String byteValueToReadableString(long byteValue) {
+        double outputValue = byteValue / 1024.0;
+        String unit = KB;
+        if (outputValue >= 1024) {
+            outputValue = outputValue / 1024.0;
+            unit = MB;
+        }
+        // We cut decimal places to avoid localization issues
+        return (long) outputValue + " " + unit;
+    }
 }

hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/JpaArtifactManagement.java

@@ -8,6 +8,8 @@
  */
 package org.eclipse.hawkbit.repository.jpa;
 
+import java.io.IOException;
+import java.io.InputStream;
 import java.util.Optional;
 
 import org.eclipse.hawkbit.artifact.repository.ArtifactRepository;
@@ -23,11 +25,11 @@
 import org.eclipse.hawkbit.repository.exception.EntityNotFoundException;
 import org.eclipse.hawkbit.repository.exception.InvalidMD5HashException;
 import org.eclipse.hawkbit.repository.exception.InvalidSHA1HashException;
-import org.eclipse.hawkbit.repository.exception.QuotaExceededException;
 import org.eclipse.hawkbit.repository.jpa.configuration.Constants;
 import org.eclipse.hawkbit.repository.jpa.model.JpaArtifact;
 import org.eclipse.hawkbit.repository.jpa.model.JpaSoftwareModule;
 import org.eclipse.hawkbit.repository.jpa.utils.QuotaHelper;
+import org.eclipse.hawkbit.repository.jpa.utils.FileSizeAndStorageQuotaCheckingInputStream;
 import org.eclipse.hawkbit.repository.model.Artifact;
 import org.eclipse.hawkbit.repository.model.ArtifactUpload;
 import org.eclipse.hawkbit.repository.model.SoftwareModule;
@@ -53,10 +55,6 @@ public class JpaArtifactManagement implements ArtifactManagement {
 
     private static final Logger LOG = LoggerFactory.getLogger(JpaArtifactManagement.class);
 
-    private static final String MAX_ARTIFACT_SIZE_EXCEEDED = "Quota exceeded: The artifact '%s' (%s bytes) which has been uploaded for software module '%s' exceeds the maximum artifact size of %s bytes.";
-
-    private static final String MAX_ARTIFACT_SIZE_TOTAL_EXCEEDED = "Quota exceeded: The artifact '%s' (%s bytes) cannot be uploaded. The maximum total artifact storage of %s bytes would be exceeded.";
-
     private final LocalArtifactRepository localArtifactRepository;
 
     private final SoftwareModuleRepository softwareModuleRepository;
@@ -105,8 +103,6 @@ public Artifact create(final ArtifactUpload artifactUpload) {
         final Artifact existing = checkForExistingArtifact(filename, artifactUpload.overrideExisting(), softwareModule);
 
         assertArtifactQuota(moduleId, 1);
-        assertMaxArtifactSizeQuota(filename, moduleId, artifactUpload.getFilesize());
-        assertMaxArtifactStorageQuota(filename, artifactUpload.getFilesize());
 
         return getOrCreateArtifact(artifactUpload)
                 .map(artifact -> storeArtifactMetadata(softwareModule, filename, artifact, existing)).orElse(null);
@@ -125,11 +121,16 @@ private Optional<AbstractDbArtifact> getOrCreateArtifact(final ArtifactUpload ar
     }
 
     private AbstractDbArtifact storeArtifact(final ArtifactUpload artifactUpload) {
-        try {
-            return artifactRepository.store(tenantAware.getCurrentTenant(), artifactUpload.getInputStream(),
+        AbstractDbArtifact dbArtifact = null;
+        try(final InputStream quotaStream = wrapInQuotaStream(artifactUpload.getInputStream())) {
+
+            dbArtifact = artifactRepository.store(tenantAware.getCurrentTenant(), quotaStream,
                     artifactUpload.getFilename(), artifactUpload.getContentType(),
-                    new DbArtifactHash(artifactUpload.getProvidedSha1Sum(), artifactUpload.getProvidedMd5Sum(), artifactUpload.getProvidedSha256Sum()));
-        } catch (final ArtifactStoreException e) {
+                    new DbArtifactHash(artifactUpload.getProvidedSha1Sum(), artifactUpload.getProvidedMd5Sum(),
+                            artifactUpload.getProvidedSha256Sum()));
+
+            return dbArtifact;
+        } catch (final ArtifactStoreException | IOException e) {
             throw new ArtifactUploadFailedException(e);
         } catch (final HashNotMatchException e) {
             if (e.getHashFunction().equals(HashNotMatchException.SHA1)) {
@@ -145,31 +146,13 @@ private void assertArtifactQuota(final long id, final int requested) {
                 Artifact.class, SoftwareModule.class, localArtifactRepository::countBySoftwareModuleId);
     }
 
-    private void assertMaxArtifactSizeQuota(final String filename, final long id, final long artifactSize) {
+    private InputStream wrapInQuotaStream(final InputStream in) {
         final long maxArtifactSize = quotaManagement.getMaxArtifactSize();
-        if (maxArtifactSize <= 0) {
-            return;
-        }
-        if (artifactSize > maxArtifactSize) {
-            final String msg = String.format(MAX_ARTIFACT_SIZE_EXCEEDED, filename, artifactSize, id, maxArtifactSize);
-            LOG.warn(msg);
-            throw new QuotaExceededException(msg);
-        }
-    }
 
-    private void assertMaxArtifactStorageQuota(final String filename, final long artifactSize) {
+        final long currentlyUsed = localArtifactRepository.getSumOfUndeletedArtifactSize().orElse(0L);
         final long maxArtifactSizeTotal = quotaManagement.getMaxArtifactStorage();
-        if (maxArtifactSizeTotal <= 0) {
-            return;
-        }
 
-        final Long currentlyUsed = localArtifactRepository.getSumOfUndeletedArtifactSize().orElse(0L);
-        if (currentlyUsed + artifactSize > maxArtifactSizeTotal) {
-            final String msg = String.format(MAX_ARTIFACT_SIZE_TOTAL_EXCEEDED, filename, artifactSize,
-                    maxArtifactSizeTotal);
-            LOG.warn(msg);
-            throw new QuotaExceededException(msg);
-        }
+        return new FileSizeAndStorageQuotaCheckingInputStream(in, maxArtifactSize, maxArtifactSizeTotal - currentlyUsed);
     }
 
     @Override

hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/utils/FileSizeAndStorageQuotaCheckingInputStream.java

@@ -0,0 +1,84 @@
+/**
+ * Copyright (c) 2019 Bosch Software Innovations GmbH and others.
+ *
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Eclipse Public License v1.0
+ * which accompanies this distribution, and is available at
+ * http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.eclipse.hawkbit.repository.jpa.utils;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.eclipse.hawkbit.repository.exception.QuotaExceededException;
+
+public class FileSizeAndStorageQuotaCheckingInputStream extends FilterInputStream {
+
+    private final long quota;
+
+    private final QuotaExceededException.QuotaType quotaType;
+
+    private long size;
+
+    /**
+     * Creates a <code>QuotaInputStream</code> using the input stream in and a
+     * limiting quota
+     * 
+     * @param sizeLimit
+     *            Quota file size limit in byte
+     * @param storageLeft
+     *            Storage left until quota is reached
+     */
+    public FileSizeAndStorageQuotaCheckingInputStream(final InputStream in, final long sizeLimit, final long storageLeft) {
+        super(in);
+
+        // only limit to lower bound and set appropriate error type
+        this.quota = Math.min(sizeLimit, storageLeft);
+        if (quota == sizeLimit) {
+            quotaType = QuotaExceededException.QuotaType.SIZE_QUOTA;
+        } else {
+            quotaType = QuotaExceededException.QuotaType.STORAGE_QUOTA;
+        }
+    }
+
+    @Override
+    public int read() throws IOException {
+        final int read = super.read();
+
+        if ((size + read) > quota) {
+            throw new QuotaExceededException(quotaType, quota);
+        } else if (read >= 0) {
+            size += read;
+        }
+
+        return read;
+    }
+
+    @Override
+    public int read(final byte[] b) throws IOException {
+        final int read = super.read(b);
+
+        if ((size + read) > quota) {
+            throw new QuotaExceededException(quotaType, quota);
+        } else if (read >= 0) {
+            size += read;
+        }
+
+        return read;
+    }
+
+    @Override
+    public int read(final byte[] b, final int off, final int len) throws IOException {
+        final int read = super.read(b, off, len);
+
+        if ((size + read) > quota) {
+            throw new QuotaExceededException(quotaType, quota);
+        } else if (read >= 0) {
+            size += read;
+        }
+
+        return read;
+    }
+}

hawkbit-repository/hawkbit-repository-jpa/src/main/java/org/eclipse/hawkbit/repository/jpa/utils/QuotaHelper.java

@@ -17,7 +17,7 @@
 import org.slf4j.LoggerFactory;
 
 /**
- * Helper class to check assignment quotas.
+ * Helper class to check quotas.
  */
 public final class QuotaHelper {
 
@@ -26,6 +26,8 @@ public final class QuotaHelper {
      */
     private static final Logger LOG = LoggerFactory.getLogger(QuotaHelper.class);
 
+    private static final String MAX_ASSIGNMENT_QUOTA_EXCEEDED = "Quota exceeded: Cannot assign %s entities at once. The maximum is %s.";
+
     private QuotaHelper() {
         // no need to instantiate this class
     }
@@ -142,8 +144,7 @@ public static <T> void assertAssignmentQuota(final T parentId, final long reques
      */
     public static void assertAssignmentRequestSizeQuota(final long requested, final long limit) {
         if (requested > limit) {
-            final String message = String.format(
-                    "Quota exceeded: Cannot assign %s entities at once. The maximum is %s.", requested, limit);
+            final String message = String.format(MAX_ASSIGNMENT_QUOTA_EXCEEDED, requested, limit);
             LOG.warn(message);
             throw new QuotaExceededException(message);
         }