TW-80648 - Added toggle to disable integrity verification in storage …

…settings Merge-request: TC-MR-6964 Merged-by: Dmitrii Kirkhmeier <Dmitrii.Kirkhmeier@jetbrains.com>
JetBrains · Sep 21, 2023 · 509fe77 · 509fe77
1 parent 568193e
commit 509fe77
Show file tree

Hide file tree

Showing 12 changed files with 33 additions and 5 deletions.
diff --git a/s3-artifact-storage-server/kotlin-dsl/S3CommonSettings.xml b/s3-artifact-storage-server/kotlin-dsl/S3CommonSettings.xml
@@ -30,6 +30,11 @@
         Whether to force Virtual Host Addressing
       </description>
     </param>
+    <param name="storage.s3.verifyIntegrityAfterUpload" dslName="verifyIntegrityAfterUpload" type="boolean">
+      <description>
+        Whether to verify integrity of artifacts after upload
+      </description>
+    </param>
     <param name="storage.s3.upload.multipart_threshold" dslName="multipartThreshold">
       <description>
         Initiates multipart upload for files larger than the specified value.

diff --git a/...rver/src/main/java/jetbrains/buildServer/artifacts/s3/settings/S3PropertiesProcessor.java b/...rver/src/main/java/jetbrains/buildServer/artifacts/s3/settings/S3PropertiesProcessor.java
@@ -4,6 +4,7 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
+import jetbrains.buildServer.artifacts.s3.S3Constants;
 import jetbrains.buildServer.artifacts.s3.S3Util;
 import jetbrains.buildServer.artifacts.s3.cloudfront.CloudFrontConstants;
 import jetbrains.buildServer.clouds.amazon.connector.featureDevelopment.ChosenAwsConnPropertiesProcessor;
@@ -42,6 +43,10 @@ public Collection<InvalidProperty> process(Map<String, String> params) {
       }
     }
 
+    if (StringUtil.isEmptyOrSpaces(params.get(S3Constants.S3_VERIFY_INTEGRITY_AFTER_UPLOAD))) {
+      params.put(S3Constants.S3_VERIFY_INTEGRITY_AFTER_UPLOAD, "false");
+    }
+
     return invalids;
   }
 }
diff --git a/...age-server/src/main/java/jetbrains/buildServer/artifacts/s3/web/S3ParametersProvider.java b/...age-server/src/main/java/jetbrains/buildServer/artifacts/s3/web/S3ParametersProvider.java
@@ -48,6 +48,9 @@ public String getContainersPath() {
   public String getForceVirtualHostAddressing() {
     return S3Constants.S3_FORCE_VIRTUAL_HOST_ADDRESSING;
   }
+  public String getVerifyIntegrityAfterUpload(){
+    return S3Constants.S3_VERIFY_INTEGRITY_AFTER_UPLOAD;
+  }
 
   public String getEnableAccelerateMode() {
     return S3Constants.S3_ENABLE_ACCELERATE_MODE;

diff --git a/...t-storage-server/src/main/resources/buildServerResources/s3_storage_settings_react_ui.jsp b/...t-storage-server/src/main/resources/buildServerResources/s3_storage_settings_react_ui.jsp
@@ -129,6 +129,7 @@
 
     usePresignUrlsForUpload: "<bs:forJs>${propertiesBean.properties[params.usePresignUrlsForUpload]}</bs:forJs>" === "true",
     forceVirtualHostAddressing: "<bs:forJs>${propertiesBean.properties[params.forceVirtualHostAddressing]}</bs:forJs>" === "true",
+    verifyIntegrityAfterUpload: "<bs:forJs>${propertiesBean.properties[params.verifyIntegrityAfterUpload]}</bs:forJs>" === "true",
     transferAccelerationOn: "<bs:forJs>${Boolean.parseBoolean(transferAccelerationOn)}</bs:forJs>" === "true",
     enableAccelerateMode: "<bs:forJs>${propertiesBean.properties[params.enableAccelerateMode]}</bs:forJs>" === "true",
     multipartUploadThreshold: "<bs:forJs>${propertiesBean.properties[params.multipartUploadThreshold]}</bs:forJs>",

diff --git a/s3-artifact-storage-ui/src/App/ProtocolSettings/ProtocolSettings.tsx b/s3-artifact-storage-ui/src/App/ProtocolSettings/ProtocolSettings.tsx
@@ -35,6 +35,11 @@ export default function ProtocolSettings() {
         label="Force virtual host addressing"
         disabled={s3TransferAcceleration ?? false}
       />
+        <FormCheckbox
+            name={FormFields.CONNECTION_VERIFY_IAU_TOGGLE}
+            control={control}
+            label="Verify file integrity after upload"
+        />
     </section>
   );
 }
diff --git a/s3-artifact-storage-ui/src/App/appConstants.tsx b/s3-artifact-storage-ui/src/App/appConstants.tsx
@@ -29,6 +29,7 @@ export enum FormFields {
   CLOUD_FRONT_PRIVATE_KEY = 'prop:secure:storage_s3_cloudfront_privateKey',
   CONNECTION_PRESIGNED_URL_TOGGLE = 'prop:storage_s3_upload_presignedUrl_enabled',
   CONNECTION_FORCE_VHA_TOGGLE = 'prop:storage_s3_forceVirtualHostAddressing',
+  CONNECTION_VERIFY_IAU_TOGGLE = 'prop:storage_s3_verifyIntegrityAfterUpload',
   CONNECTION_TRANSFER_ACCELERATION_TOGGLE = 'prop:storage_s3_accelerateModeEnabled',
   CONNECTION_MULTIPART_THRESHOLD = 'prop:storage_s3_upload_multipart-threshold',
   CONNECTION_MULTIPART_CHUNKSIZE = 'prop:storage_s3_upload_multipart-chunksize',

diff --git a/s3-artifact-storage-ui/src/hooks/useS3Form.tsx b/s3-artifact-storage-ui/src/hooks/useS3Form.tsx
@@ -66,6 +66,9 @@ export default function useS3Form() {
     ? true
     : config.forceVirtualHostAddressing;
 
+  const verifyIntegrityAfterUpload =
+    config.isNewStorage || config.verifyIntegrityAfterUpload;
+
   const environmentTypeValue = useMemo(() => {
     if (storageTypeValue?.key === S3_COMPATIBLE) {
       return AWS_ENV_TYPE_ARRAY[1];
@@ -99,6 +102,7 @@ export default function useS3Form() {
       [FormFields.CONNECTION_PRESIGNED_URL_TOGGLE]:
         config.usePresignUrlsForUpload,
       [FormFields.CONNECTION_FORCE_VHA_TOGGLE]: forceVirtualHostAddressingValue,
+      [FormFields.CONNECTION_VERIFY_IAU_TOGGLE]: verifyIntegrityAfterUpload,
       [FormFields.CONNECTION_TRANSFER_ACCELERATION_TOGGLE]:
         config.enableAccelerateMode,
       [FormFields.CONNECTION_MULTIPART_THRESHOLD]:

diff --git a/s3-artifact-storage-ui/src/types/index.ts b/s3-artifact-storage-ui/src/types/index.ts
@@ -41,6 +41,7 @@ export type Config = {
   cloudFrontPrivateKey: string;
   usePresignUrlsForUpload: boolean;
   forceVirtualHostAddressing: boolean;
+  verifyIntegrityAfterUpload: boolean;
   enableAccelerateMode: boolean;
   multipartUploadThreshold: string;
   multipartUploadPartSize: string;
@@ -76,6 +77,7 @@ export interface S3FormInput {
   [FormFields.CLOUD_FRONT_PRIVATE_KEY]: string;
   [FormFields.CONNECTION_PRESIGNED_URL_TOGGLE]: boolean;
   [FormFields.CONNECTION_FORCE_VHA_TOGGLE]: boolean;
+  [FormFields.CONNECTION_VERIFY_IAU_TOGGLE]: boolean;
   [FormFields.CONNECTION_TRANSFER_ACCELERATION_TOGGLE]: boolean;
   [FormFields.CONNECTION_MULTIPART_THRESHOLD]: string;
   [FormFields.CONNECTION_MULTIPART_CHUNKSIZE]: string;

diff --git a/teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/S3Constants.java b/teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/S3Constants.java
@@ -42,6 +42,8 @@ public class S3Constants {
   public static final String S3_PRESIGNED_UPLOAD_INTERNAL_CHUNK_SIZE = "teamcity.internal.storage.s3.upload.presignedUrl.internalChunkSize";
   public static final String S3_FORCE_VIRTUAL_HOST_ADDRESSING = "storage.s3.forceVirtualHostAddressing";
 
+  public static final String S3_VERIFY_INTEGRITY_AFTER_UPLOAD = "storage.s3.verifyIntegrityAfterUpload";
+
   public static final String S3_TRANSFER_ACCELERATION_FEATURE_ENABLED = "teamcity.s3.transfer.acceleration.enabled";
   public static final String S3_ENABLE_ACCELERATE_MODE = "storage.s3.accelerateModeEnabled";
   public static final String S3_CLEANUP_BATCH_SIZE = "storage.s3.cleanup.batchSize";

diff --git a/...y-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/S3PresignedUrlProviderImpl.java b/...y-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/S3PresignedUrlProviderImpl.java
@@ -90,7 +90,7 @@ private String generateUrl(@NotNull final HttpMethod httpMethod,
       if (uploadId != null) {
         request.addRequestParameter("uploadId", uploadId);
       }
-      if (S3Util.isConsistencyCheckEnabled(settings.getProjectSettings()) && digest != null) {
+      if (S3Util.isConsistencyCheckEnabled(settings.toRawSettings()) && digest != null) {
         request.setContentMd5(digest);
       }
 

diff --git a/teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/S3Util.java b/teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/S3Util.java
@@ -277,9 +277,9 @@ public static int getUrlTtlSeconds(@NotNull final Map<String, String> configurat
     return Integer.parseInt(configuration.getOrDefault(S3_URL_LIFETIME_SEC, String.valueOf(TeamCityProperties.getInteger(S3_URL_LIFETIME_SEC, DEFAULT_URL_LIFETIME_SEC))));
   }
 
-  public static boolean isConsistencyCheckEnabled(@NotNull final Map<String, String> configuration) {
-    return Boolean.parseBoolean(
-      configuration.getOrDefault(S3_ENABLE_CONSISTENCY_CHECK, TeamCityProperties.getProperty(S3_ENABLE_CONSISTENCY_CHECK, String.valueOf(DEFAULT_ENABLE_CONSISTENCY_CHECK))));
+  public static boolean isConsistencyCheckEnabled(Map<String, String> storageSettings) {
+    return Boolean.parseBoolean(TeamCityProperties.getProperty(S3_ENABLE_CONSISTENCY_CHECK, String.valueOf(DEFAULT_ENABLE_CONSISTENCY_CHECK))) &&
+           Boolean.parseBoolean(storageSettings.getOrDefault(S3_VERIFY_INTEGRITY_AFTER_UPLOAD, "true"));
   }
 
   @NotNull

diff --git a/teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/S3FileUploader.java b/teamcity-s3-sdk/src/main/java/jetbrains/buildServer/artifacts/s3/publish/S3FileUploader.java
@@ -54,7 +54,7 @@ public static S3Util.S3AdvancedConfiguration configuration(@NotNull final Map<St
       .withConnectionTimeout(jetbrains.buildServer.artifacts.s3.S3Util.getConnectionTimeout(sharedConfigurationParameters))
       .withNumberOfThreads(jetbrains.buildServer.artifacts.s3.S3Util.getNumberOfThreadsForFiles(sharedConfigurationParameters))
       .withUrlTtlSeconds(jetbrains.buildServer.artifacts.s3.S3Util.getUrlTtlSeconds(sharedConfigurationParameters))
-      .withConsistencyCheckEnabled(jetbrains.buildServer.artifacts.s3.S3Util.isConsistencyCheckEnabled(sharedConfigurationParameters))
+      .withConsistencyCheckEnabled(jetbrains.buildServer.artifacts.s3.S3Util.isConsistencyCheckEnabled(artifactStorageSettings))
       .withAcl(jetbrains.buildServer.artifacts.s3.S3Util.getAcl(artifactStorageSettings, sharedConfigurationParameters))
       .allowPlainHttpUpload(jetbrains.buildServer.artifacts.s3.S3Util.isAllowPlainHttpUpload(sharedConfigurationParameters))
       .withShutdownClient();