HADOOP-19854: [ABFS][Backport3.5] Namespace Detection for FNS Accounts with Soft Delete Enabled

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java

@@ -162,6 +162,7 @@
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.INFINITE_LEASE_DURATION;
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_BLOB_DOMAIN_NAME;
 import static org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations.X_MS_ENCRYPTION_CONTEXT;
+import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_SOFT_DELETE_NOT_SUPPORTED;
 import static org.apache.hadoop.fs.azurebfs.utils.UriUtils.isKeyForDirectorySet;
 
 /**
@@ -422,17 +423,35 @@ private synchronized boolean getNamespaceEnabledInformationFromServer(
     } catch (AbfsRestOperationException ex) {
       // Get ACL status is a HEAD request, its response doesn't contain errorCode
       // So can only rely on its status code to determine account type.
-      if (HttpURLConnection.HTTP_BAD_REQUEST != ex.getStatusCode()) {
+      int status = ex.getStatusCode();
+      String message = ex.getMessage();
+
+      // Case 1: 409: Soft delete not supported
+      if (status == HttpURLConnection.HTTP_CONFLICT
+              && message != null
+              && message.contains(ERR_SOFT_DELETE_NOT_SUPPORTED)) {
+        /*
+         * HTTP_CONFLICT with soft delete not supported error indicates a FNS account.
+         * This occurs when:
+         * 1. FNS-Blob endpoint is used (due to DFS endpoint usage for getAcl call)
+         * 2. FNS-DFS endpoint is used (later internally converted to Blob)
+         * For both cases, namespace should be disabled. The exception is irrelevant here
+         * since we'll be using Blob endpoint ultimately for both the cases here.
+         * No need to throw the exception.
+         */
+        LOG.debug("Ignore soft-delete error. Setting namespace enabled to false.");
+        setNamespaceEnabled(false);
+      } else if (status == HttpURLConnection.HTTP_BAD_REQUEST) { // Case 2: 400
+        // If getAcl fails with 400, namespace is disabled.
+        LOG.debug("Failed to get ACL status with 400. Inferring namespace disabled and ignoring error", ex);
+        setNamespaceEnabled(false);
+      } else {
         // If getAcl fails with anything other than 400, namespace is enabled.
         setNamespaceEnabled(true);
         // Continue to throw exception as earlier.
         LOG.debug("Failed to get ACL status with non 400. Inferring namespace enabled", ex);
         throw ex;
       }
-      // If getAcl fails with 400, namespace is disabled.
-      LOG.debug("Failed to get ACL status with 400. "
-          + "Inferring namespace disabled and ignoring error", ex);
-      setNamespaceEnabled(false);
     } catch (AzureBlobFileSystemException ex) {
       throw ex;
     }

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsErrors.java

@@ -59,6 +59,10 @@ public final class AbfsErrors {
       + "and cannot be appended to by the Azure Data Lake Storage Service API";
   public static final String CONDITION_NOT_MET = "The condition specified using "
       + "HTTP conditional header(s) is not met.";
+  public static final String ERR_READ_ON_DIRECTORY = "Read operation not permitted on a directory.";
+  public static final String ERR_OPENFILE_ON_DIRECTORY = "openFileForRead must be used with files and not directories";
+  public static final String ERR_SOFT_DELETE_NOT_SUPPORTED = "This endpoint does not support BlobStorageEvents or SoftDelete.";
+
   /**
    * Exception message on filesystem init if token-provider-auth-type configs are provided
    */

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestGetNameSpaceEnabled.java

@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.util.UUID;
 
+import org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants;
 import org.junit.jupiter.api.Test;
 import org.assertj.core.api.Assertions;
 import org.mockito.Mockito;
@@ -38,6 +39,7 @@
 import org.apache.hadoop.fs.azurebfs.utils.TracingContext;
 
 import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
+import static java.net.HttpURLConnection.HTTP_CONFLICT;
 import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
 import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
 import static java.net.HttpURLConnection.HTTP_UNAVAILABLE;
@@ -47,6 +49,7 @@
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_BLOB_DOMAIN_NAME;
 import static org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes.ABFS_DFS_DOMAIN_NAME;
 import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.FS_AZURE_ACCOUNT_KEY;
+import static org.apache.hadoop.fs.azurebfs.services.AbfsErrors.ERR_SOFT_DELETE_NOT_SUPPORTED;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.doReturn;
@@ -266,6 +269,8 @@ public void ensureGetAclDetermineHnsStatusAccurately() throws Exception {
         true, true);
     ensureGetAclDetermineHnsStatusAccuratelyInternal(HTTP_UNAVAILABLE,
         true, true);
+    ensureGetAclDetermineHnsStatusAccuratelyInternal(HTTP_CONFLICT,
+            false, false);
   }
 
   private void ensureGetAclDetermineHnsStatusAccuratelyInternal(int statusCode,
@@ -274,8 +279,11 @@ private void ensureGetAclDetermineHnsStatusAccuratelyInternal(int statusCode,
     AbfsClient mockClient = mock(AbfsClient.class);
     store.getAbfsConfiguration().setIsNamespaceEnabledAccountForTesting(Trilean.UNKNOWN);
     doReturn(mockClient).when(store).getClient(AbfsServiceType.DFS);
+    String errorMsg = statusCode == HTTP_CONFLICT
+            ? ERR_SOFT_DELETE_NOT_SUPPORTED
+            : Integer.toString(statusCode);
     AbfsRestOperationException ex = new AbfsRestOperationException(
-        statusCode, null, Integer.toString(statusCode), null);
+        statusCode, null, errorMsg, null);
     doThrow(ex).when(mockClient).getAclStatus(anyString(), any(TracingContext.class));
 
     if (isExceptionExpected) {
@@ -301,6 +309,99 @@ private void ensureGetAclDetermineHnsStatusAccuratelyInternal(int statusCode,
         .getAclStatus(anyString(), any(TracingContext.class));
   }
 
+    /**
+     * Verify that for FNS accounts, the error code returned by the server
+     * is either HTTP 400 (Bad Request) or HTTP 409 (if soft-delete enabled). This validates the expected
+     * server behavior when getAcl is called on FNS accounts.
+     */
+    @Test
+    public void testFNSAccountReturnsExpectedErrorCodes() throws Exception {
+      assumeHnsDisabled();
+      Configuration config = getConfigurationWithoutHnsConfig();
+
+      // Spy on the client to capture the actual exception thrown
+      try (AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.newInstance(config)) {
+        AzureBlobFileSystemStore spyStore = Mockito.spy(fs.getAbfsStore());
+        AbfsClient spyClient = Mockito.spy(spyStore.getClient(AbfsServiceType.DFS));
+
+        doReturn(spyClient).when(spyStore).getClient(AbfsServiceType.DFS);
+
+        // Force namespace to be unknown to trigger server call
+        spyStore.getAbfsConfiguration().setIsNamespaceEnabledAccountForTesting(Trilean.UNKNOWN);
+
+        // Capture any exception that might be thrown during getAclStatus
+        AbfsRestOperationException capturedException = null;
+        try {
+          spyClient.getAclStatus(AbfsHttpConstants.ROOT_PATH,
+              getTestTracingContext(fs, false));
+        } catch (AbfsRestOperationException ex) {
+          capturedException = ex;
+        }
+
+        // For FNS accounts, we expect either 400 or 409
+
+        // NOTE: 409 (soft-delete not supported error) would come explicitly when we have
+        // the test account's soft-delete enabled. If soft-delete is disabled for the test account,
+        // we should get 400 instead.
+        if (capturedException != null) {
+          int statusCode = capturedException.getStatusCode();
+          String errorMessage = capturedException.getMessage();
+
+          Assertions.assertThat(statusCode)
+              .describedAs("FNS account should return either 400 or 409 status code")
+              .isIn(HTTP_BAD_REQUEST, HTTP_CONFLICT);
+
+          // If it's 409, verify it contains unsupported soft delete error message
+          if (statusCode == HTTP_CONFLICT) {
+            Assertions.assertThat(errorMessage)
+                .describedAs("HTTP 409 response should contain soft delete error message")
+                .contains(ERR_SOFT_DELETE_NOT_SUPPORTED);
+          }
+        }
+
+        // Verify that namespace is set to false regardless of which error was returned
+        boolean isHnsEnabled = spyStore.getIsNamespaceEnabled(getTestTracingContext(fs, false));
+        Assertions.assertThat(isHnsEnabled)
+            .describedAs("FNS account should have namespace disabled")
+            .isFalse();
+      }
+    }
+
+    /**
+     * Verify behavior when getAcl call fails with unexpected error codes
+     * (neither 400 nor 409). In such cases, namespace should be set to true and exception
+     * should be propagated.
+     */
+    @Test
+    public void testErrorCodeSetsNamespaceToTrueAndThrowsException() throws Exception {
+      // Create mock setup to simulate unexpected error code
+      AzureBlobFileSystemStore store = Mockito.spy(getFileSystem().getAbfsStore());
+      AbfsClient mockClient = mock(AbfsClient.class);
+      store.getAbfsConfiguration().setIsNamespaceEnabledAccountForTesting(Trilean.UNKNOWN);
+      doReturn(mockClient).when(store).getClient(AbfsServiceType.DFS);
+
+      // Simulate 500 Internal Server Error (unexpected error code)
+      AbfsRestOperationException unexpectedException = new AbfsRestOperationException(
+          HTTP_INTERNAL_ERROR, null, "Internal Server Error", null);
+      doThrow(unexpectedException).when(mockClient)
+          .getAclStatus(anyString(), any(TracingContext.class));
+
+      // Attempt to get namespace status should throw the exception
+      try {
+        store.getIsNamespaceEnabled(getTestTracingContext(getFileSystem(), false));
+        Assertions.fail("Expected AbfsRestOperationException to be thrown");
+      } catch (AbfsRestOperationException ex) {
+        Assertions.assertThat(ex.getStatusCode())
+            .describedAs("Exception should have 500 status code")
+            .isEqualTo(HTTP_INTERNAL_ERROR);
+      }
+
+      // Even though exception was thrown, namespace should be set to true
+      Assertions.assertThat(store.getAbfsConfiguration().getIsNamespaceEnabledAccount())
+          .describedAs("Namespace should be set to TRUE for unexpected error codes")
+          .isEqualTo(Trilean.TRUE);
+    }
+
   @Test
   public void testAccountSpecificConfig() throws Exception {
     Configuration rawConfig = new Configuration();