[BACKPORT 2.16][PLAT-7538][PLAT-7546] YCQL backup fails when Snapshot…

… details contain custom types, move rearrange_snapshot_dir logs to verbose

Summary:
[PLAT-7538]
Original PR: https://phabricator.dev.yugabyte.com/D23160
Original commit hash: bd2c1b9
- Fixed the Success marker parsing to overlook custom types( other than NAMESPACE, TABLE )

[PLAT-7546]
Original PR: https://phabricator.dev.yugabyte.com/D23171
Original commit hash:  4dc5778
- Moved `rearrange_snapshot_dir` warning logs to verbose mode

Test Plan:
[PLAT-7538]
- Modified UTs to cover this case
- Verified backup/restore with custom types

[PLAT-7546]
- No tests

Reviewers: vpatibandla, kkg

Reviewed By: kkg

Subscribers: jenkins-bot, yugaware

Differential Revision: https://phabricator.dev.yugabyte.com/D23166

managed/devops/bin/yb_backup.py

@@ -2425,12 +2425,13 @@ def rearrange_snapshot_dirs(
                 # FOLLOWERS replicas are not in the 'tablets_by_tserver_ip' list
                 # (the list 'tablets_by_tserver_ip' contains only the LEADER replicas).
                 if tablet_id not in tablets_by_tserver_ip[tserver_ip]:
-                    logging.warning(
-                        ("Found a snapshot directory '{}' on tablet server '{}' that is not "
-                         "present in the list of tablets we are interested in that have this "
-                         "tserver hosting it ({}), skipping.").format(
-                            snapshot_dir, tserver_ip,
-                            ", ".join(sorted(tablets_by_tserver_ip[tserver_ip]))))
+                    if self.args.verbose:
+                        logging.warning(
+                            ("Found a snapshot directory '{}' on tablet server '{}' that is not "
+                             "present in the list of tablets we are interested in that have this "
+                             "tserver hosting it ({}), skipping.").format(
+                                snapshot_dir, tserver_ip,
+                                ", ".join(sorted(tablets_by_tserver_ip[tserver_ip]))))
                     continue
 
                 tablet_id_to_snapshot_dirs.setdefault(tablet_id, set()).add(snapshot_dir)

managed/src/main/java/com/yugabyte/yw/common/YbcBackupUtil.java

@@ -10,11 +10,13 @@
 import static java.util.stream.Collectors.joining;
 
 import com.fasterxml.jackson.annotation.JsonAlias;
+import com.fasterxml.jackson.annotation.JsonEnumDefaultValue;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.fasterxml.jackson.annotation.JsonTypeInfo.As;
 import com.fasterxml.jackson.annotation.JsonTypeInfo.Id;
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -98,7 +100,10 @@ public enum SnapshotObjectType {
     @EnumValue("NAMESPACE")
     NAMESPACE,
     @EnumValue("TABLE")
-    TABLE;
+    TABLE,
+    @JsonEnumDefaultValue
+    @EnumValue("default_type")
+    DEFAULT_TYPE;
 
     public static class Constants {
       public static final String NAMESPACE = "NAMESPACE";
@@ -159,7 +164,11 @@ public static class SnapshotObjectDetails {
 
       @NotNull
       @Valid
-      @JsonTypeInfo(use = Id.NAME, property = "type", include = As.EXTERNAL_PROPERTY)
+      @JsonTypeInfo(
+          use = Id.NAME,
+          property = "type",
+          include = As.EXTERNAL_PROPERTY,
+          defaultImpl = SnapshotObjectData.class)
       @JsonSubTypes(
           value = {
             @JsonSubTypes.Type(value = TableData.class, name = SnapshotObjectType.Constants.TABLE),
@@ -170,7 +179,7 @@ public static class SnapshotObjectDetails {
       public SnapshotObjectData data;
 
       @JsonIgnoreProperties(ignoreUnknown = true)
-      public abstract static class SnapshotObjectData {
+      public static class SnapshotObjectData {
         @JsonAlias("name")
         @NotNull
         public String snapshotObjectName;
@@ -204,6 +213,8 @@ public static class NamespaceData extends SnapshotObjectData {
    */
   public YbcBackupResponse parseYbcBackupResponse(String metadata) {
     ObjectMapper mapper = new ObjectMapper();
+    // For custom types in Snapshot Info.
+    mapper.enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_USING_DEFAULT_VALUE);
     YbcBackupResponse successMarker = null;
     try {
       successMarker = mapper.readValue(metadata, YbcBackupResponse.class);

managed/src/test/java/com/yugabyte/yw/common/YbcBackupUtilTest.java

@@ -170,12 +170,23 @@ public void testExtractSuccessFileWithoutRegion(String dataFile) throws IOExcept
             .get(1)
             .type
             .equals(YbcBackupUtil.SnapshotObjectType.TABLE));
+    assertTrue(
+        ybcBackupResponse
+            .snapshotObjectDetails
+            .get(2)
+            .type
+            .equals(YbcBackupUtil.SnapshotObjectType.DEFAULT_TYPE));
+
     assertTrue(
         ybcBackupResponse.snapshotObjectDetails.get(0).data
             instanceof YbcBackupResponse.SnapshotObjectDetails.NamespaceData);
     assertTrue(
         ybcBackupResponse.snapshotObjectDetails.get(1).data
             instanceof YbcBackupResponse.SnapshotObjectDetails.TableData);
+    // Verify custom type does not fail
+    assertTrue(
+        ybcBackupResponse.snapshotObjectDetails.get(2).data
+            instanceof YbcBackupResponse.SnapshotObjectDetails.SnapshotObjectData);
   }
 
   @Test

managed/src/test/resources/backup/ybc_success_file_without_regions.json

@@ -26,6 +26,12 @@
           "namespace_id": "000033df000030008000000000000000",
           "namespace_name": "bar"
         }
+      },
+      {
+        "type": "udtype",
+        "data": {
+          "name": "sample_type"
+        }
       }
     ],
     "cloud_store_spec": {