[server] Fix CorruptIndexException for cached remote log index

fluss-common/src/main/java/com/alibaba/fluss/exception/CorruptIndexException.java → fluss-server/src/main/java/com/alibaba/fluss/server/exception/CorruptIndexException.java

@@ -5,7 +5,7 @@
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *      http://www.apache.org/licenses/LICENSE-2.0
+ *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.alibaba.fluss.exception;
+package com.alibaba.fluss.server.exception;
 
 import com.alibaba.fluss.annotation.PublicEvolving;
 
@@ -25,7 +25,7 @@
  * @since 0.1
  */
 @PublicEvolving
-public class CorruptIndexException extends RetriableException {
+public class CorruptIndexException extends RuntimeException {
     public CorruptIndexException(String message) {
         super(message);
     }

fluss-server/src/main/java/com/alibaba/fluss/server/log/AbstractIndex.java

@@ -16,8 +16,8 @@
 
 package com.alibaba.fluss.server.log;
 
-import com.alibaba.fluss.exception.CorruptIndexException;
 import com.alibaba.fluss.exception.IndexOffsetOverflowException;
+import com.alibaba.fluss.server.exception.CorruptIndexException;
 import com.alibaba.fluss.utils.FileUtils;
 import com.alibaba.fluss.utils.IOUtils;
 import com.alibaba.fluss.utils.OperatingSystem;

fluss-server/src/main/java/com/alibaba/fluss/server/log/OffsetIndex.java

@@ -16,9 +16,9 @@
 
 package com.alibaba.fluss.server.log;
 
-import com.alibaba.fluss.exception.CorruptIndexException;
 import com.alibaba.fluss.exception.IndexOffsetOverflowException;
 import com.alibaba.fluss.exception.InvalidOffsetException;
+import com.alibaba.fluss.server.exception.CorruptIndexException;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

fluss-server/src/main/java/com/alibaba/fluss/server/log/TimeIndex.java

@@ -16,8 +16,8 @@
 
 package com.alibaba.fluss.server.log;
 
-import com.alibaba.fluss.exception.CorruptIndexException;
 import com.alibaba.fluss.exception.InvalidOffsetException;
+import com.alibaba.fluss.server.exception.CorruptIndexException;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

fluss-server/src/main/java/com/alibaba/fluss/server/log/remote/RemoteLogIndexCache.java

@@ -17,10 +17,10 @@
 package com.alibaba.fluss.server.log.remote;
 
 import com.alibaba.fluss.annotation.VisibleForTesting;
-import com.alibaba.fluss.exception.CorruptIndexException;
 import com.alibaba.fluss.exception.FlussRuntimeException;
 import com.alibaba.fluss.exception.RemoteStorageException;
 import com.alibaba.fluss.remote.RemoteLogSegment;
+import com.alibaba.fluss.server.exception.CorruptIndexException;
 import com.alibaba.fluss.server.log.OffsetIndex;
 import com.alibaba.fluss.server.log.OffsetPosition;
 import com.alibaba.fluss.server.log.StorageAction;
@@ -406,16 +406,27 @@ private void init() throws IOException {
                     if (Files.exists(offsetIndexFile.toPath())
                             && Files.exists(timestampIndexFile.toPath())) {
                         long offset = offsetFromRemoteIndexCacheFileName(indexFileName);
-                        OffsetIndex offsetIndex =
-                                new OffsetIndex(offsetIndexFile, offset, Integer.MAX_VALUE, false);
-                        offsetIndex.sanityCheck();
-
-                        TimeIndex timeIndex =
-                                new TimeIndex(timestampIndexFile, offset, Integer.MAX_VALUE, false);
-                        timeIndex.sanityCheck();
-
-                        Entry entry = new Entry(offsetIndex, timeIndex);
-                        internalCache.put(remoteSegmentId, entry);
+                        try {
+                            OffsetIndex offsetIndex =
+                                    new OffsetIndex(
+                                            offsetIndexFile, offset, Integer.MAX_VALUE, false);
+                            offsetIndex.sanityCheck();
+
+                            TimeIndex timeIndex =
+                                    new TimeIndex(
+                                            timestampIndexFile, offset, Integer.MAX_VALUE, false);
+                            timeIndex.sanityCheck();
+
+                            Entry entry = new Entry(offsetIndex, timeIndex);
+                            internalCache.put(remoteSegmentId, entry);
+                        } catch (CorruptIndexException e) {
+                            LOG.debug(
+                                    "Remote offset/time log index is corrupt, delete corrupt index.",
+                                    e);
+                            // let's delete offset index & time index
+                            Files.deleteIfExists(offsetIndexFile.toPath());
+                            Files.deleteIfExists(timestampIndexFile.toPath());
+                        }
                     } else {
                         // Delete all of them if any one of those indexes is not available for a
                         // specific segment id.

fluss-server/src/test/java/com/alibaba/fluss/server/log/TimeIndexTest.java

@@ -16,8 +16,8 @@
 
 package com.alibaba.fluss.server.log;
 
-import com.alibaba.fluss.exception.CorruptIndexException;
 import com.alibaba.fluss.exception.InvalidOffsetException;
+import com.alibaba.fluss.server.exception.CorruptIndexException;
 
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;

fluss-server/src/test/java/com/alibaba/fluss/server/log/remote/RemoteLogIndexCacheTest.java

@@ -31,7 +31,10 @@
 import org.junit.jupiter.params.provider.ValueSource;
 
 import java.io.File;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
 import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
 import java.util.Arrays;
 import java.util.UUID;
 
@@ -106,6 +109,34 @@ void testFetchTimeIndexFromRemoteLogStorage(boolean partitionTable) throws Excep
         assertThat(offsetPosition.getOffset()).isEqualTo(resultOffset);
     }
 
+    @ParameterizedTest
+    @ValueSource(booleans = {true, false})
+    void testInitWithCorruptIndex(boolean partitionTable) throws Exception {
+        LogTablet logTablet = makeLogTabletAndAddSegments(partitionTable);
+        // 1. first upload one segment to remote.
+        RemoteLogSegment remoteLogSegment = copyLogSegmentToRemote(logTablet, remoteLogStorage, 0);
+
+        rlIndexCache = new RemoteLogIndexCache(1024 * 1024L, remoteLogStorage, tempDir);
+        File file = rlIndexCache.getIndexEntry(remoteLogSegment).timeIndex().file();
+        // mock corrupt index
+        try (FileChannel fileChannel = FileChannel.open(file.toPath(), StandardOpenOption.APPEND)) {
+            for (int i = 0; i < 12; i++) {
+                fileChannel.write(ByteBuffer.wrap(new byte[] {0}));
+            }
+        }
+
+        // re-initialize index cache and lookup offset
+        rlIndexCache = new RemoteLogIndexCache(1024 * 1024L, remoteLogStorage, tempDir);
+        TimeIndex timeIndex = rlIndexCache.getIndexEntry(remoteLogSegment).timeIndex();
+        TimestampOffset timestampOffset = timeIndex.entry(0);
+
+        OffsetIndex offsetIndex = rlIndexCache.getIndexEntry(remoteLogSegment).offsetIndex();
+        OffsetPosition offsetPosition = offsetIndex.lookup(timestampOffset.offset);
+        long resultOffset =
+                rlIndexCache.lookupOffsetForTimestamp(remoteLogSegment, timestampOffset.timestamp);
+        assertThat(offsetPosition.getOffset()).isEqualTo(resultOffset);
+    }
+
     @ParameterizedTest
     @ValueSource(booleans = {true, false})
     void testPositionForNonExistingIndexFromRemoteStorage(boolean partitionTable) throws Exception {