[draft]Branch fs spi phase5 clean master

build.sh

@@ -641,6 +641,14 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
     modules+=("fe-extension-spi")
     modules+=("fe-extension-loader")
     modules+=("fe-core")
+    # Filesystem SPI plugin modules (loaded at runtime as plugins)
+    modules+=("fe-filesystem/fe-filesystem-spi")
+    for _fs_mod in s3 oss cos obs azure hdfs local broker; do
+        if [[ -d "${DORIS_HOME}/fe/fe-filesystem/fe-filesystem-${_fs_mod}" ]]; then
+            modules+=("fe-filesystem/fe-filesystem-${_fs_mod}")
+        fi
+    done
+    unset _fs_mod
     if [[ "${WITH_TDE_DIR}" != "" ]]; then
         modules+=("fe-${WITH_TDE_DIR}")
     fi
@@ -913,6 +921,17 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
     if [[ "${BUILD_JUICEFS}" == "ON" ]]; then
         install -d "${DORIS_OUTPUT}/fe/lib/juicefs"
     fi
+    # Maven Build Cache may restore only the JAR on a cache hit, skipping the
+    # dependency:copy-dependencies execution that populates target/lib/.  Run it
+    # explicitly when needed so the output lib directory is always complete.
+    if [[ ! -d "${DORIS_HOME}/fe/fe-core/target/lib" ]]; then
+        echo "target/lib not found (build cache hit), running dependency:copy-dependencies..."
+        (cd "${DORIS_HOME}/fe" && "${MVN_CMD}" dependency:copy-dependencies \
+            -pl fe-core \
+            -DoutputDirectory="${DORIS_HOME}/fe/fe-core/target/lib" \
+            -DincludeScope=runtime \
+            --no-transfer-progress -q)
+    fi
     cp -r -p "${DORIS_HOME}/fe/fe-core/target/lib"/* "${DORIS_OUTPUT}/fe/lib"/
     cp -r -p "${DORIS_HOME}/fe/fe-core/target/doris-fe.jar" "${DORIS_OUTPUT}/fe/lib"/
     if [[ "${WITH_TDE_DIR}" != "" ]]; then
@@ -952,6 +971,27 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
     mkdir -p "${DORIS_OUTPUT}/fe/plugins/hadoop_conf/"
     mkdir -p "${DORIS_OUTPUT}/fe/plugins/java_extensions/"
 
+    # Deploy filesystem provider plugins as independent plugin directories
+    # Each sub-directory is one storage backend loaded at runtime by FileSystemPluginManager.
+    FS_PLUGIN_DIR="${DORIS_OUTPUT}/fe/plugins/filesystem"
+    for fs_module in s3 azure oss cos obs hdfs local broker; do
+        fs_plugin_target="${FS_PLUGIN_DIR}/${fs_module}"
+        fs_module_dir="${DORIS_HOME}/fe/fe-filesystem/fe-filesystem-${fs_module}"
+        if [ ! -d "${fs_module_dir}" ]; then
+            continue
+        fi
+        mkdir -p "${fs_plugin_target}"
+        # Copy main JAR
+        cp -f "${fs_module_dir}/target/doris-fe-filesystem-${fs_module}"*.jar "${fs_plugin_target}/" 2>/dev/null || true
+        # Copy transitive dependencies flat into the same directory, excluding spi JARs already on FE classpath
+        (cd "${DORIS_HOME}/fe" && "${MVN_CMD}" dependency:copy-dependencies \
+            -pl "fe-filesystem/fe-filesystem-${fs_module}" \
+            -DoutputDirectory="${fs_plugin_target}" \
+            -DexcludeArtifactIds="fe-filesystem-spi,fe-extension-spi" \
+            --no-transfer-progress -q 2>/dev/null) || true
+    done
+    unset FS_PLUGIN_DIR fs_module fs_plugin_target fs_module_dir
+
     if [ "${TARGET_SYSTEM}" = "Darwin" ] || [ "${TARGET_SYSTEM}" = "Linux" ]; then
       mkdir -p "${DORIS_OUTPUT}/fe/arthas"
       rm -rf "${DORIS_OUTPUT}/fe/arthas/*"

conf/fe.conf

@@ -71,3 +71,8 @@ sys_log_mode = ASYNC
 # qe_max_connection = 1024
 # qe_query_timeout_second = 300
 # qe_slow_log_ms = 5000
+
+# Directory containing filesystem provider plugin subdirectories.
+# Each subdirectory is one storage backend (e.g., s3/, hdfs/, azure/).
+# If empty, only classpath-based built-in providers are used.
+# filesystem_plugin_root = ${DORIS_HOME}/plugins/filesystem

fe/.mvn/maven-build-cache-config.xml

@@ -30,6 +30,10 @@ under the License.
             <glob>{*.java,*.g4,*.xml,*.properties}</glob>
             <includes>
                 <include>src/</include>
+                <!-- Checkstyle configuration files affect all modules' style checks.
+                     Include them globally so any change to checkstyle rules invalidates
+                     all cached checkstyle results. -->
+                <include glob="{*.xml,*.txt}" recursive="true">check/checkstyle</include>
                 <!-- External source directories under gensrc/ are referenced by plugins via
                      Maven properties (e.g. ${project.parent.basedir}/../gensrc/thrift).
                      The cache extension's tagScanConfig cannot resolve these Maven properties,
@@ -44,4 +48,16 @@ under the License.
             </includes>
         </global>
     </input>
+    <executionControl>
+        <runAlways>
+            <!-- Checkstyle is a quality gate that must always execute; never use a cached result. -->
+            <executions>
+                <execution artifactId="maven-checkstyle-plugin">
+                    <execIds>
+                        <execId>validate</execId>
+                    </execIds>
+                </execution>
+            </executions>
+        </runAlways>
+    </executionControl>
 </cache>

fe/fe-common/src/main/java/org/apache/doris/common/Config.java

@@ -3424,6 +3424,11 @@ public static int metaServiceRpcRetryTimes() {
     @ConfField(description = {"Security plugin directory."})
     public static String security_plugins_dir = EnvUtils.getDorisHome() + "/plugins/security";
 
+    @ConfField(description = {"Directory containing filesystem provider plugin subdirectories. "
+            + "Each subdirectory is one storage backend (e.g., s3/, hdfs/, azure/). "
+            + "If empty, only classpath-based built-in providers are used (test/dev mode)."})
+    public static String filesystem_plugin_root = EnvUtils.getDorisHome() + "/plugins/filesystem";
+
     @ConfField(description = {"Authorization plugin configuration file path. Must be under DORIS_HOME. "
             + "Default is conf/authorization.conf."})
     public static String authorization_config_file_path = "/conf/authorization.conf";

fe/fe-core/pom.xml

@@ -69,6 +69,22 @@ under the License.
             <artifactId>fe-common</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <!-- fe-filesystem-spi: compile-time SPI interfaces for FileSystemFactory -->
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>fe-filesystem-spi</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <!-- fe-filesystem impl modules: runtime dependencies removed in Phase 4 P4.1.
+             Providers are now loaded from plugins/filesystem/ directory at startup
+             via FileSystemPluginManager + DirectoryPluginRuntimeManager. -->
+        <!-- fe-filesystem-local: test-scope only, used for unit testing FileSystemTransferUtil etc. -->
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>fe-filesystem-local</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>${project.groupId}</groupId>
             <artifactId>fe-thrift</artifactId>

fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java

@@ -39,8 +39,6 @@
 import org.apache.doris.common.util.TimeUtils;
 import org.apache.doris.common.util.Util;
 import org.apache.doris.datasource.property.storage.StorageProperties;
-import org.apache.doris.fs.FileSystemFactory;
-import org.apache.doris.fs.remote.RemoteFileSystem;
 import org.apache.doris.info.TableNameInfo;
 import org.apache.doris.info.TableRefInfo;
 import org.apache.doris.nereids.trees.plans.commands.BackupCommand;
@@ -208,11 +206,9 @@ public void createRepository(CreateRepositoryCommand command) throws DdlExceptio
                     "broker does not exist: " + command.getBrokerName());
         }
 
-        RemoteFileSystem fileSystem;
-        fileSystem = FileSystemFactory.get(command.getStorageProperties());
         long repoId = env.getNextId();
         Repository repo = new Repository(repoId, command.getName(), command.isReadOnly(), command.getLocation(),
-                fileSystem);
+                command.getStorageProperties());
 
         Status st = repoMgr.addAndInitRepoIfNotExist(repo, false);
         if (!st.ok()) {
@@ -244,12 +240,10 @@ public void alterRepository(String repoName, Map<String, String> newProps)
             }
             // Merge new properties with the existing repository's properties
             Map<String, String> mergedProps = mergeProperties(oldRepo, newProps);
-            // Create new remote file system with merged properties
-            RemoteFileSystem fileSystem = FileSystemFactory.get(StorageProperties.createPrimary(mergedProps));
-            // Create new Repository instance with updated file system
+            // Create new Repository instance with merged properties
             Repository newRepo = new Repository(
                     oldRepo.getId(), oldRepo.getName(), oldRepo.isReadOnly(),
-                    oldRepo.getLocation(), fileSystem
+                    oldRepo.getLocation(), StorageProperties.createPrimary(mergedProps)
             );
             // Verify the repository can be connected with new settings
             if (!newRepo.ping()) {
@@ -277,7 +271,7 @@ public void alterRepository(String repoName, Map<String, String> newProps)
      */
     private Map<String, String> mergeProperties(Repository repo, Map<String, String> newProps) {
         // General case: just override old props with new ones
-        Map<String, String> combined = new HashMap<>(repo.getRemoteFileSystem().getProperties());
+        Map<String, String> combined = new HashMap<>(repo.getFileSystemDescriptor().getProperties());
         combined.putAll(newProps);
         return combined;
     }

fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java

@@ -437,7 +437,7 @@ public synchronized Status updateRepo(Repository repo) {
                     continue;
                 }
                 ((UploadTask) task).updateBrokerProperties(
-                        repo.getRemoteFileSystem().getStorageProperties().getBackendConfigProperties());
+                        repo.getFileSystemDescriptor().getBackendConfigProperties());
                 AgentTaskQueue.updateTask(beId, TTaskType.UPLOAD, signature, task);
             }
             LOG.info("finished to update upload job properties. {}", this);
@@ -821,8 +821,8 @@ private void uploadSnapshot() {
                 long signature = env.getNextId();
                 UploadTask task = new UploadTask(null, beId, signature, jobId, dbId, srcToDest,
                         brokers.get(0),
-                        repo.getRemoteFileSystem().getStorageProperties().getBackendConfigProperties(),
-                        repo.getRemoteFileSystem().getStorageType(), repo.getLocation());
+                        repo.getFileSystemDescriptor().getBackendConfigProperties(),
+                        repo.getFileSystemDescriptor().getThriftStorageType(), repo.getLocation());
                 batchTask.addTask(task);
                 unfinishedTaskIds.put(signature, beId);
             }