Skip to content

Branch fs spi review #62024

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 2 commits into from
Apr 2, 2026
Merged

Branch fs spi review #62024

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a42026b
[fix](fe) Fix multiple bugs and improve SPI design in fe-filesystem m…
morningman Apr 1, 2026
d60105b
[refactor](fe) Split fe-filesystem-spi into api and spi modules
morningman Apr 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -641,7 +641,8 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
modules+=("fe-extension-spi")
modules+=("fe-extension-loader")
modules+=("fe-core")
# Filesystem SPI plugin modules (loaded at runtime as plugins)
# Filesystem API and SPI plugin modules (loaded at runtime as plugins)
modules+=("fe-filesystem/fe-filesystem-api")
modules+=("fe-filesystem/fe-filesystem-spi")
for _fs_mod in s3 oss cos obs azure hdfs local broker; do
if [[ -d "${DORIS_HOME}/fe/fe-filesystem/fe-filesystem-${_fs_mod}" ]]; then
Expand Down Expand Up @@ -987,7 +988,7 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
(cd "${DORIS_HOME}/fe" && "${MVN_CMD}" dependency:copy-dependencies \
-pl "fe-filesystem/fe-filesystem-${fs_module}" \
-DoutputDirectory="${fs_plugin_target}" \
-DexcludeArtifactIds="fe-filesystem-spi,fe-extension-spi" \
-DexcludeArtifactIds="fe-filesystem-api,fe-filesystem-spi,fe-extension-spi" \
--no-transfer-progress -q 2>/dev/null) || true
done
unset FS_PLUGIN_DIR fs_module fs_plugin_target fs_module_dir
Expand Down
9 changes: 8 additions & 1 deletion fe/fe-core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,14 @@ under the License.
<artifactId>fe-common</artifactId>
<version>${project.version}</version>
</dependency>
<!-- fe-filesystem-spi: compile-time SPI interfaces for FileSystemFactory -->
<!-- fe-filesystem-api: consumer-facing filesystem API (FileSystem, Location, FileEntry, etc.) -->
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>fe-filesystem-api</artifactId>
<version>${project.version}</version>
</dependency>
<!-- fe-filesystem-spi: provider contracts (ObjFileSystem, ObjStorage, etc.) —
needed by S3Resource which directly invokes ObjStorage for connectivity validation. -->
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>fe-filesystem-spi</artifactId>
Expand Down
30 changes: 15 additions & 15 deletions fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,11 @@
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.datasource.property.storage.BrokerProperties;
import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.filesystem.spi.DorisInputFile;
import org.apache.doris.filesystem.spi.DorisOutputFile;
import org.apache.doris.filesystem.spi.FileEntry;
import org.apache.doris.filesystem.spi.FileIterator;
import org.apache.doris.filesystem.spi.Location;
import org.apache.doris.filesystem.DorisInputFile;
import org.apache.doris.filesystem.DorisOutputFile;
import org.apache.doris.filesystem.FileEntry;
import org.apache.doris.filesystem.FileIterator;
import org.apache.doris.filesystem.Location;
import org.apache.doris.foundation.fs.FsStorageType;
import org.apache.doris.fs.FileSystemDescriptor;
import org.apache.doris.fs.FileSystemFactory;
Expand Down Expand Up @@ -139,7 +139,7 @@ public class Repository implements Writable, GsonPostProcessable {

/** SPI filesystem for I/O operations; transient — rebuilt in {@link #gsonPostProcess()}.
* Null for BROKER repositories, which resolve a live broker endpoint per I/O call. */
private transient org.apache.doris.filesystem.spi.FileSystem spiFs;
private transient org.apache.doris.filesystem.FileSystem spiFs;

public FileSystemDescriptor getFileSystemDescriptor() {
return fileSystemDescriptor;
Expand Down Expand Up @@ -275,7 +275,7 @@ public String getErrorMsg() {
* instance that <b>must</b> be closed by calling {@link #releaseSpiFs}.</li>
* </ul>
*/
private org.apache.doris.filesystem.spi.FileSystem acquireSpiFs() throws IOException {
private org.apache.doris.filesystem.FileSystem acquireSpiFs() throws IOException {
if (spiFs != null) {
return spiFs;
}
Expand All @@ -298,7 +298,7 @@ private org.apache.doris.filesystem.spi.FileSystem acquireSpiFs() throws IOExcep
* Releases an SPI FileSystem acquired via {@link #acquireSpiFs}.
* Closes broker per-call instances; leaves the shared non-broker instance open.
*/
private void releaseSpiFs(org.apache.doris.filesystem.spi.FileSystem fs) {
private void releaseSpiFs(org.apache.doris.filesystem.FileSystem fs) {
if (fs != spiFs) {
try {
fs.close();
Expand All @@ -309,7 +309,7 @@ private void releaseSpiFs(org.apache.doris.filesystem.spi.FileSystem fs) {
}

/** Uploads a local file to a remote path via the SPI filesystem. */
private static void spiUploadFile(org.apache.doris.filesystem.spi.FileSystem fs,
private static void spiUploadFile(org.apache.doris.filesystem.FileSystem fs,
String localFilePath, String remotePath) throws IOException {
DorisOutputFile outputFile = fs.newOutputFile(Location.of(remotePath));
try (java.io.InputStream in = Files.newInputStream(Paths.get(localFilePath));
Expand Down Expand Up @@ -338,7 +338,7 @@ public Status initRepository() {
}

String repoInfoFilePath = assembleRepoInfoFilePath();
org.apache.doris.filesystem.spi.FileSystem fs;
org.apache.doris.filesystem.FileSystem fs;
try {
fs = acquireSpiFs();
} catch (IOException e) {
Expand Down Expand Up @@ -460,7 +460,7 @@ public boolean ping() {
String path = location + "/" + joinPrefix(PREFIX_REPO, name) + "/" + FILE_REPO_INFO;
try {
URI checkUri = new URI(path);
org.apache.doris.filesystem.spi.FileSystem fs = acquireSpiFs();
org.apache.doris.filesystem.FileSystem fs = acquireSpiFs();
try {
boolean exists = fs.exists(Location.of(checkUri.normalize().toString()));
if (!exists) {
Expand Down Expand Up @@ -489,7 +489,7 @@ public Status listSnapshots(List<String> snapshotNames) {
// eg. __palo_repository_repo_name/__ss_*
String listPath = Joiner.on(PATH_DELIMITER).join(location, joinPrefix(PREFIX_REPO, name), PREFIX_SNAPSHOT_DIR)
+ "*";
org.apache.doris.filesystem.spi.FileSystem fs;
org.apache.doris.filesystem.FileSystem fs;
try {
fs = acquireSpiFs();
} catch (IOException e) {
Expand Down Expand Up @@ -610,7 +610,7 @@ public Status upload(String localFilePath, String remoteFilePath) {
Preconditions.checkState(!Strings.isNullOrEmpty(md5sum));
String finalRemotePath = assembleFileNameWithSuffix(remoteFilePath, md5sum);

org.apache.doris.filesystem.spi.FileSystem fs;
org.apache.doris.filesystem.FileSystem fs;
try {
fs = acquireSpiFs();
} catch (IOException e) {
Expand Down Expand Up @@ -647,7 +647,7 @@ public Status upload(String localFilePath, String remoteFilePath) {

// remoteFilePath must be a file(not dir) and does not contain checksum
public Status download(String remoteFilePath, String localFilePath) {
org.apache.doris.filesystem.spi.FileSystem fs;
org.apache.doris.filesystem.FileSystem fs;
try {
fs = acquireSpiFs();
} catch (IOException e) {
Expand Down Expand Up @@ -822,7 +822,7 @@ private List<String> getSnapshotInfo(String snapshotName, String timestamp) {
LOG.debug("assemble infoFilePath: {}, snapshot: {}", infoFilePath, snapshotName);
}
try {
org.apache.doris.filesystem.spi.FileSystem fs = acquireSpiFs();
org.apache.doris.filesystem.FileSystem fs = acquireSpiFs();
List<FileEntry> results;
try {
results = fs.listFiles(Location.of(infoFilePath + "*"));
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/catalog/AzureResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ protected static void pingAzure(String bucketName, String rootPath,
Arrays.fill(contentData, (byte) 'A');

try {
org.apache.doris.filesystem.spi.FileSystem fileSystem =
org.apache.doris.filesystem.FileSystem fileSystem =
FileSystemFactory.getFileSystem(newProperties);
Preconditions.checkState(fileSystem instanceof ObjFileSystem,
"Expected object-storage filesystem for Azure resource");
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
import org.apache.doris.common.security.authentication.AuthenticationConfig;
import org.apache.doris.common.util.DatasourcePrintableMap;
import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.filesystem.spi.FileSystem;
import org.apache.doris.filesystem.spi.Location;
import org.apache.doris.filesystem.FileSystem;
import org.apache.doris.filesystem.Location;
import org.apache.doris.fs.FileSystemFactory;

import com.google.common.base.Preconditions;
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ protected static void pingS3(String bucketName, String rootPath, Map<String, Str
Arrays.fill(contentData, (byte) 'A');

try {
org.apache.doris.filesystem.spi.FileSystem fileSystem =
org.apache.doris.filesystem.FileSystem fileSystem =
FileSystemFactory.getFileSystem(newProperties);
Preconditions.checkState(fileSystem instanceof ObjFileSystem,
"Expected object-storage filesystem for S3 resource");
Expand Down
6 changes: 3 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/common/plugin/CloudPluginDownloader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@

import org.apache.doris.cloud.proto.Cloud;
import org.apache.doris.cloud.rpc.MetaServiceProxy;
import org.apache.doris.filesystem.spi.DorisInputFile;
import org.apache.doris.filesystem.spi.Location;
import org.apache.doris.filesystem.DorisInputFile;
import org.apache.doris.filesystem.Location;
import org.apache.doris.fs.FileSystemFactory;
import org.apache.doris.service.FrontendOptions;

Expand Down Expand Up @@ -133,7 +133,7 @@ private static String doDownload(Cloud.ObjectStoreInfoPB objInfo, String remoteP

// Download via SPI FileSystem
Map<String, String> properties = buildProperties(objInfo);
org.apache.doris.filesystem.spi.FileSystem fileSystem =
org.apache.doris.filesystem.FileSystem fileSystem =
FileSystemFactory.getFileSystem(properties);
DorisInputFile inputFile = fileSystem.newInputFile(Location.of(remotePath));
try (InputStream in = inputFile.newStream()) {
Expand Down
6 changes: 3 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.hive.HiveExternalMetaCache;
import org.apache.doris.datasource.property.storage.BrokerProperties;
import org.apache.doris.filesystem.spi.FileEntry;
import org.apache.doris.filesystem.spi.FileSystem;
import org.apache.doris.filesystem.spi.Location;
import org.apache.doris.filesystem.FileEntry;
import org.apache.doris.filesystem.FileSystem;
import org.apache.doris.filesystem.Location;
import org.apache.doris.fs.FileSystemFactory;
import org.apache.doris.service.FrontendOptions;
import org.apache.doris.thrift.TBrokerCloseReaderRequest;
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.property.storage.AzurePropertyUtils;
import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.filesystem.spi.FileSystemType;
import org.apache.doris.filesystem.FileSystemType;
import org.apache.doris.foundation.property.StoragePropertiesException;
import org.apache.doris.fs.SchemaTypeMapper;
import org.apache.doris.thrift.TFileType;
Expand Down
10 changes: 5 additions & 5 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/hive/AcidUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@
import org.apache.doris.datasource.hive.AcidInfo.DeleteDeltaInfo;
import org.apache.doris.datasource.hive.HiveExternalMetaCache.FileCacheValue;
import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.filesystem.spi.FileEntry;
import org.apache.doris.filesystem.spi.FileSystem;
import org.apache.doris.filesystem.spi.FileSystemTransferUtil;
import org.apache.doris.filesystem.spi.Location;
import org.apache.doris.filesystem.FileEntry;
import org.apache.doris.filesystem.FileSystem;
import org.apache.doris.filesystem.FileSystemTransferUtil;
import org.apache.doris.filesystem.Location;

import lombok.EqualsAndHashCode;
import lombok.Getter;
Expand Down Expand Up @@ -419,7 +419,7 @@ && isValidBase(fileSystem, dirPath, base, validWriteIdList)) {
return fileCacheValue;
}

private static String locationName(org.apache.doris.filesystem.spi.Location location) {
private static String locationName(org.apache.doris.filesystem.Location location) {
String uri = location.uri();
int idx = uri.lastIndexOf('/');
return idx >= 0 ? uri.substring(idx + 1) : uri;
Expand Down
8 changes: 4 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@
import org.apache.doris.common.profile.SummaryProfile;
import org.apache.doris.datasource.NameMapping;
import org.apache.doris.datasource.statistics.CommonStatistics;
import org.apache.doris.filesystem.spi.FileEntry;
import org.apache.doris.filesystem.spi.FileSystem;
import org.apache.doris.filesystem.spi.FileSystemUtil;
import org.apache.doris.filesystem.spi.Location;
import org.apache.doris.filesystem.FileEntry;
import org.apache.doris.filesystem.FileSystem;
import org.apache.doris.filesystem.FileSystemUtil;
import org.apache.doris.filesystem.Location;
import org.apache.doris.filesystem.spi.ObjFileSystem;
import org.apache.doris.foundation.util.PathUtils;
import org.apache.doris.fs.SpiSwitchingFileSystem;
Expand Down
10 changes: 5 additions & 5 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,11 @@
import org.apache.doris.datasource.metacache.CacheSpec;
import org.apache.doris.datasource.metacache.MetaCacheEntry;
import org.apache.doris.datasource.metacache.MetaCacheEntryDef;
import org.apache.doris.filesystem.spi.BlockInfo;
import org.apache.doris.filesystem.spi.FileEntry;
import org.apache.doris.filesystem.spi.FileSystem;
import org.apache.doris.filesystem.spi.FileSystemIOException;
import org.apache.doris.filesystem.spi.RemoteIterator;
import org.apache.doris.filesystem.BlockInfo;
import org.apache.doris.filesystem.FileEntry;
import org.apache.doris.filesystem.FileSystem;
import org.apache.doris.filesystem.FileSystemIOException;
import org.apache.doris.filesystem.RemoteIterator;
import org.apache.doris.fs.DirectoryLister;
import org.apache.doris.fs.FileSystemCache;
import org.apache.doris.fs.FileSystemDirectoryLister;
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import org.apache.doris.common.UserException;
import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.statistics.CommonStatistics;
import org.apache.doris.filesystem.spi.FileSystem;
import org.apache.doris.filesystem.FileSystem;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.qe.ConnectContext;

Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/fileio/DelegateFileIO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
package org.apache.doris.datasource.iceberg.fileio;

import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.filesystem.spi.FileSystem;
import org.apache.doris.filesystem.spi.Location;
import org.apache.doris.filesystem.FileSystem;
import org.apache.doris.filesystem.Location;
import org.apache.doris.fs.FileSystemFactory;

import org.apache.iceberg.DataFile;
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/fileio/DelegateInputFile.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

package org.apache.doris.datasource.iceberg.fileio;

import org.apache.doris.filesystem.spi.DorisInputFile;
import org.apache.doris.filesystem.DorisInputFile;

import org.apache.iceberg.exceptions.NotFoundException;
import org.apache.iceberg.io.InputFile;
Expand Down
6 changes: 3 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/fileio/DelegateOutputFile.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@

package org.apache.doris.datasource.iceberg.fileio;

import org.apache.doris.filesystem.spi.DorisOutputFile;
import org.apache.doris.filesystem.spi.FileSystem;
import org.apache.doris.filesystem.spi.Location;
import org.apache.doris.filesystem.DorisOutputFile;
import org.apache.doris.filesystem.FileSystem;
import org.apache.doris.filesystem.Location;

import com.google.common.io.CountingOutputStream;
import org.apache.iceberg.io.InputFile;
Expand Down
2 changes: 1 addition & 1 deletion ...src/main/java/org/apache/doris/datasource/iceberg/fileio/DelegateSeekableInputStream.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

package org.apache.doris.datasource.iceberg.fileio;

import org.apache.doris.filesystem.spi.DorisInputStream;
import org.apache.doris.filesystem.DorisInputStream;

import org.apache.iceberg.io.SeekableInputStream;

Expand Down
8 changes: 4 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/fs/DirectoryLister.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@
package org.apache.doris.fs;

import org.apache.doris.catalog.TableIf;
import org.apache.doris.filesystem.spi.FileEntry;
import org.apache.doris.filesystem.spi.FileSystem;
import org.apache.doris.filesystem.spi.FileSystemIOException;
import org.apache.doris.filesystem.spi.RemoteIterator;
import org.apache.doris.filesystem.FileEntry;
import org.apache.doris.filesystem.FileSystem;
import org.apache.doris.filesystem.FileSystemIOException;
import org.apache.doris.filesystem.RemoteIterator;

public interface DirectoryLister {
RemoteIterator<FileEntry> listFiles(FileSystem fs, boolean recursive, TableIf table, String location)
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import org.apache.doris.common.CacheFactory;
import org.apache.doris.common.Config;
import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.doris.filesystem.spi.FileSystem;
import org.apache.doris.filesystem.FileSystem;

import com.github.benmanes.caffeine.cache.LoadingCache;

Expand Down
12 changes: 6 additions & 6 deletions fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemDirectoryLister.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@
package org.apache.doris.fs;

import org.apache.doris.catalog.TableIf;
import org.apache.doris.filesystem.spi.FileEntry;
import org.apache.doris.filesystem.spi.FileSystem;
import org.apache.doris.filesystem.spi.FileSystemIOException;
import org.apache.doris.filesystem.spi.FileSystemTransferUtil;
import org.apache.doris.filesystem.spi.RemoteIterator;
import org.apache.doris.filesystem.spi.SimpleRemoteIterator;
import org.apache.doris.filesystem.FileEntry;
import org.apache.doris.filesystem.FileSystem;
import org.apache.doris.filesystem.FileSystemIOException;
import org.apache.doris.filesystem.FileSystemTransferUtil;
import org.apache.doris.filesystem.RemoteIterator;
import org.apache.doris.filesystem.SimpleRemoteIterator;

import java.io.IOException;
import java.util.List;
Expand Down
Loading
Loading