feat(controller): add file system support for StorageAccessService

server/controller/pom.xml

@@ -155,12 +155,17 @@
             <artifactId>junit-jupiter-engine</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
     <build>
         <plugins>
             <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
-                <version>3.0.0-M7</version>
             </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
@@ -229,12 +234,10 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
-                <version>3.8.1</version>
             </plugin>
             <plugin>
                 <groupId>com.github.os72</groupId>
                 <artifactId>protoc-jar-maven-plugin</artifactId>
-                <version>3.11.4</version>
                 <executions>
                     <execution>
                         <phase>generate-sources</phase>

server/controller/src/main/resources/application.yaml

@@ -34,6 +34,7 @@ sw:
       job-template-path: ${SW_K8S_JOB_TEMPLATE_PATH:}
   storage:
     path-prefix: ${SW_STORAGE_PREFIX:StarWhale}
+    fs-root-dir: ${SW_STORAGE_FS_ROOT_DIR:/usr/local/starwhale}
     s3-config:
       bucket: ${SW_STORAGE_BUCKET:starwhale}
       accessKey: ${SW_STORAGE_ACCESSKEY:starwhale}

server/controller/src/test/java/ai/starwhale/mlops/objectstore/impl/FileIteratorTest.java

@@ -1,3 +1,18 @@
+/*
+ * Copyright 2022 Starwhale, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package ai.starwhale.mlops.objectstore.impl;
 
 import org.junit.jupiter.api.BeforeEach;

server/controller/src/test/java/ai/starwhale/mlops/objectstore/impl/FileSystemObjectStoreTest.java

@@ -1,3 +1,18 @@
+/*
+ * Copyright 2022 Starwhale, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package ai.starwhale.mlops.objectstore.impl;
 
 import ai.starwhale.mlops.memory.SwBufferManager;

server/pom.xml

@@ -26,6 +26,11 @@
         <pagehelper.starter.version>1.4.1</pagehelper.starter.version>
         <guava.version>31.1-jre</guava.version>
         <json-flattener.version>0.13.0</json-flattener.version>
+        <junit.version>5.8.2</junit.version>
+        <hamcrest.version>2.2</hamcrest.version>
+        <maven-surefile-plugin.version>3.0.0-M7</maven-surefile-plugin.version>
+        <maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version>
+        <protoc-jar-maven-plugin.version>3.11.4</protoc-jar-maven-plugin.version>
         <starwhale.version>0.1.0-SNAPSHOT</starwhale.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
@@ -125,7 +130,16 @@
                 <artifactId>client-java</artifactId>
                 <version>16.0.0</version>
             </dependency>
-
+            <dependency>
+                <groupId>org.junit.jupiter</groupId>
+                <artifactId>junit-jupiter-engine</artifactId>
+                <version>${junit.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.hamcrest</groupId>
+                <artifactId>hamcrest</artifactId>
+                <version>${hamcrest.version}</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 
@@ -276,6 +290,7 @@
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-compiler-plugin</artifactId>
+                    <version>${maven-compiler-plugin.version}</version>
                     <configuration>
                         <parameters>true</parameters>
                     </configuration>
@@ -423,6 +438,16 @@
                         </execution>
                     </executions>
                 </plugin>
+                <plugin>
+                    <groupId>org.apache.maven.plugins</groupId>
+                    <artifactId>maven-surefire-plugin</artifactId>
+                    <version>${maven-surefile-plugin.version}</version>
+                </plugin>
+                <plugin>
+                    <groupId>com.github.os72</groupId>
+                    <artifactId>protoc-jar-maven-plugin</artifactId>
+                    <version>${protoc-jar-maven-plugin.version}</version>
+                </plugin>
             </plugins>
         </pluginManagement>
         <plugins>

server/storage-access-layer/pom.xml

@@ -7,8 +7,8 @@
   -->
 
 <project xmlns="http://maven.apache.org/POM/4.0.0"
-  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
         <artifactId>server</artifactId>
         <groupId>ai.starwhale</groupId>
@@ -36,6 +36,20 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-autoconfigure</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-engine</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <dependencyManagement>
@@ -50,5 +64,18 @@
         </dependencies>
     </dependencyManagement>
 
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
 
 </project>

server/storage-access-layer/src/main/java/ai/starwhale/mlops/storage/fs/FileIterator.java

@@ -0,0 +1,109 @@
+/*
+ * Copyright 2022 Starwhale, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ai.starwhale.mlops.storage.fs;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+
+public class FileIterator implements Iterator<String> {
+    private final List<List<File>> stack = new ArrayList<>();
+    private final File rootDir;
+    private String next;
+
+    public FileIterator(File rootDir, String prefix) {
+        if (!rootDir.isDirectory()) {
+            throw new IllegalArgumentException(rootDir + " not found or is not a directory");
+        }
+        this.rootDir = rootDir;
+        var root = rootDir;
+        var path = prefix.split("/", -1);
+        if (path.length > 1) {
+            for (int i = 0; i < path.length - 1; ++i) {
+                root = new File(root, path[i]);
+                if (!root.isDirectory()) {
+                    return;
+                }
+            }
+            prefix = path[path.length - 1];
+        }
+        var candidates = new ArrayList<File>();
+        for (var fn : Objects.requireNonNull(root.list())) {
+            if (fn.startsWith(prefix)) {
+                candidates.add(new File(root, fn));
+            }
+        }
+        if (candidates.isEmpty()) {
+            return;
+        }
+        candidates.sort(FileIterator::compareFilePaths);
+        this.stack.add(candidates);
+        this.findNext();
+    }
+
+    @Override
+    public boolean hasNext() {
+        return this.next != null;
+    }
+
+    @Override
+    public String next() {
+        if (this.next == null) {
+            return null;
+        }
+        var ret = this.next;
+        this.next = null;
+        this.findNext();
+        return ret;
+    }
+
+    private void findNext() {
+        while (!this.stack.isEmpty()) {
+            var last = this.stack.get(this.stack.size() - 1);
+            if (last.isEmpty()) {
+                this.stack.remove(this.stack.size() - 1);
+                continue;
+            }
+            var next = last.remove(last.size() - 1);
+            if (!next.isDirectory()) {
+                var names = new ArrayList<String>();
+                for (var path : this.rootDir.toPath().relativize(next.toPath())) {
+                    names.add(path.toString());
+                }
+                this.next = String.join("/", names);
+                return;
+            }
+            var candidates = new ArrayList<>(Arrays.asList(Objects.requireNonNull(next.listFiles())));
+            candidates.sort(FileIterator::compareFilePaths);
+            this.stack.add(candidates);
+        }
+    }
+
+    private static int compareFilePaths(File a, File b) {
+        var x = a.getPath();
+        var y = b.getPath();
+        if (a.isDirectory()) {
+            x += "/";
+        }
+        if (b.isDirectory()) {
+            y += "/";
+        }
+        return y.compareTo(x);
+    }
+}