paginate through large folders

seaweedfs · Dec 17, 2018 · d11cfdb · d11cfdb
1 parent 5271278
commit d11cfdb
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 15 deletions.
diff --git a/other/java/client/pom.xml b/other/java/client/pom.xml
@@ -4,7 +4,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.0.2</version>
+    <version>1.0.3</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java
@@ -5,6 +5,7 @@
 
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
@@ -137,7 +138,23 @@ public FilerProto.Entry.Builder newFileEntry(String name, int mode,
     }
 
     public List<FilerProto.Entry> listEntries(String path) {
-        return listEntries(path, "", "", 100000);
+        List<FilerProto.Entry> results = new ArrayList<FilerProto.Entry>();
+        String lastFileName = "";
+        for (int limit = Integer.MAX_VALUE; limit > 0; ) {
+            List<FilerProto.Entry> t = listEntries(path, "", lastFileName, 1024);
+            if (t == null) {
+                break;
+            }
+            int nSize = t.size();
+            if (nSize > 0) {
+                limit -= nSize;
+                lastFileName = t.get(nSize - 1).getName();
+            }
+            if (t.size() < 1024) {
+                break;
+            }
+        }
+        return results;
     }
 
     public List<FilerProto.Entry> listEntries(String path, String entryPrefix, String lastEntryName, int limit) {

diff --git a/other/java/hdfs/pom.xml b/other/java/hdfs/pom.xml
@@ -4,9 +4,14 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
+    <properties>
+        <seaweedfs.client.version>1.0.3</seaweedfs.client.version>
+        <hadoop.version>3.1.1</hadoop.version>
+    </properties>
+
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-hadoop-client</artifactId>
-    <version>1.0.2</version>
+    <version>${seaweedfs.client.version}</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -133,10 +138,6 @@
         </plugins>
     </build>
 
-    <properties>
-        <hadoop.version>3.1.1</hadoop.version>
-    </properties>
-
     <dependencies>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
@@ -146,7 +147,7 @@
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-client</artifactId>
-            <version>1.0.2</version>
+            <version>${seaweedfs.client.version}</version>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>

diff --git a/other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
@@ -310,9 +310,6 @@ public boolean truncate(Path f, long newLength) throws IOException {
             getClass().getSimpleName() + " FileSystem implementation");
     }
 
-    /**
-     * See {@link FileContext#createSymlink(Path, Path, boolean)}.
-     */
     @Override
     public void createSymlink(final Path target, final Path link,
                               final boolean createParent) throws AccessControlException,
@@ -324,10 +321,6 @@ public void createSymlink(final Path target, final Path link,
             "Filesystem does not support symlinks!");
     }
 
-    /**
-     * See {@link AbstractFileSystem#supportsSymlinks()}.
-     */
-    @Override
     public boolean supportsSymlinks() {
         return false;
     }