Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Break and stop the pagination if we have the results #152

Merged
merged 3 commits into from Feb 23, 2019
Merged

Break and stop the pagination if we have the results #152

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9bba797
Break and stop the pagination if we have the results
Fokko Feb 21, 2019
84a0213
Fix test and breaking condition
medb Feb 23, 2019
b05e6cf
Release notes
medb Feb 23, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 3 additions & 0 deletions gcs/CHANGES.txt
View file
Open in desktop
Expand Up @@ -6,6 +6,9 @@

3. Do not parallelize GCS list requests, because it leads to too high QPS.

4. Fix bug when GCS connector lists all files in directory instead of
specified limit.


1.9.14 - 2019-02-13

Expand Down
3 changes: 2 additions & 1 deletion gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageImpl.java
View file
Open in desktop
Expand Up @@ -1211,7 +1211,8 @@ private void listStorageObjectsAndPrefixes(
}
pageToken =
listStorageObjectsAndPrefixesPage(listObject, maxResults, listedObjects, listedPrefixes);
} while (pageToken != null);
} while (pageToken != null
&& (maxResults <= 0 || listedObjects.size() + listedPrefixes.size() < maxResults));
}

private String listStorageObjectsAndPrefixesPage(
Expand Down
33 changes: 15 additions & 18 deletions gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageTest.java
View file
Open in desktop
Expand Up @@ -2758,12 +2758,9 @@ public void testListObjectNamesPrefix()
verify(mockStorageObjectsList, times(2)).execute();
}

/**
* Test GoogleCloudStorage.listObjectNames(3) with maxResults set.
*/
/** Test GoogleCloudStorage.listObjectNames(3) with maxResults set. */
@Test
public void testListObjectNamesPrefixLimited()
throws IOException {
public void testListObjectNamesPrefixLimited() throws IOException {
String objectPrefix = "foo/bar/baz/";
String delimiter = "/";
long maxResults = 3;
Expand All @@ -2772,17 +2769,18 @@ public void testListObjectNamesPrefixLimited()
.thenReturn(mockStorageObjectsList);
when(mockStorageObjectsList.getPrefix()).thenReturn(objectPrefix);
when(mockStorageObjectsList.execute())
.thenReturn(new Objects()
.setPrefixes(ImmutableList.of(
"foo/bar/baz/dir0/",
"foo/bar/baz/dir1/"))
.setNextPageToken("token0"))
.thenReturn(new Objects()
.setItems(ImmutableList.of(
new StorageObject().setName("foo/bar/baz/"),
new StorageObject().setName("foo/bar/baz/obj0"),
new StorageObject().setName("foo/bar/baz/obj1")))
.setNextPageToken(null));
.thenReturn(
new Objects()
.setPrefixes(ImmutableList.of("foo/bar/baz/dir0/", "foo/bar/baz/dir1/"))
.setItems(
ImmutableList.of(
new StorageObject().setName("foo/bar/baz/"),
new StorageObject().setName("foo/bar/baz/obj0")))
.setNextPageToken("token0"))
.thenReturn(
new Objects()
.setItems(ImmutableList.of(new StorageObject().setName("foo/bar/baz/obj1")))
.setNextPageToken(null));

List<String> objectNames =
gcs.listObjectNames(BUCKET_NAME, objectPrefix, delimiter, maxResults);
Expand All @@ -2796,9 +2794,8 @@ public void testListObjectNamesPrefixLimited()
verify(mockStorageObjectsList).setDelimiter(eq(delimiter));
verify(mockStorageObjectsList).setIncludeTrailingDelimiter(eq(Boolean.FALSE));
verify(mockStorageObjectsList).setPrefix(eq(objectPrefix));
verify(mockStorageObjectsList).setPageToken("token0");
verify(mockStorageObjectsList).getPrefix();
verify(mockStorageObjectsList, times(2)).execute();
verify(mockStorageObjectsList).execute();
}

/**
Expand Down