Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add arg builder to removeObjects API #967

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
91 changes: 67 additions & 24 deletions api/src/main/java/io/minio/MinioClient.java
Expand Up @@ -114,6 +114,8 @@
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.KeyManager;
import javax.net.ssl.KeyManagerFactory;
Expand Down Expand Up @@ -3116,30 +3118,69 @@ public void removeObject(RemoveObjectArgs args)
* @param bucketName Name of the bucket.
* @param objectNames List of Object names in the bucket.
* @return Iterable&ltResult&ltDeleteError&gt&gt - Lazy iterator contains object removal status.
* @deprecated use {@link #removeObjects(RemoveObjectsArgs)}
*/
@Deprecated
public Iterable<Result<DeleteError>> removeObjects(
final String bucketName, final Iterable<String> objectNames) {
Stream<DeleteObject> stream =
StreamSupport.stream(objectNames.spliterator(), false)
.map(
name -> {
return new DeleteObject(name);
});
return removeObjects(
RemoveObjectsArgs.builder().bucket(bucketName).objects(stream::iterator).build());
}

/**
* Removes multiple objects lazily. Its required to iterate the returned Iterable to perform
* removal.
*
* <pre>Example:{@code
* List<DeleteObject> objects = new LinkedList<>();
* objects.add(new DeleteObject("my-objectname1"));
* objects.add(new DeleteObject("my-objectname2"));
* objects.add(new DeleteObject("my-objectname3"));
* Iterable<Result<DeleteError>> results =
* minioClient.removeObjects(
* RemoveObjectsArgs.builder().bucket("my-bucketname").objects(objects).build());
* for (Result<DeleteError> result : results) {
* DeleteError error = errorResult.get();
* System.out.println(
* "Error in deleting object " + error.objectName() + "; " + error.message());
* }
* }</pre>
*
* @param args {@link RemoveObjectsArgs} object.
* @return Iterable&ltResult&ltDeleteError&gt&gt - Lazy iterator contains object removal status.
*/
public Iterable<Result<DeleteError>> removeObjects(RemoveObjectsArgs args) {
checkArgs(args);

return new Iterable<Result<DeleteError>>() {
@Override
public Iterator<Result<DeleteError>> iterator() {
return new Iterator<Result<DeleteError>>() {
private Result<DeleteError> error;
private Iterator<DeleteError> errorIterator;
private boolean completed = false;
private Iterator<String> objectNameIter = objectNames.iterator();
private Iterator<DeleteObject> objectIter = args.objects().iterator();

private synchronized void populate() {
List<DeleteError> errorList = null;
try {
List<DeleteObject> objectList = new LinkedList<DeleteObject>();
List<DeleteObject> objectList = new LinkedList<>();
int i = 0;
while (objectNameIter.hasNext() && i < 1000) {
objectList.add(new DeleteObject(objectNameIter.next()));
while (objectIter.hasNext() && i < 1000) {
objectList.add(objectIter.next());
i++;
}

if (i > 0) {
DeleteResult result = deleteObjects(bucketName, objectList, true);
if (objectList.size() > 0) {
DeleteResult result =
deleteObjects(
args.bucket(), objectList, args.quiet(), args.bypassGovernanceMode());
errorList = result.errorList();
}
} catch (ErrorResponseException
Expand Down Expand Up @@ -7118,32 +7159,34 @@ protected String createMultipartUpload(
* @throws XmlParserException thrown to indicate XML parsing error.
*/
protected DeleteResult deleteObjects(
String bucketName, List<DeleteObject> objectList, boolean quiet)
String bucketName, List<DeleteObject> objectList, boolean quiet, boolean bypassGovernanceMode)
throws InvalidBucketNameException, NoSuchAlgorithmException, InsufficientDataException,
IOException, InvalidKeyException, ServerException, XmlParserException,
ErrorResponseException, InternalException, InvalidResponseException {
Map<String, String> queryParamMap = new HashMap<>();
queryParamMap.put("delete", "");
Map<String, String> queryParams = new HashMap<>();
queryParams.put("delete", "");

DeleteRequest request = new DeleteRequest(objectList, quiet);
Response response = executePost(bucketName, null, null, queryParamMap, request);

String bodyContent = "";
try (ResponseBody body = response.body()) {
bodyContent = new String(body.bytes(), StandardCharsets.UTF_8);
Map<String, String> headers = null;
if (bypassGovernanceMode) {
headers = new HashMap<>();
headers.put("x-amz-bypass-governance-retention", "true");
}

try {
if (Xml.validate(DeleteError.class, bodyContent)) {
DeleteError error = Xml.unmarshal(DeleteError.class, bodyContent);
return new DeleteResult(error);
DeleteRequest request = new DeleteRequest(objectList, quiet);
try (Response response = executePost(bucketName, null, headers, queryParams, request)) {
String bodyContent = new String(response.body().bytes(), StandardCharsets.UTF_8);
try {
if (Xml.validate(DeleteError.class, bodyContent)) {
DeleteError error = Xml.unmarshal(DeleteError.class, bodyContent);
return new DeleteResult(error);
}
} catch (XmlParserException e) {
// As it is not <Error> message, parse it as <DeleteResult> message.
// Ignore this exception
}
} catch (XmlParserException e) {
// As it is not <Error> message, parse it as <DeleteResult> message.
// Ignore this exception
}

return Xml.unmarshal(DeleteResult.class, bodyContent);
return Xml.unmarshal(DeleteResult.class, bodyContent);
}
}

private Map<String, String> getCommonListObjectsQueryParams(ListObjectsArgs args) {
Expand Down
62 changes: 62 additions & 0 deletions api/src/main/java/io/minio/RemoveObjectsArgs.java
@@ -0,0 +1,62 @@
/*
* MinIO Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2020 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package io.minio;

import io.minio.messages.DeleteObject;
import java.util.LinkedList;

/** Argument class of MinioClient.removeObjects(). */
public class RemoveObjectsArgs extends BucketArgs {
private boolean bypassGovernanceMode;
private Iterable<DeleteObject> objects = new LinkedList<>();
private boolean quiet;

public boolean bypassGovernanceMode() {
return bypassGovernanceMode;
}

public Iterable<DeleteObject> objects() {
return objects;
}

public boolean quiet() {
return quiet;
}

public static Builder builder() {
return new Builder();
}

/** Argument builder of {@link RemoveObjectsArgs}. */
public static final class Builder extends BucketArgs.Builder<Builder, RemoveObjectsArgs> {
public Builder bypassGovernanceMode(boolean flag) {
operations.add(args -> args.bypassGovernanceMode = flag);
return this;
}

public Builder objects(Iterable<DeleteObject> objects) {
validateNotNull(objects, "objects");
operations.add(args -> args.objects = objects);
return this;
}

public Builder quiet(boolean flag) {
operations.add(args -> args.quiet = flag);
return this;
}
}
}
18 changes: 15 additions & 3 deletions api/src/main/java/io/minio/messages/DeletedObject.java
Expand Up @@ -25,18 +25,30 @@ public class DeletedObject {
@Element(name = "Key")
private String name;

@Element(name = "VersionId")
@Element(name = "VersionId", required = false)
private String versionId;

@Element(name = "DeleteMarker")
@Element(name = "DeleteMarker", required = false)
private boolean deleteMarker;

@Element(name = "DeleteMarkerVersionId")
@Element(name = "DeleteMarkerVersionId", required = false)
private String deleteMarkerVersionId;

public DeletedObject() {}

public String name() {
return name;
}

public String versionId() {
return versionId;
}

public boolean deleteMarker() {
return deleteMarker;
}

public String deleteMarkerVersionId() {
return deleteMarkerVersionId;
}
}
29 changes: 16 additions & 13 deletions docs/API.md
Expand Up @@ -1496,31 +1496,33 @@ minioClient.removeObject(
```

<a name="removeObjects"></a>
### removeObjects(String bucketName, Iterable<String> objectNames)
`public Iterable<Result<DeleteError>> removeObjects(String bucketName, Iterable<String> objectNames)` _[[Javadoc]](http://minio.github.io/minio-java/io/minio/MinioClient.html#removeObjects-java.lang.String-java.lang.Iterable-)_
### removeObjects(RemoveObjectsArgs args)
`public Iterable<Result<DeleteError>> removeObjects(RemoveObjectsArgs args)` _[[Javadoc]](http://minio.github.io/minio-java/io/minio/MinioClient.html#removeObjects-io.minio.RemoveObjectsArgs-)_

Removes multiple objects lazily. Its required to iterate the returned Iterable to perform removal.

__Parameters__
| Parameter | Type | Description |
|:----------------|:-------------------|:-------------------------------|
| ``bucketName`` | _String_ | Name of the bucket. |
| ``objectNames`` | _Iterable<String>_ | List of objects in the bucket. |
| Parameter | Type | Description |
|:----------|:----------------------|:------------|
| ``args`` | _[RemoveObjectsArgs]_ | Arguments. |

| Returns |
|:------------------------------------------------------------------------------------|
| _Iterable<[Result]<[DeleteError]>>_ - Lazy iterator contains object removal status. |

__Example__
```java
List<String> myObjectNames = new LinkedList<String>();
objectNames.add("my-objectname1");
objectNames.add("my-objectname2");
objectNames.add("my-objectname3");
Iterable<Result<DeleteError>> results = minioClient.removeObjects("my-bucketname", myObjectNames);
List<DeleteObject> objects = new LinkedList<>();
objects.add(new DeleteObject("my-objectname1"));
objects.add(new DeleteObject("my-objectname2"));
objects.add(new DeleteObject("my-objectname3"));
Iterable<Result<DeleteError>> results =
minioClient.removeObjects(
RemoveObjectsArgs.builder().bucket("my-bucketname").objects(objects).build());
for (Result<DeleteError> result : results) {
DeleteError error = errorResult.get();
System.out.println("Error in deleting object " + error.objectName() + "; " + error.message());
DeleteError error = result.get();
System.out.println(
"Error in deleting object " + error.objectName() + "; " + error.message());
}
```

Expand Down Expand Up @@ -1744,3 +1746,4 @@ ObjectStat objectStat =
[DeleteDefaultRetentionArgs]: http://minio.github.io/minio-java/io/minio/DeleteDefaultRetentionArgs.html
[RemoveIncompleteUploadArgs]: http://minio.github.io/minio-java/io/minio/RemoveIncompleteUploadArgs.html
[GetPresignedObjectUrlArgs]: http://minio.github.io/minio-java/io/minio/GetPresignedObjectUrlArgs.html
[RemoveObjectsArgs]: http://minio.github.io/minio-java/io/minio/RemoveObjectsArgs.html
23 changes: 12 additions & 11 deletions examples/RemoveObjects.java
Expand Up @@ -15,9 +15,11 @@
*/

import io.minio.MinioClient;
import io.minio.RemoveObjectsArgs;
import io.minio.Result;
import io.minio.errors.MinioException;
import io.minio.messages.DeleteError;
import io.minio.messages.DeleteObject;
import java.io.IOException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
Expand All @@ -40,18 +42,17 @@ public static void main(String[] args)
// MinioClient minioClient = new MinioClient("https://s3.amazonaws.com", "YOUR-ACCESSKEYID",
// "YOUR-SECRETACCESSKEY");

List<String> objectNames = new LinkedList<String>();
objectNames.add("my-objectname1");
objectNames.add("my-objectname2");
objectNames.add("my-objectname3");

// Remove object all objects 'objectNames' list from 'my-bucketname'.
// It is required to traverse over the returned Iterable for lazy evaluation.
for (Result<DeleteError> errorResult :
minioClient.removeObjects("my-bucketname", objectNames)) {
DeleteError error = errorResult.get();
List<DeleteObject> objects = new LinkedList<>();
objects.add(new DeleteObject("my-objectname1"));
objects.add(new DeleteObject("my-objectname2"));
objects.add(new DeleteObject("my-objectname3"));
Iterable<Result<DeleteError>> results =
minioClient.removeObjects(
RemoveObjectsArgs.builder().bucket("my-bucketname").objects(objects).build());
for (Result<DeleteError> result : results) {
DeleteError error = result.get();
System.out.println(
"Failed to remove '" + error.objectName() + "'. Error:" + error.message());
"Error in deleting object " + error.objectName() + "; " + error.message());
}
} catch (MinioException e) {
System.out.println("Error occurred: " + e);
Expand Down