Skip to content

Commit 2433d90

Browse files
rpmoorerpmoore
authored
Merge pull request #389 from RachelTucker/get_object_parsing_overload
Get object parsing overload
2 parents e11c1fd + 526270e commit 2433d90

File tree

6 files changed

+332
-0
lines changed

6 files changed

+332
-0
lines changed

‎ds3-sdk-integration/src/test/java/com/spectralogic/ds3client/integration/CustomParser_Test.java‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
/*
2+
* ****************************************************************************
3+
* Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved.
4+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
5+
* this file except in compliance with the License. A copy of the License is located at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* or in the "license" file accompanying this file.
10+
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
* specific language governing permissions and limitations under the License.
13+
* ****************************************************************************
14+
*/
15+
16+
package com.spectralogic.ds3client.integration;
17+
18+
import com.google.common.collect.Lists;
19+
import com.spectralogic.ds3client.Ds3Client;
20+
import com.spectralogic.ds3client.commands.GetObjectRequest;
21+
import com.spectralogic.ds3client.commands.GetObjectResponse;
22+
import com.spectralogic.ds3client.commands.parsers.interfaces.GetObjectCustomParserParameters;
23+
import com.spectralogic.ds3client.commands.parsers.utils.Function;
24+
import com.spectralogic.ds3client.commands.spectrads3.GetBulkJobSpectraS3Request;
25+
import com.spectralogic.ds3client.commands.spectrads3.GetBulkJobSpectraS3Response;
26+
import com.spectralogic.ds3client.exceptions.ContentLengthNotMatchException;
27+
import com.spectralogic.ds3client.helpers.Ds3ClientHelpers;
28+
import com.spectralogic.ds3client.helpers.channelbuilders.ObjectInputStreamBuilder;
29+
import com.spectralogic.ds3client.integration.test.helpers.TempStorageIds;
30+
import com.spectralogic.ds3client.integration.test.helpers.TempStorageUtil;
31+
import com.spectralogic.ds3client.models.ChecksumType;
32+
import com.spectralogic.ds3client.models.bulk.Ds3Object;
33+
import com.spectralogic.ds3client.networking.FailedRequestException;
34+
import com.spectralogic.ds3client.networking.WebResponse;
35+
import com.spectralogic.ds3client.utils.IOUtils;
36+
import org.junit.*;
37+
38+
import java.io.ByteArrayInputStream;
39+
import java.io.ByteArrayOutputStream;
40+
import java.io.IOException;
41+
import java.io.InputStream;
42+
import java.net.URISyntaxException;
43+
import java.util.UUID;
44+
45+
import static com.spectralogic.ds3client.commands.parsers.utils.ResponseParserUtils.getSizeFromHeaders;
46+
import static com.spectralogic.ds3client.integration.Util.deleteAllContents;
47+
import static org.hamcrest.CoreMatchers.is;
48+
import static org.junit.Assert.assertThat;
49+
import static org.junit.Assert.fail;
50+
51+
public class CustomParser_Test {
52+
53+
private static final Ds3Client client = Util.fromEnv();
54+
private static final Ds3ClientHelpers HELPERS = Ds3ClientHelpers.wrap(client);
55+
private static final String TEST_ENV_NAME = "custom_parser_test";
56+
private static final String BUCKET_NAME = "custom_parser_bucket";
57+
private static final String OBJ_NAME = "stream_object_test.txt";
58+
private static final String OBJ_CONTENT = "Content of stream object";
59+
private static final byte[] OBJ_BYTES = OBJ_CONTENT.getBytes();
60+
private static TempStorageIds envStorageIds;
61+
private static UUID envDataPolicyId;
62+
63+
@BeforeClass
64+
public static void startup() throws IOException {
65+
envDataPolicyId = TempStorageUtil.setupDataPolicy(TEST_ENV_NAME, false, ChecksumType.Type.MD5, client);
66+
envStorageIds = TempStorageUtil.setup(TEST_ENV_NAME, envDataPolicyId, client);
67+
}
68+
69+
@AfterClass
70+
public static void teardown() throws IOException {
71+
TempStorageUtil.teardown(TEST_ENV_NAME, envStorageIds, client);
72+
client.close();
73+
}
74+
75+
@Before
76+
public void beforeRunningTestMethod() throws IOException {
77+
//Create a bucket and transfer object to BP using ObjectInputStreamBuilder
78+
HELPERS.ensureBucketExists(BUCKET_NAME, envDataPolicyId);
79+
80+
final Ds3Object obj = new Ds3Object(OBJ_NAME, OBJ_BYTES.length);
81+
final Ds3ClientHelpers.Job job = HELPERS.startWriteJob(BUCKET_NAME, Lists.newArrayList(obj));
82+
83+
job.transfer(new ObjectInputStreamBuilder() {
84+
85+
@Override
86+
public InputStream buildInputStream(final String key) {
87+
return new ByteArrayInputStream(OBJ_BYTES);
88+
}
89+
});
90+
}
91+
92+
@After
93+
public void afterRunningTestMethod() throws IOException {
94+
deleteAllContents(client, BUCKET_NAME);
95+
}
96+
97+
@Test
98+
public void getObjectCustomParser_Test() throws IOException, URISyntaxException {
99+
final Ds3Object object = new Ds3Object(OBJ_NAME);
100+
final GetBulkJobSpectraS3Response getBulkJobSpectraS3Response = client
101+
.getBulkJobSpectraS3(new GetBulkJobSpectraS3Request(BUCKET_NAME, Lists.newArrayList(object)));
102+
103+
final ByteArrayOutputStream stream = new ByteArrayOutputStream();
104+
final GetObjectRequest request = new GetObjectRequest(
105+
BUCKET_NAME,
106+
OBJ_NAME,
107+
getBulkJobSpectraS3Response.getMasterObjectList().getJobId(),
108+
0,
109+
stream);
110+
111+
client.getObject(request, new Function<GetObjectCustomParserParameters, GetObjectResponse>() {
112+
@Override
113+
public GetObjectResponse apply(final GetObjectCustomParserParameters getObjectParserParams) {
114+
final WebResponse webResponse = getObjectParserParams.getWebResponse();
115+
final long objectSize = getSizeFromHeaders(webResponse.getHeaders());
116+
try (final InputStream responseStream = webResponse.getResponseStream()) {
117+
final long totalBytes = IOUtils.copy(
118+
responseStream,
119+
getObjectParserParams.getDestinationChannel(),
120+
getObjectParserParams.getBufferSize(),
121+
getObjectParserParams.getObjectName(),
122+
false);
123+
getObjectParserParams.getDestinationChannel().close();
124+
125+
if (objectSize != -1 && totalBytes != objectSize) {
126+
throw new ContentLengthNotMatchException(getObjectParserParams.getObjectName(), objectSize, totalBytes);
127+
}
128+
129+
} catch (final IOException e) {
130+
fail();
131+
}
132+
return null;
133+
}
134+
});
135+
assertThat(stream.toByteArray(), is(OBJ_BYTES));
136+
}
137+
138+
@SuppressWarnings("deprecation")
139+
@Test (expected = FailedRequestException.class)
140+
public void getObjectCustomParserNoObject_Test() throws IOException{
141+
final String objectName = "doesNotExistObject";
142+
143+
final GetObjectRequest request = new GetObjectRequest(
144+
BUCKET_NAME,
145+
objectName,
146+
new NullChannel());
147+
148+
//Get object that does not exist
149+
client.getObject(request, new Function<GetObjectCustomParserParameters, GetObjectResponse>() {
150+
@Override
151+
public GetObjectResponse apply(final GetObjectCustomParserParameters getObjectParserParams) {
152+
return null;
153+
}
154+
});
155+
}
156+
}

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@
2525
import com.spectralogic.ds3client.models.JobNode;
2626
import com.spectralogic.ds3client.networking.ConnectionDetails;
2727

28+
import com.spectralogic.ds3client.commands.parsers.interfaces.GetObjectCustomParserParameters;
29+
import com.spectralogic.ds3client.commands.parsers.utils.Function;
30+
2831
import java.io.Closeable;
2932
import java.io.IOException;
3033

@@ -1988,5 +1991,8 @@ GetObjectResponse getObject(final GetObjectRequest request)
19881991
throws IOException;
19891992

19901993

1994+
GetObjectResponse getObject(final GetObjectRequest request,
1995+
final Function<GetObjectCustomParserParameters, GetObjectResponse> responseParser) throws IOException;
1996+
19911997
Ds3Client newForNode(final JobNode node);
19921998
}

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@
2626
import com.spectralogic.ds3client.networking.NetworkClient;
2727
import com.spectralogic.ds3client.networking.NetworkClientImpl;
2828

29+
import com.spectralogic.ds3client.commands.parsers.interfaces.GetObjectCustomParserParameters;
30+
import com.spectralogic.ds3client.commands.parsers.utils.Function;
31+
2932
public class Ds3ClientImpl implements Ds3Client {
3033
private final NetworkClient netClient;
3134

@@ -1224,6 +1227,18 @@ public GetObjectResponse getObject(final GetObjectRequest request) throws IOExce
12241227
.response(this.netClient.getResponse(request));
12251228
}
12261229

1230+
@Override
1231+
public GetObjectResponse getObject(
1232+
final GetObjectRequest request,
1233+
final Function<GetObjectCustomParserParameters, GetObjectResponse> parsingFunction) throws IOException {
1234+
return new GetObjectCustomParser(
1235+
request.getChannel(),
1236+
this.netClient.getConnectionDetails().getBufferSize(),
1237+
request.getObjectName(),
1238+
parsingFunction)
1239+
.response(this.netClient.getResponse(request));
1240+
}
1241+
12271242
@Override
12281243
public Ds3Client newForNode(final JobNode node) {
12291244
final ConnectionDetails newConnectionDetails = ConnectionDetailsImpl.newForNode(node, this.getConnectionDetails());

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/GetObjectCustomParser.java‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/*
2+
* ******************************************************************************
3+
* Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved.
4+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
5+
* this file except in compliance with the License. A copy of the License is located at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* or in the "license" file accompanying this file.
10+
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
* specific language governing permissions and limitations under the License.
13+
* ****************************************************************************
14+
*/
15+
16+
package com.spectralogic.ds3client;
17+
18+
import com.spectralogic.ds3client.commands.GetObjectRequest;
19+
import com.spectralogic.ds3client.commands.GetObjectResponse;
20+
import com.spectralogic.ds3client.commands.parsers.interfaces.AbstractResponseParser;
21+
import com.spectralogic.ds3client.commands.parsers.interfaces.GetObjectCustomParserParameters;
22+
import com.spectralogic.ds3client.commands.parsers.utils.Function;
23+
import com.spectralogic.ds3client.commands.parsers.utils.ResponseParserUtils;
24+
import com.spectralogic.ds3client.networking.WebResponse;
25+
26+
import java.io.IOException;
27+
import java.nio.channels.WritableByteChannel;
28+
29+
/**
30+
* Creates a custom parser which is used in {@link com.spectralogic.ds3client.Ds3Client#getObject(GetObjectRequest, Function)} )}
31+
*/
32+
class GetObjectCustomParser extends AbstractResponseParser<GetObjectResponse> {
33+
34+
private final int[] expectedStatusCodes = new int[]{200, 206};
35+
36+
private final WritableByteChannel destinationChannel;
37+
private final int bufferSize;
38+
private final String objectName;
39+
private final Function<GetObjectCustomParserParameters, GetObjectResponse> parsingFunction;
40+
41+
public GetObjectCustomParser(
42+
final WritableByteChannel destinationChannel,
43+
final int bufferSize,
44+
final String objectName,
45+
final Function<GetObjectCustomParserParameters, GetObjectResponse> parsingFunction) {
46+
47+
this.destinationChannel = destinationChannel;
48+
this.bufferSize = bufferSize;
49+
this.objectName = objectName;
50+
this.parsingFunction = parsingFunction;
51+
}
52+
53+
@Override
54+
public GetObjectResponse parseXmlResponse(final WebResponse response) throws IOException {
55+
final int statusCode = response.getStatusCode();
56+
if (ResponseParserUtils.validateStatusCode(statusCode, expectedStatusCodes)) {
57+
final GetObjectCustomParserParameters params = new GetObjectCustomParserParameters(
58+
response,
59+
destinationChannel,
60+
bufferSize,
61+
objectName);
62+
63+
return parsingFunction.apply(params);
64+
}
65+
throw ResponseParserUtils.createFailedRequest(response, expectedStatusCodes);
66+
}
67+
}

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/interfaces/GetObjectCustomParserParameters.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*
2+
* ******************************************************************************
3+
* Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved.
4+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
5+
* this file except in compliance with the License. A copy of the License is located at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* or in the "license" file accompanying this file.
10+
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
* specific language governing permissions and limitations under the License.
13+
* ****************************************************************************
14+
*/
15+
16+
package com.spectralogic.ds3client.commands.parsers.interfaces;
17+
18+
import com.spectralogic.ds3client.networking.WebResponse;
19+
20+
import java.nio.channels.WritableByteChannel;
21+
22+
public class GetObjectCustomParserParameters {
23+
24+
private final WebResponse webResponse;
25+
private final WritableByteChannel destinationChannel;
26+
private final int bufferSize;
27+
private final String objectName;
28+
29+
public GetObjectCustomParserParameters(
30+
final WebResponse webResponse,
31+
final WritableByteChannel destinationChannel,
32+
final int bufferSize,
33+
final String objectName) {
34+
this.webResponse = webResponse;
35+
this.destinationChannel = destinationChannel;
36+
this.bufferSize = bufferSize;
37+
this.objectName = objectName;
38+
}
39+
40+
public WritableByteChannel getDestinationChannel() {
41+
return destinationChannel;
42+
}
43+
44+
public int getBufferSize() {
45+
return bufferSize;
46+
}
47+
48+
public String getObjectName() {
49+
return objectName;
50+
}
51+
52+
public WebResponse getWebResponse() {
53+
return webResponse;
54+
}
55+
}

‎ds3-sdk/src/main/java/com/spectralogic/ds3client/commands/parsers/utils/Function.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
* ******************************************************************************
3+
* Copyright 2014-2016 Spectra Logic Corporation. All Rights Reserved.
4+
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use
5+
* this file except in compliance with the License. A copy of the License is located at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* or in the "license" file accompanying this file.
10+
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
11+
* CONDITIONS OF ANY KIND, either express or implied. See the License for the
12+
* specific language governing permissions and limitations under the License.
13+
* ****************************************************************************
14+
*/
15+
16+
package com.spectralogic.ds3client.commands.parsers.utils;
17+
18+
/**
19+
* Defines the interface for a function that takes in specified input,
20+
* and produced specified output.
21+
*
22+
* This provides a Java 7 simple compatible version of {@link java.util.function.Function}
23+
*
24+
* @param <T> function input
25+
* @param <R> function output
26+
*/
27+
public interface Function<T, R> {
28+
29+
/**
30+
* Same as {@link java.util.function.Function#apply(Object)}
31+
*/
32+
R apply(T t);
33+
}

0 commit comments

Comments
 (0)