From 3faddac0892010194971f376cf55115e8f946029 Mon Sep 17 00:00:00 2001 From: Hans-Peter Klett Date: Sun, 14 Sep 2014 16:03:38 -0600 Subject: [PATCH 01/16] Added the ObjectPart entity. --- .../ds3client/helpers/ObjectPart.java | 50 +++++++++++++++++++ .../ds3client/helpers/ObjectPart_Test.java | 47 +++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectPart.java create mode 100644 sdk/src/test/java/com/spectralogic/ds3client/helpers/ObjectPart_Test.java diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectPart.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectPart.java new file mode 100644 index 000000000..729663c0d --- /dev/null +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectPart.java @@ -0,0 +1,50 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +public final class ObjectPart { + private final long offset; + private final long length; + + public ObjectPart(final long offset, final long length) { + this.offset = offset; + this.length = length; + } + + public long getOffset() { + return this.offset; + } + + public long getLength() { + return this.length; + } + + public long getEnd() { + return this.offset + this.length - 1; + } + + @Override + public boolean equals(final Object obj) { + if (obj instanceof ObjectPart) { + final ObjectPart otherPart = (ObjectPart)obj; + return + this.offset == otherPart.offset + && this.length == otherPart.length; + } else { + return false; + } + } +} diff --git a/sdk/src/test/java/com/spectralogic/ds3client/helpers/ObjectPart_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/helpers/ObjectPart_Test.java new file mode 100644 index 000000000..6f03f5af7 --- /dev/null +++ b/sdk/src/test/java/com/spectralogic/ds3client/helpers/ObjectPart_Test.java @@ -0,0 +1,47 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +public class ObjectPart_Test { + @Test + public void assignsFields() { + final ObjectPart objectPart = new ObjectPart(10L, 11L); + assertThat(objectPart.getOffset(), is(10L)); + assertThat(objectPart.getLength(), is(11L)); + assertThat(objectPart.getEnd(), is(20L)); + } + + @Test + public void equalsWorks() { + final ObjectPart original = new ObjectPart(10L, 11L); + final Object other1 = new ObjectPart(10L, 11L); + final Object other2 = new ObjectPart(11L, 11L); + final Object other3 = new ObjectPart(10L, 12L); + final Object other4 = new ObjectPart(15L, 15L); + final Object other5 = "foo"; + assertThat(original.equals(original), is(true)); + assertThat(original.equals(other1), is(true)); + assertThat(original.equals(other2), is(false)); + assertThat(original.equals(other3), is(false)); + assertThat(original.equals(other4), is(false)); + assertThat(original.equals(other5), is(false)); + } +} From 1a7266e08ca05f530b6e920ce7cfc8b7d30ed875 Mon Sep 17 00:00:00 2001 From: Hans-Peter Klett Date: Sun, 14 Sep 2014 19:01:30 -0600 Subject: [PATCH 02/16] Created object part tracking logic. --- .../helpers/DataTransferredListener.java | 20 +++ .../helpers/ObjectCompletedListener.java | 20 +++ .../helpers/ObjectPartComparator.java | 43 ++++++ .../ds3client/helpers/ObjectPartTracker.java | 24 ++++ .../helpers/ObjectPartTrackerImpl.java | 97 +++++++++++++ .../helpers/ObjectPartComparator_Test.java | 51 +++++++ ...jectPartTrackerImpl_CompletePart_Test.java | 133 ++++++++++++++++++ .../helpers/ObjectPartTrackerImpl_Test.java | 47 +++++++ 8 files changed, 435 insertions(+) create mode 100644 sdk/src/main/java/com/spectralogic/ds3client/helpers/DataTransferredListener.java create mode 100644 sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectCompletedListener.java create mode 100644 sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectPartComparator.java create mode 100644 sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectPartTracker.java create mode 100644 sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectPartTrackerImpl.java create mode 100644 sdk/src/test/java/com/spectralogic/ds3client/helpers/ObjectPartComparator_Test.java create mode 100644 sdk/src/test/java/com/spectralogic/ds3client/helpers/ObjectPartTrackerImpl_CompletePart_Test.java create mode 100644 sdk/src/test/java/com/spectralogic/ds3client/helpers/ObjectPartTrackerImpl_Test.java diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/DataTransferredListener.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/DataTransferredListener.java new file mode 100644 index 000000000..fd3607b77 --- /dev/null +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/DataTransferredListener.java @@ -0,0 +1,20 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +interface DataTransferredListener { + void dataTransferred(final long size); +} diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectCompletedListener.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectCompletedListener.java new file mode 100644 index 000000000..03c3fa490 --- /dev/null +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectCompletedListener.java @@ -0,0 +1,20 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +interface ObjectCompletedListener { + void objectCompleted(final String name); +} diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectPartComparator.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectPartComparator.java new file mode 100644 index 000000000..899c28ffe --- /dev/null +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectPartComparator.java @@ -0,0 +1,43 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import java.util.Comparator; + +public class ObjectPartComparator implements Comparator { + private static final ObjectPartComparator instance = new ObjectPartComparator(); + + public static ObjectPartComparator instance() { + return instance; + } + + private ObjectPartComparator() { + // Use singleton + } + + @Override + public int compare(final ObjectPart o1, final ObjectPart o2) { + return intOf(signum(o1.getOffset() - o2.getOffset())); + } + + private static long signum(final long value) { + return value < 0 ? -1 : (value == 0 ? 0 : 1); + } + + private static int intOf(final long value) { + return (int)value; + } +} diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectPartTracker.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectPartTracker.java new file mode 100644 index 000000000..35114b00f --- /dev/null +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectPartTracker.java @@ -0,0 +1,24 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +interface ObjectPartTracker { + ObjectPartTracker attachDataTransferredListener(final DataTransferredListener listener); + ObjectPartTracker attachObjectCompletedListener(final ObjectCompletedListener listener); + + void completePart(final ObjectPart part); + boolean containsPart(final ObjectPart part); +} diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectPartTrackerImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectPartTrackerImpl.java new file mode 100644 index 000000000..591e05bbf --- /dev/null +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectPartTrackerImpl.java @@ -0,0 +1,97 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import java.security.InvalidParameterException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.TreeSet; + +class ObjectPartTrackerImpl implements ObjectPartTracker { + private final String name; + private final TreeSet parts; + private final Collection dataTransferredListeners = new ArrayList<>(); + private final Collection objectCompletedListeners = new ArrayList<>(); + + public ObjectPartTrackerImpl(final String name, final Collection parts) { + this.name = name; + this.parts = new TreeSet(ObjectPartComparator.instance()); + this.parts.addAll(parts); + validateParts(); + } + + private void validateParts() { + long lastEnd = -1L; + for (final ObjectPart part : this.parts) { + if (part.getOffset() <= lastEnd) { + throw new InvalidParameterException(); + } + lastEnd = part.getEnd(); + } + } + + @Override + public ObjectPartTracker attachDataTransferredListener(final DataTransferredListener listener) { + this.dataTransferredListeners.add(listener); + return this; + } + + @Override + public ObjectPartTracker attachObjectCompletedListener(final ObjectCompletedListener listener) { + this.objectCompletedListeners.add(listener); + return this; + } + + @Override + public void completePart(final ObjectPart part) { + final ObjectPart existingPart = this.parts.floor(part); + if (existingPart == null) { + throw new IllegalStateException("The object part was not available to be marked completed."); + } + if (part.getEnd() > existingPart.getEnd()) { + throw new IllegalStateException("The object part was not available to be marked completed."); + } + this.parts.remove(existingPart); + if (part.getOffset() > existingPart.getOffset()) { + this.parts.add(new ObjectPart(existingPart.getOffset(), part.getOffset() - existingPart.getOffset())); + } + if (part.getEnd() < existingPart.getEnd()) { + this.parts.add(new ObjectPart(part.getEnd() + 1, existingPart.getEnd() - part.getEnd())); + } + onDataTransferred(part.getLength()); + if (this.parts.size() == 0) { + onObjectCompleted(); + } + } + + private void onDataTransferred(final long size) { + for (final DataTransferredListener listener : this.dataTransferredListeners) { + listener.dataTransferred(size); + } + } + + private void onObjectCompleted() { + for (final ObjectCompletedListener listener : this.objectCompletedListeners) { + listener.objectCompleted(this.name); + } + } + + @Override + public boolean containsPart(final ObjectPart part) { + final ObjectPart existingPart = this.parts.ceiling(part); + return existingPart != null && existingPart.getLength() == part.getLength(); + } +} diff --git a/sdk/src/test/java/com/spectralogic/ds3client/helpers/ObjectPartComparator_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/helpers/ObjectPartComparator_Test.java new file mode 100644 index 000000000..4657ed1aa --- /dev/null +++ b/sdk/src/test/java/com/spectralogic/ds3client/helpers/ObjectPartComparator_Test.java @@ -0,0 +1,51 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +public class ObjectPartComparator_Test { + @Test + public void compare() { + final List objects = Arrays.asList( + new ObjectPart(3L, 2L), + new ObjectPart(0L, 2L), + new ObjectPart(2L, 1L), + new ObjectPart(0L, 1L) + ); + Collections.sort(objects, ObjectPartComparator.instance()); + final List expected = Arrays.asList( + new ObjectPart(0L, 1L), + new ObjectPart(0L, 2L), + new ObjectPart(2L, 1L), + new ObjectPart(3L, 2L) + ); + for (int i = 0; i < expected.size(); i++) + { + final ObjectPart current = objects.get(i); + final ObjectPart expectedObj = expected.get(i); + assertThat(current.getOffset(), is(expectedObj.getOffset())); + assertThat(current.getOffset(), is(expectedObj.getOffset())); + } + } +} diff --git a/sdk/src/test/java/com/spectralogic/ds3client/helpers/ObjectPartTrackerImpl_CompletePart_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/helpers/ObjectPartTrackerImpl_CompletePart_Test.java new file mode 100644 index 000000000..fcc3656ac --- /dev/null +++ b/sdk/src/test/java/com/spectralogic/ds3client/helpers/ObjectPartTrackerImpl_CompletePart_Test.java @@ -0,0 +1,133 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +@RunWith(Parameterized.class) +public class ObjectPartTrackerImpl_CompletePart_Test { + private final String name; + private final boolean shouldSucceed; + private final Collection parts; + + public ObjectPartTrackerImpl_CompletePart_Test(final String name, final Boolean shouldSucceed, final Collection parts) { + this.name = name; + this.shouldSucceed = shouldSucceed; + this.parts = parts; + } + + @Parameters(name = "{0} should succeed == {1}") + public static Collection testCases() { + return Arrays.asList( + new Object[] { "entire part", true, Arrays.asList( + new ObjectPart(100L, 100L), + new ObjectPart(0L, 100L), + new ObjectPart(200L, 100L) + ) }, + new Object[] { "first then second part", true, Arrays.asList( + new ObjectPart(100L, 75L), + new ObjectPart(175L, 25L), + new ObjectPart(0L, 100L), + new ObjectPart(200L, 100L) + ) }, + new Object[] { "second then first part", true, Arrays.asList( + new ObjectPart(175L, 25L), + new ObjectPart(100L, 75L), + new ObjectPart(0L, 100L), + new ObjectPart(200L, 100L) + ) }, + new Object[] { "middle then first then second part", true, Arrays.asList( + new ObjectPart(125L, 50L), + new ObjectPart(100L, 25L), + new ObjectPart(175L, 25L), + new ObjectPart(0L, 100L), + new ObjectPart(200L, 100L) + ) }, + new Object[] { "completely before", false, Arrays.asList(new ObjectPart(0L, 99L)) }, + new Object[] { "just before", false, Arrays.asList(new ObjectPart(0L, 100L)) }, + new Object[] { "overlapping before", false, Arrays.asList(new ObjectPart(50L, 100L)) }, + new Object[] { "overlapping both", false, Arrays.asList(new ObjectPart(50L, 200L)) }, + new Object[] { "overlapping after", false, Arrays.asList(new ObjectPart(150L, 100L)) }, + new Object[] { "just after", false, Arrays.asList(new ObjectPart(200L, 100L)) }, + new Object[] { "completely after", false, Arrays.asList(new ObjectPart(201L, 99L)) } + ); + } + + @Test + public void completePartFiresExpectedEvents() { + if (this.shouldSucceed) { + checkSuccess(); + } else { + checkFailure(); + } + } + + private void checkSuccess() { + final ObjectPartTracker tracker = new ObjectPartTrackerImpl(this.name, Arrays.asList( + new ObjectPart(0L, 100L), + new ObjectPart(100L, 100L), + new ObjectPart(200L, 100L) + )); + final List events = new ArrayList<>(); + tracker.attachDataTransferredListener(new DataTransferredListener() { + @Override + public void dataTransferred(final long size) { + events.add(Long.valueOf(size)); + } + }); + tracker.attachObjectCompletedListener(new ObjectCompletedListener() { + @Override + public void objectCompleted(final String name) { + events.add(name); + } + }); + for (final ObjectPart part : this.parts) { + tracker.completePart(part); + } + assertThat(events.toArray(), is(equalTo(buildExpectedEvents().toArray()))); + } + + private Collection buildExpectedEvents() { + final Collection expectedEvents = new ArrayList<>(); + for (final ObjectPart part : this.parts) { + expectedEvents.add(part.getLength()); + } + expectedEvents.add(this.name); + return expectedEvents; + } + + private void checkFailure() { + final ObjectPartTracker tracker = new ObjectPartTrackerImpl(this.name, Arrays.asList(new ObjectPart(100L, 100L))); + try { + tracker.completePart(this.parts.iterator().next()); + Assert.fail(); + } catch (final IllegalStateException e) { + } + } +} diff --git a/sdk/src/test/java/com/spectralogic/ds3client/helpers/ObjectPartTrackerImpl_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/helpers/ObjectPartTrackerImpl_Test.java new file mode 100644 index 000000000..8cb9f05ac --- /dev/null +++ b/sdk/src/test/java/com/spectralogic/ds3client/helpers/ObjectPartTrackerImpl_Test.java @@ -0,0 +1,47 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import org.junit.Test; + +import java.security.InvalidParameterException; +import java.util.Arrays; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +public class ObjectPartTrackerImpl_Test { + @Test + public void invalidPartsFail() { + try { + new ObjectPartTrackerImpl("foo", Arrays.asList( + new ObjectPart(0L, 100L), + new ObjectPart(99L, 100L) + )); + fail(); + } catch (final InvalidParameterException e) { + } + } + + @Test + public void containsPartWorks() { + final ObjectPartTracker tracker = new ObjectPartTrackerImpl("foo", Arrays.asList(new ObjectPart(0L, 100L), new ObjectPart(100L, 100L))); + assertThat(tracker.containsPart(new ObjectPart(100L, 100L)), is(true)); + assertThat(tracker.containsPart(new ObjectPart(100L, 50L)), is(false)); + assertThat(tracker.containsPart(new ObjectPart(150L, 50L)), is(false)); + } +} From 4761dafee1fbbdb44145f40b5ac5bc17665a2413 Mon Sep 17 00:00:00 2001 From: Hans-Peter Klett Date: Sun, 14 Sep 2014 19:12:43 -0600 Subject: [PATCH 03/16] Updated the object part tracker for concurrency --- .../helpers/ObjectPartTrackerImpl.java | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectPartTrackerImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectPartTrackerImpl.java index 591e05bbf..c6bc5789c 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectPartTrackerImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ObjectPartTrackerImpl.java @@ -33,30 +33,20 @@ public ObjectPartTrackerImpl(final String name, final Collection par validateParts(); } - private void validateParts() { - long lastEnd = -1L; - for (final ObjectPart part : this.parts) { - if (part.getOffset() <= lastEnd) { - throw new InvalidParameterException(); - } - lastEnd = part.getEnd(); - } - } - @Override - public ObjectPartTracker attachDataTransferredListener(final DataTransferredListener listener) { + public synchronized ObjectPartTracker attachDataTransferredListener(final DataTransferredListener listener) { this.dataTransferredListeners.add(listener); return this; } @Override - public ObjectPartTracker attachObjectCompletedListener(final ObjectCompletedListener listener) { + public synchronized ObjectPartTracker attachObjectCompletedListener(final ObjectCompletedListener listener) { this.objectCompletedListeners.add(listener); return this; } @Override - public void completePart(final ObjectPart part) { + public synchronized void completePart(final ObjectPart part) { final ObjectPart existingPart = this.parts.floor(part); if (existingPart == null) { throw new IllegalStateException("The object part was not available to be marked completed."); @@ -76,6 +66,22 @@ public void completePart(final ObjectPart part) { onObjectCompleted(); } } + + @Override + public synchronized boolean containsPart(final ObjectPart part) { + final ObjectPart existingPart = this.parts.ceiling(part); + return existingPart != null && existingPart.getLength() == part.getLength(); + } + + private void validateParts() { + long lastEnd = -1L; + for (final ObjectPart part : this.parts) { + if (part.getOffset() <= lastEnd) { + throw new InvalidParameterException(); + } + lastEnd = part.getEnd(); + } + } private void onDataTransferred(final long size) { for (final DataTransferredListener listener : this.dataTransferredListeners) { @@ -88,10 +94,4 @@ private void onObjectCompleted() { listener.objectCompleted(this.name); } } - - @Override - public boolean containsPart(final ObjectPart part) { - final ObjectPart existingPart = this.parts.ceiling(part); - return existingPart != null && existingPart.getLength() == part.getLength(); - } } From aff9ac3e8b7c3a8d11e1242ca10e010c94010b15 Mon Sep 17 00:00:00 2001 From: Hans-Peter Klett Date: Mon, 15 Sep 2014 13:15:29 -0600 Subject: [PATCH 04/16] Created a cache that auto-closes generated values. --- .../ds3client/helpers/AutoCloseableCache.java | 71 +++++++++ .../helpers/AutoCloseableCache_Test.java | 148 ++++++++++++++++++ 2 files changed, 219 insertions(+) create mode 100644 sdk/src/main/java/com/spectralogic/ds3client/helpers/AutoCloseableCache.java create mode 100644 sdk/src/test/java/com/spectralogic/ds3client/helpers/AutoCloseableCache_Test.java diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/AutoCloseableCache.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/AutoCloseableCache.java new file mode 100644 index 000000000..3f872d51d --- /dev/null +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/AutoCloseableCache.java @@ -0,0 +1,71 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +class AutoCloseableCache implements AutoCloseable { + private final ValueBuilder valueBuilder; + private final Set closedKeys = new HashSet<>(); + private Map values = new HashMap<>(); + + public interface ValueBuilder { + Value get(final Key key); + } + + public AutoCloseableCache(final ValueBuilder valueBuilder) { + this.valueBuilder = valueBuilder; + } + + public synchronized Value get(final Key key) { + if (this.values == null) { + throw new IllegalStateException("Cache already closed."); + } + if (this.closedKeys.contains(key)) { + throw new IllegalStateException("Cache has already closed the requested key."); + } + Value value = this.values.get(key); + if (value == null) { + value = this.valueBuilder.get(key); + this.values.put(key, value); + } + return value; + } + + public synchronized void close(final Key key) throws Exception { + if (this.values == null) { + throw new IllegalStateException("Cache already closed."); + } + final Value value = this.values.remove(key); + if (value != null) { + value.close(); + } + this.closedKeys.add(key); + } + + @Override + public synchronized void close() throws Exception { + if (this.values != null) { + for (final Value value : this.values.values()) { + value.close(); + } + this.values = null; + } + } +} diff --git a/sdk/src/test/java/com/spectralogic/ds3client/helpers/AutoCloseableCache_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/helpers/AutoCloseableCache_Test.java new file mode 100644 index 000000000..ecf7af9d6 --- /dev/null +++ b/sdk/src/test/java/com/spectralogic/ds3client/helpers/AutoCloseableCache_Test.java @@ -0,0 +1,148 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import com.spectralogic.ds3client.helpers.AutoCloseableCache.ValueBuilder; + +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.sameInstance; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +public class AutoCloseableCache_Test { + @Test + public void returnsSameInstanceWhenCalledTwice() throws Exception { + final AutoCloseableCache closeableCache = + new AutoCloseableCache<>(new CloseableImplValueBuilder()); + final CloseableImpl item = closeableCache.get("foo"); + assertThat(item.getKey(), is("foo")); + assertThat(closeableCache.get("foo"), is(sameInstance(item))); + closeableCache.close(); + } + + @Test + public void cacheDisposesAllItemsExactlyOnce() throws Exception { + final AutoCloseableCache closeableCache = + new AutoCloseableCache<>(new CloseableImplValueBuilder()); + + final CloseableImpl item1 = closeableCache.get("foo"); + final CloseableImpl item2 = closeableCache.get("bar"); + + assertThat(item1.getCloseCallCount(), is(0)); + assertThat(item2.getCloseCallCount(), is(0)); + + closeableCache.close(); + + assertThat(item1.getCloseCallCount(), is(1)); + assertThat(item2.getCloseCallCount(), is(1)); + + closeableCache.close(); + + assertThat(item1.getCloseCallCount(), is(1)); + assertThat(item2.getCloseCallCount(), is(1)); + } + + @Test + public void cacheCanCloseItemsEarly() throws Exception { + final AutoCloseableCache closeableCache = + new AutoCloseableCache<>(new CloseableImplValueBuilder()); + + final CloseableImpl item1 = closeableCache.get("foo"); + final CloseableImpl item2 = closeableCache.get("bar"); + + assertThat(item1.getCloseCallCount(), is(0)); + assertThat(item2.getCloseCallCount(), is(0)); + + closeableCache.close("foo"); + + assertThat(item1.getCloseCallCount(), is(1)); + assertThat(item2.getCloseCallCount(), is(0)); + + try { + closeableCache.get("foo"); + fail(); + } catch (final IllegalStateException e) { + } + + closeableCache.close(); + + assertThat(item1.getCloseCallCount(), is(1)); + assertThat(item2.getCloseCallCount(), is(1)); + } + + @Test + public void cacheCanCloseBeforeGet() throws Exception { + final AutoCloseableCache closeableCache = + new AutoCloseableCache<>(new CloseableImplValueBuilder()); + + closeableCache.close("foo"); + + try { + closeableCache.get("foo"); + fail(); + } catch (final IllegalStateException e) { + } + } + + @Test + public void cacheCallsFailWhenDisposed() throws Exception { + final AutoCloseableCache closeableCache = + new AutoCloseableCache<>(new CloseableImplValueBuilder()); + closeableCache.close(); + + try { + closeableCache.get("foo"); + fail(); + } catch (final IllegalStateException e) { + } + try { + closeableCache.close("foo"); + fail(); + } catch (final IllegalStateException e) { + } + } + + private final class CloseableImplValueBuilder implements ValueBuilder { + @Override + public CloseableImpl get(final String key) { + return new CloseableImpl(key); + } + } + + private static final class CloseableImpl implements AutoCloseable { + private final String key; + private int closeCallCount = 0; + + public CloseableImpl(final String key) { + this.key = key; + } + + public String getKey() { + return this.key; + } + + public int getCloseCallCount() { + return this.closeCallCount; + } + + @Override + public void close() throws Exception { + this.closeCallCount++; + } + } +} From 2c55f23dec60c0bbd47c13c6fd325d0553b47cb1 Mon Sep 17 00:00:00 2001 From: Hans-Peter Klett Date: Mon, 15 Sep 2014 15:49:50 -0600 Subject: [PATCH 05/16] Added a windowed seekable byte channel. --- .../utils/TruncateNotAllowedException.java | 25 ++ .../utils/WindowedSeekableByteChannel.java | 127 ++++++++++ .../WindowedSeekableByteChannel_Test.java | 230 ++++++++++++++++++ 3 files changed, 382 insertions(+) create mode 100644 sdk/src/main/java/com/spectralogic/ds3client/utils/TruncateNotAllowedException.java create mode 100644 sdk/src/main/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel.java create mode 100644 sdk/src/test/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel_Test.java diff --git a/sdk/src/main/java/com/spectralogic/ds3client/utils/TruncateNotAllowedException.java b/sdk/src/main/java/com/spectralogic/ds3client/utils/TruncateNotAllowedException.java new file mode 100644 index 000000000..b2eab8e50 --- /dev/null +++ b/sdk/src/main/java/com/spectralogic/ds3client/utils/TruncateNotAllowedException.java @@ -0,0 +1,25 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.utils; + +import java.io.IOException; + +public class TruncateNotAllowedException extends IOException { + private static final long serialVersionUID = -7378017389677465526L; + + public TruncateNotAllowedException() { + } +} diff --git a/sdk/src/main/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel.java b/sdk/src/main/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel.java new file mode 100644 index 000000000..5df9963dc --- /dev/null +++ b/sdk/src/main/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel.java @@ -0,0 +1,127 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.utils; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; + +public class WindowedSeekableByteChannel implements SeekableByteChannel { + private final SeekableByteChannel channel; + private final Object lock; + private final long offset; + private final long length; + private long position = 0L; + private boolean isOpen = true; + private final Copy readCopy = new ReadCopy(); + private final Copy writeCopy = new WriteCopy(); + + public WindowedSeekableByteChannel( + final SeekableByteChannel channel, + final Object lock, + final long offset, + final long length) { + this.channel = channel; + this.lock = lock; + this.offset = offset; + this.length = length; + } + + @Override + public boolean isOpen() { + return this.isOpen; + } + + @Override + public void close() throws IOException { + this.isOpen = false; + } + + @Override + public int read(final ByteBuffer dst) throws IOException { + return copy(dst, readCopy); + } + + @Override + public int write(final ByteBuffer src) throws IOException { + return copy(src, writeCopy); + } + + private int copy(final ByteBuffer buffer, final Copy copy) throws IOException { + synchronized (this.lock) { + if (this.position >= this.length) { + return -1; + } + + final int oldLimit = buffer.limit(); + final long distanceFromEnd = this.length - this.position; + if (buffer.remaining() > distanceFromEnd) { + buffer.limit(oldLimit + (int)distanceFromEnd - buffer.remaining()); + } + + this.channel.position(this.offset + this.position); + final int bytesCopied = copy.copy(buffer); + this.position += bytesCopied; + + buffer.limit(oldLimit); + + return bytesCopied; + } + } + + @Override + public long position() throws IOException { + synchronized (this.lock) { + return this.position; + } + } + + @Override + public SeekableByteChannel position(final long newPosition) throws IOException { + synchronized (this.lock) { + this.position = newPosition; + return this; + } + } + + @Override + public long size() throws IOException { + return this.length; + } + + @Override + public SeekableByteChannel truncate(final long size) throws IOException { + throw new TruncateNotAllowedException(); + } + + private interface Copy { + int copy(final ByteBuffer buffer) throws IOException; + } + + private final class ReadCopy implements Copy { + @Override + public int copy(final ByteBuffer buffer) throws IOException { + return WindowedSeekableByteChannel.this.channel.read(buffer); + } + } + + private final class WriteCopy implements Copy { + @Override + public int copy(final ByteBuffer buffer) throws IOException { + return WindowedSeekableByteChannel.this.channel.write(buffer); + } + } +} diff --git a/sdk/src/test/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel_Test.java new file mode 100644 index 000000000..0872ea1df --- /dev/null +++ b/sdk/src/test/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel_Test.java @@ -0,0 +1,230 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.utils; + +import org.apache.commons.io.IOUtils; +import org.junit.Test; + +import java.io.IOException; +import java.io.Writer; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.SeekableByteChannel; +import java.nio.charset.Charset; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoMoreInteractions; + +public class WindowedSeekableByteChannel_Test { + @Test(timeout = 1000) + public void closeChangesIsOpen() throws IOException { + try (final SeekableByteChannel channel = stringToChannel("aabbbcccc")) { + final Object lock = new Object(); + final SeekableByteChannel window = new WindowedSeekableByteChannel(channel, lock, 0L, 2L); + assertThat(window.isOpen(), is(true)); + window.close(); + assertThat(window.isOpen(), is(false)); + } + } + + @Test(timeout = 1000) + public void readChannelSections() throws IOException { + try (final SeekableByteChannel channel = stringToChannel("aabbbcccc")) { + final Object lock = new Object(); + final SeekableByteChannel channelOfAs = new WindowedSeekableByteChannel(channel, lock, 0L, 2L); + final SeekableByteChannel channelOfBs = new WindowedSeekableByteChannel(channel, lock, 2L, 3L); + final SeekableByteChannel channelOfCs = new WindowedSeekableByteChannel(channel, lock, 5L, 4L); + + assertThat(channelOfAs.size(), is(2L)); + assertThat(channelOfBs.size(), is(3L)); + assertThat(channelOfCs.size(), is(4L)); + + assertThat(channelToString(channelOfAs), is("aa")); + assertThat(channelToString(channelOfBs), is("bbb")); + assertThat(channelToString(channelOfCs), is("cccc")); + } + } + + @Test(timeout = 1000) + public void writeChannelSections() throws IOException { + try (final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel()) { + final Object lock = new Object(); + final SeekableByteChannel channelOfAs = new WindowedSeekableByteChannel(channel, lock, 0L, 2L); + final SeekableByteChannel channelOfBs = new WindowedSeekableByteChannel(channel, lock, 2L, 3L); + final SeekableByteChannel channelOfCs = new WindowedSeekableByteChannel(channel, lock, 5L, 4L); + + assertThat(channelOfAs.size(), is(2L)); + assertThat(channelOfBs.size(), is(3L)); + assertThat(channelOfCs.size(), is(4L)); + + writeToChannel("cccc", channelOfCs); + writeToChannel("bbb", channelOfBs); + writeToChannel("aa", channelOfAs); + + channel.position(0); + + assertThat(channel.toString(), is("aabbbcccc")); + } + } + + @Test(timeout = 1000) + public void writeDoesNotExceedWindow() throws IOException { + try (final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel()) { + final Object lock = new Object(); + try (final WindowedSeekableByteChannel window = new WindowedSeekableByteChannel(channel, lock, 2L, 7L)) { + final ByteBuffer buffer = Charset.forName("UTF-8").encode("0123456789"); + buffer.position(1); + assertThat(window.write(buffer), is(7)); + assertThat(window.position(), is(7L)); + assertThat(buffer.position(), is(8)); + assertThat(buffer.limit(), is(10)); + + assertThat(window.write(buffer), is(-1)); + assertThat(window.position(), is(7L)); + + assertThat(channel.size(), is(9L)); + channel.position(0); + + assertThat(channel.toString(), is("\0\0" + "1234567")); + } + } + } + + @Test(timeout = 1000) + public void readPositionTracking() throws IOException { + try (final SeekableByteChannel channel = stringToChannel("aabbbcccc")) { + final Object lock = new Object(); + try (final WindowedSeekableByteChannel window = new WindowedSeekableByteChannel(channel, lock, 2L, 7L)) { + final byte[] bytes = new byte[10]; + final ByteBuffer buffer = ByteBuffer.wrap(bytes); + + buffer.limit(3); + assertThat(window.read(buffer), is(3)); + assertThat(window.position(), is(3L)); + assertThat(buffer.position(), is(3)); + assertThat(buffer.limit(), is(3)); + assertThat(new String(bytes, 0, 3, Charset.forName("UTF-8")), is("bbb")); + + buffer.limit(10); + assertThat(window.read(buffer), is(4)); + assertThat(window.position(), is(7L)); + assertThat(buffer.position(), is(7)); + assertThat(buffer.limit(), is(10)); + assertThat(new String(bytes, 3, 4, Charset.forName("UTF-8")), is("cccc")); + } + } + } + + @Test(timeout = 1000) + public void writePositionTracking() throws IOException { + try (final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel()) { + final Object lock = new Object(); + try (final WindowedSeekableByteChannel window = new WindowedSeekableByteChannel(channel, lock, 2L, 7L)) { + ByteBuffer buffer = Charset.forName("UTF-8").encode("bbb"); + assertThat(window.write(buffer), is(3)); + assertThat(buffer.position(), is(3)); + + buffer = Charset.forName("UTF-8").encode("cccc"); + assertThat(window.write(buffer), is(4)); + assertThat(buffer.position(), is(4)); + + channel.position(0); + + assertThat(channel.toString(), is("\0\0bbbcccc")); + } + } + } + + @Test(timeout = 1000) + public void positionUpdate() throws IOException { + try (final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel()) { + final Object lock = new Object(); + try (final WindowedSeekableByteChannel window = new WindowedSeekableByteChannel(channel, lock, 2L, 7L)) { + assertThat(window.position(), is(0L)); + assertThat(window.position(5L), is((SeekableByteChannel)window)); + assertThat(window.position(), is(5L)); + assertThat(window.position(), is(5L)); + } + } + } + + @Test(timeout = 1000) + public void positionThenWriteReturnsEOF() throws IOException { + try (final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel()) { + final Object lock = new Object(); + try (final WindowedSeekableByteChannel window = new WindowedSeekableByteChannel(channel, lock, 2L, 7L)) { + window.position(10L); + assertThat(window.position(), is(10L)); + final ByteBuffer fakeBuffer = mock(ByteBuffer.class); + assertThat(window.write(fakeBuffer), is(-1)); + verifyNoMoreInteractions(fakeBuffer); + } + } + } + + @Test(timeout = 1000) + public void positionThenReadReturnsEOF() throws IOException { + try (final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel()) { + final Object lock = new Object(); + try (final WindowedSeekableByteChannel window = new WindowedSeekableByteChannel(channel, lock, 2L, 7L)) { + window.position(10L); + assertThat(window.position(), is(10L)); + final ByteBuffer fakeBuffer = mock(ByteBuffer.class); + assertThat(window.read(fakeBuffer), is(-1)); + verifyNoMoreInteractions(fakeBuffer); + } + } + } + + @Test(timeout = 1000) + public void truncateNotAllowed() throws IOException { + try (final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel()) { + final Object lock = new Object(); + try (final WindowedSeekableByteChannel window = new WindowedSeekableByteChannel(channel, lock, 2L, 7L)) { + try { + window.truncate(5L); + fail(); + } catch (final TruncateNotAllowedException e) { + } + try { + window.truncate(10L); + fail(); + } catch (final TruncateNotAllowedException e) { + } + } + } + } + + private static void writeToChannel(final String string, final SeekableByteChannel channel) throws IOException { + final Writer writer = Channels.newWriter(channel, "UTF-8"); + writer.write(string); + writer.flush(); + } + + private static SeekableByteChannel stringToChannel(final String string) throws IOException { + final SeekableByteChannel channel = new ByteArraySeekableByteChannel(); + writeToChannel(string, channel); + channel.position(0); + return channel; + } + + private static String channelToString(final SeekableByteChannel channel) throws IOException { + return IOUtils.toString(Channels.newReader(channel, "UTF-8")); + } +} From 60b24f8e949374947f84d7e505ce39e7f35d5378 Mon Sep 17 00:00:00 2001 From: Hans-Peter Klett Date: Mon, 15 Sep 2014 16:07:47 -0600 Subject: [PATCH 06/16] Enforced close on the window channel. --- .../utils/WindowedSeekableByteChannel.java | 24 +++++++++++-- .../WindowedSeekableByteChannel_Test.java | 35 +++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/sdk/src/main/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel.java b/sdk/src/main/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel.java index 5df9963dc..2a60e55d8 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel.java @@ -42,12 +42,16 @@ public WindowedSeekableByteChannel( @Override public boolean isOpen() { - return this.isOpen; + synchronized (this.lock) { + return this.isOpen; + } } @Override public void close() throws IOException { - this.isOpen = false; + synchronized (this.lock) { + this.isOpen = false; + } } @Override @@ -62,6 +66,9 @@ public int write(final ByteBuffer src) throws IOException { private int copy(final ByteBuffer buffer, final Copy copy) throws IOException { synchronized (this.lock) { + if (!this.isOpen) { + throw new IllegalStateException("Object already closed"); + } if (this.position >= this.length) { return -1; } @@ -85,6 +92,9 @@ private int copy(final ByteBuffer buffer, final Copy copy) throws IOException { @Override public long position() throws IOException { synchronized (this.lock) { + if (!this.isOpen) { + throw new IllegalStateException("Object already closed"); + } return this.position; } } @@ -92,6 +102,9 @@ public long position() throws IOException { @Override public SeekableByteChannel position(final long newPosition) throws IOException { synchronized (this.lock) { + if (!this.isOpen) { + throw new IllegalStateException("Object already closed"); + } this.position = newPosition; return this; } @@ -99,7 +112,12 @@ public SeekableByteChannel position(final long newPosition) throws IOException { @Override public long size() throws IOException { - return this.length; + synchronized (this.lock) { + if (!this.isOpen) { + throw new IllegalStateException("Object already closed"); + } + return this.length; + } } @Override diff --git a/sdk/src/test/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel_Test.java index 0872ea1df..41a7c8078 100644 --- a/sdk/src/test/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel_Test.java +++ b/sdk/src/test/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel_Test.java @@ -211,6 +211,41 @@ public void truncateNotAllowed() throws IOException { } } + @Test(timeout = 1000) + public void interactAfterCloseNotAllowed() throws IOException { + try (final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel()) { + final Object lock = new Object(); + try (final WindowedSeekableByteChannel window = new WindowedSeekableByteChannel(channel, lock, 2L, 7L)) { + window.close(); + try { + window.read(ByteBuffer.wrap(new byte[10])); + fail(); + } catch (final IllegalStateException e) { + } + try { + window.write(ByteBuffer.wrap(new byte[10])); + fail(); + } catch (final IllegalStateException e) { + } + try { + window.position(); + fail(); + } catch (final IllegalStateException e) { + } + try { + window.position(1L); + fail(); + } catch (final IllegalStateException e) { + } + try { + window.size(); + fail(); + } catch (final IllegalStateException e) { + } + } + } + } + private static void writeToChannel(final String string, final SeekableByteChannel channel) throws IOException { final Writer writer = Channels.newWriter(channel, "UTF-8"); writer.write(string); From 12f1df5bf15f374d59bec86275fcc03ac4ed43f7 Mon Sep 17 00:00:00 2001 From: Hans-Peter Klett Date: Mon, 15 Sep 2014 16:23:07 -0600 Subject: [PATCH 07/16] Created a closeable factory for windowed channels. --- .../utils/WindowedChannelFactory.java | 36 +++++++++++++ .../utils/WindowedChannelFactory_Test.java | 52 +++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 sdk/src/main/java/com/spectralogic/ds3client/utils/WindowedChannelFactory.java create mode 100644 sdk/src/test/java/com/spectralogic/ds3client/utils/WindowedChannelFactory_Test.java diff --git a/sdk/src/main/java/com/spectralogic/ds3client/utils/WindowedChannelFactory.java b/sdk/src/main/java/com/spectralogic/ds3client/utils/WindowedChannelFactory.java new file mode 100644 index 000000000..91b520877 --- /dev/null +++ b/sdk/src/main/java/com/spectralogic/ds3client/utils/WindowedChannelFactory.java @@ -0,0 +1,36 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.utils; + +import java.nio.channels.SeekableByteChannel; + +public class WindowedChannelFactory implements AutoCloseable { + private final SeekableByteChannel channel; + private final Object lock = new Object(); + + public WindowedChannelFactory(final SeekableByteChannel channel) { + this.channel = channel; + } + + public SeekableByteChannel get(final long offset, final long length) { + return new WindowedSeekableByteChannel(this.channel, this.lock, offset, length); + } + + @Override + public void close() throws Exception { + this.channel.close(); + } +} diff --git a/sdk/src/test/java/com/spectralogic/ds3client/utils/WindowedChannelFactory_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/utils/WindowedChannelFactory_Test.java new file mode 100644 index 000000000..73132481c --- /dev/null +++ b/sdk/src/test/java/com/spectralogic/ds3client/utils/WindowedChannelFactory_Test.java @@ -0,0 +1,52 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.utils; + +import org.apache.commons.io.IOUtils; +import org.junit.Test; + +import java.io.Writer; +import java.nio.channels.Channels; +import java.nio.channels.SeekableByteChannel; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; +import static org.mockito.Mockito.*; + +public class WindowedChannelFactory_Test { + @Test + public void getReturnsWindow() throws Exception { + try (final ByteArraySeekableByteChannel channel = new ByteArraySeekableByteChannel()) { + final Writer writer = Channels.newWriter(channel, "UTF-8"); + writer.write("0123456789"); + writer.close(); + + try (final WindowedChannelFactory windowedChannelFactory = new WindowedChannelFactory(channel)) { + try (final SeekableByteChannel window = windowedChannelFactory.get(2L, 6L)) { + assertThat(IOUtils.toString(Channels.newReader(window, "UTF-8")), is("234567")); + } + } + } + } + + @Test + public void closeClosesUnderlyingChannel() throws Exception { + final SeekableByteChannel channel = mock(SeekableByteChannel.class); + new WindowedChannelFactory(channel).close(); + verify(channel).close(); + verifyNoMoreInteractions(channel); + } +} From 6dcf6da5e2afcf77bb72c2612644bfc5f48c23d0 Mon Sep 17 00:00:00 2001 From: Hans-Peter Klett Date: Mon, 15 Sep 2014 16:43:32 -0600 Subject: [PATCH 08/16] Updated the parallelism in the chunk transfer. Updated the chunk transfer mechanism to be able to process several chunks at once. Currently we still only process one at a time, but this prepares us to do get available chunks for the read job. --- .../ds3client/helpers/JobImpl.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java index 2134ee210..2f3145f6e 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java @@ -28,9 +28,7 @@ import java.io.IOException; import java.security.SignatureException; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; +import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; @@ -71,10 +69,10 @@ public Job withMaxParallelRequests(final int maxParallelRequests) { } @Override - public void transfer(final ObjectChannelBuilder transferrer) + public void transfer(final ObjectTransferrer transferrer) throws SignatureException, IOException, XmlProcessingException { for (final Objects chunk : this.objectLists) { - this.transferChunk(transferrer, chunk); + this.transferChunks(transferrer, Arrays.asList(chunk)); } } @@ -83,16 +81,18 @@ protected abstract void transferItem( final UUID jobId, final String bucketName, final BulkObject ds3Object, - final ObjectChannelBuilder transferrer) throws SignatureException, IOException; - - private void transferChunk(final ObjectChannelBuilder transferrer, final Objects objects) + final ObjectTransferrer transferrer) throws SignatureException, IOException; + + private void transferChunks(final ObjectTransferrer transferrer, final Collection chunkList) throws SignatureException, IOException, XmlProcessingException { final ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(this.maxParallelRequests)); + final List> tasks = new ArrayList<>(); try { - final Ds3Client client = this.clientFactory.getClientForNodeId(objects.getNodeId()); - final List> tasks = new ArrayList<>(); - for (final BulkObject ds3Object : objects) { - tasks.add(this.createTransferTask(transferrer, service, client, ds3Object)); + for (final Objects chunk : chunkList) { + final Ds3Client client = this.clientFactory.getClientForNodeId(chunk.getNodeId()); + for (final BulkObject ds3Object : chunk) { + tasks.add(this.createTransferTask(transferrer, service, client, ds3Object)); + } } this.executeWithExceptionHandling(tasks); } finally { @@ -101,7 +101,7 @@ private void transferChunk(final ObjectChannelBuilder transferrer, final Objects } private ListenableFuture createTransferTask( - final ObjectChannelBuilder transferrer, + final ObjectTransferrer transferrer, final ListeningExecutorService service, final Ds3Client client, final BulkObject ds3Object) { From 48ca01985be704d3963fe2409d9568f27e71374c Mon Sep 17 00:00:00 2001 From: Hans-Peter Klett Date: Tue, 16 Sep 2014 10:37:53 -0600 Subject: [PATCH 09/16] Moved the helper stuff into the helpers package. --- .../{utils => helpers}/TruncateNotAllowedException.java | 4 ++-- .../ds3client/{utils => helpers}/WindowedChannelFactory.java | 4 ++-- .../{utils => helpers}/WindowedSeekableByteChannel.java | 4 ++-- .../{utils => helpers}/WindowedChannelFactory_Test.java | 4 +++- .../{utils => helpers}/WindowedSeekableByteChannel_Test.java | 4 +++- 5 files changed, 12 insertions(+), 8 deletions(-) rename sdk/src/main/java/com/spectralogic/ds3client/{utils => helpers}/TruncateNotAllowedException.java (89%) rename sdk/src/main/java/com/spectralogic/ds3client/{utils => helpers}/WindowedChannelFactory.java (92%) rename sdk/src/main/java/com/spectralogic/ds3client/{utils => helpers}/WindowedSeekableByteChannel.java (97%) rename sdk/src/test/java/com/spectralogic/ds3client/{utils => helpers}/WindowedChannelFactory_Test.java (94%) rename sdk/src/test/java/com/spectralogic/ds3client/{utils => helpers}/WindowedSeekableByteChannel_Test.java (99%) diff --git a/sdk/src/main/java/com/spectralogic/ds3client/utils/TruncateNotAllowedException.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/TruncateNotAllowedException.java similarity index 89% rename from sdk/src/main/java/com/spectralogic/ds3client/utils/TruncateNotAllowedException.java rename to sdk/src/main/java/com/spectralogic/ds3client/helpers/TruncateNotAllowedException.java index b2eab8e50..681082d5e 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/utils/TruncateNotAllowedException.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/TruncateNotAllowedException.java @@ -13,11 +13,11 @@ * **************************************************************************** */ -package com.spectralogic.ds3client.utils; +package com.spectralogic.ds3client.helpers; import java.io.IOException; -public class TruncateNotAllowedException extends IOException { +class TruncateNotAllowedException extends IOException { private static final long serialVersionUID = -7378017389677465526L; public TruncateNotAllowedException() { diff --git a/sdk/src/main/java/com/spectralogic/ds3client/utils/WindowedChannelFactory.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/WindowedChannelFactory.java similarity index 92% rename from sdk/src/main/java/com/spectralogic/ds3client/utils/WindowedChannelFactory.java rename to sdk/src/main/java/com/spectralogic/ds3client/helpers/WindowedChannelFactory.java index 91b520877..fa8fa7ba1 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/utils/WindowedChannelFactory.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/WindowedChannelFactory.java @@ -13,11 +13,11 @@ * **************************************************************************** */ -package com.spectralogic.ds3client.utils; +package com.spectralogic.ds3client.helpers; import java.nio.channels.SeekableByteChannel; -public class WindowedChannelFactory implements AutoCloseable { +class WindowedChannelFactory implements AutoCloseable { private final SeekableByteChannel channel; private final Object lock = new Object(); diff --git a/sdk/src/main/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/WindowedSeekableByteChannel.java similarity index 97% rename from sdk/src/main/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel.java rename to sdk/src/main/java/com/spectralogic/ds3client/helpers/WindowedSeekableByteChannel.java index 2a60e55d8..a0133cea0 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/WindowedSeekableByteChannel.java @@ -13,13 +13,13 @@ * **************************************************************************** */ -package com.spectralogic.ds3client.utils; +package com.spectralogic.ds3client.helpers; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.SeekableByteChannel; -public class WindowedSeekableByteChannel implements SeekableByteChannel { +class WindowedSeekableByteChannel implements SeekableByteChannel { private final SeekableByteChannel channel; private final Object lock; private final long offset; diff --git a/sdk/src/test/java/com/spectralogic/ds3client/utils/WindowedChannelFactory_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/helpers/WindowedChannelFactory_Test.java similarity index 94% rename from sdk/src/test/java/com/spectralogic/ds3client/utils/WindowedChannelFactory_Test.java rename to sdk/src/test/java/com/spectralogic/ds3client/helpers/WindowedChannelFactory_Test.java index 73132481c..79d852b9a 100644 --- a/sdk/src/test/java/com/spectralogic/ds3client/utils/WindowedChannelFactory_Test.java +++ b/sdk/src/test/java/com/spectralogic/ds3client/helpers/WindowedChannelFactory_Test.java @@ -13,7 +13,9 @@ * **************************************************************************** */ -package com.spectralogic.ds3client.utils; +package com.spectralogic.ds3client.helpers; + +import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel; import org.apache.commons.io.IOUtils; import org.junit.Test; diff --git a/sdk/src/test/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/helpers/WindowedSeekableByteChannel_Test.java similarity index 99% rename from sdk/src/test/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel_Test.java rename to sdk/src/test/java/com/spectralogic/ds3client/helpers/WindowedSeekableByteChannel_Test.java index 41a7c8078..c435f7056 100644 --- a/sdk/src/test/java/com/spectralogic/ds3client/utils/WindowedSeekableByteChannel_Test.java +++ b/sdk/src/test/java/com/spectralogic/ds3client/helpers/WindowedSeekableByteChannel_Test.java @@ -13,7 +13,9 @@ * **************************************************************************** */ -package com.spectralogic.ds3client.utils; +package com.spectralogic.ds3client.helpers; + +import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel; import org.apache.commons.io.IOUtils; import org.junit.Test; From a754acc2805f038f592e3b8b5bef8c9adaea8bc1 Mon Sep 17 00:00:00 2001 From: Hans-Peter Klett Date: Tue, 16 Sep 2014 11:17:46 -0600 Subject: [PATCH 10/16] Added a job part tracker. --- .../ds3client/helpers/JobPartTracker.java | 48 ++++++ .../helpers/JobPartTracker_Test.java | 153 ++++++++++++++++++ 2 files changed, 201 insertions(+) create mode 100644 sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTracker.java create mode 100644 sdk/src/test/java/com/spectralogic/ds3client/helpers/JobPartTracker_Test.java diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTracker.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTracker.java new file mode 100644 index 000000000..d5b5a136c --- /dev/null +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTracker.java @@ -0,0 +1,48 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import java.util.Map; + +public class JobPartTracker { + private final Map trackers; + + public JobPartTracker(final Map trackers) { + this.trackers = trackers; + } + + public void completePart(final String key, final ObjectPart objectPart) { + trackers.get(key).completePart(objectPart); + } + + public boolean containsPart(final String key, final ObjectPart objectPart) { + return trackers.get(key).containsPart(objectPart); + } + + public JobPartTracker attachDataTransferredListener(final DataTransferredListener listener) { + for (final ObjectPartTracker tracker : this.trackers.values()) { + tracker.attachDataTransferredListener(listener); + } + return this; + } + + public JobPartTracker attachObjectCompletedListener(final ObjectCompletedListener listener) { + for (final ObjectPartTracker tracker : this.trackers.values()) { + tracker.attachObjectCompletedListener(listener); + } + return this; + } +} diff --git a/sdk/src/test/java/com/spectralogic/ds3client/helpers/JobPartTracker_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/helpers/JobPartTracker_Test.java new file mode 100644 index 000000000..548968a30 --- /dev/null +++ b/sdk/src/test/java/com/spectralogic/ds3client/helpers/JobPartTracker_Test.java @@ -0,0 +1,153 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import org.junit.Test; + +import java.util.*; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +public class JobPartTracker_Test { + @Test + public void trackerCallsTrackers() { + final MockObjectPartTracker fooTracker = new MockObjectPartTracker("foo", new ArrayList<>(Arrays.asList(false, true))); + final MockObjectPartTracker barTracker = new MockObjectPartTracker("bar", new ArrayList<>(Arrays.asList(true, false))); + final Map trackers = new HashMap<>(); + trackers.put("foo", fooTracker); + trackers.put("bar", barTracker); + final JobPartTracker jobPartTracker = new JobPartTracker(trackers); + + final ObjectPart[] partsRemoved = { + new ObjectPart(10, 11), + new ObjectPart(12, 13) + }; + jobPartTracker.completePart("foo", partsRemoved[0]); + jobPartTracker.completePart("foo", partsRemoved[1]); + + final ObjectPart[] partsChecked = { + new ObjectPart(14, 15), + new ObjectPart(16, 17) + }; + assertThat(jobPartTracker.containsPart("foo", partsChecked[0]), is(false)); + assertThat(jobPartTracker.containsPart("foo", partsChecked[1]), is(true)); + + assertThat(fooTracker.getPartsRemoved(), is((Collection)Arrays.asList(partsRemoved))); + assertThat(fooTracker.getPartsChecked(), is((Collection)Arrays.asList(partsChecked))); + + assertThat(barTracker.getPartsRemoved().isEmpty(), is(true)); + assertThat(barTracker.getPartsChecked().isEmpty(), is(true)); + } + + @Test + public void TrackerEventsForward() + { + final MockObjectPartTracker fooTracker = new MockObjectPartTracker("foo", Arrays.asList(false, true)); + final MockObjectPartTracker barTracker = new MockObjectPartTracker("bar", Arrays.asList(true, false)); + final Map trackers = new HashMap<>(); + trackers.put("foo", fooTracker); + trackers.put("bar", barTracker); + + final List sizes = new ArrayList<>(); + final List objects = new ArrayList<>(); + + final JobPartTracker jobPartTracker = new JobPartTracker(trackers); + jobPartTracker.attachDataTransferredListener(new DataTransferredListener() { + @Override + public void dataTransferred(final long size) { + sizes.add(size); + } + }); + jobPartTracker.attachObjectCompletedListener(new ObjectCompletedListener() { + @Override + public void objectCompleted(final String name) { + objects.add(name); + } + }); + + fooTracker.onDataTransferred(10); + barTracker.onDataTransferred(11); + fooTracker.onDataTransferred(12); + + barTracker.onCompleted(); + fooTracker.onCompleted(); + + assertThat(sizes, is(Arrays.asList(10L, 11L, 12L))); + assertThat(objects, is(Arrays.asList("bar", "foo"))); + } + + private static class MockObjectPartTracker implements ObjectPartTracker { + private final String objectName; + private final List _containsPartResponses; + private final List dataTransferredListeners = new ArrayList<>(); + private final List objectCompletedListeners = new ArrayList<>(); + private final List _partsRemoved = new ArrayList<>(); + private final List _partsChecked = new ArrayList<>(); + + public MockObjectPartTracker(final String objectName, final List containsPartResponses) { + this.objectName = objectName; + this._containsPartResponses = containsPartResponses; + } + + public Collection getPartsRemoved() { + return this._partsRemoved; + } + + public Collection getPartsChecked() { + return this._partsChecked; + } + + public void onDataTransferred(final long size) + { + for (final DataTransferredListener listener : this.dataTransferredListeners) { + listener.dataTransferred(size); + } + } + + public void onCompleted() + { + for (final ObjectCompletedListener listener : this.objectCompletedListeners) { + listener.objectCompleted(this.objectName); + } + } + + @Override + public ObjectPartTracker attachDataTransferredListener(final DataTransferredListener listener) { + this.dataTransferredListeners.add(listener); + return this; + } + + @Override + public ObjectPartTracker attachObjectCompletedListener(final ObjectCompletedListener listener) { + this.objectCompletedListeners.add(listener); + return this; + } + + @Override + public void completePart(final ObjectPart partToRemove) + { + this._partsRemoved.add(partToRemove); + } + + @Override + public boolean containsPart(final ObjectPart part) + { + this._partsChecked.add(part); + return this._containsPartResponses.remove(0); + } + } +} From 18b7b51a335543d4a06cb40cb78dfdb31d8f37d7 Mon Sep 17 00:00:00 2001 From: Hans-Peter Klett Date: Tue, 16 Sep 2014 12:36:44 -0600 Subject: [PATCH 11/16] Created a job part tracker factory. Added a class that can create a job part tracker from a bulk object list. --- .../helpers/JobPartTrackerFactory.java | 45 ++++++++++++++ .../helpers/JobPartTrackerFactory_Test.java | 62 +++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTrackerFactory.java create mode 100644 sdk/src/test/java/com/spectralogic/ds3client/helpers/JobPartTrackerFactory_Test.java diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTrackerFactory.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTrackerFactory.java new file mode 100644 index 000000000..4ad568f58 --- /dev/null +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTrackerFactory.java @@ -0,0 +1,45 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Maps; +import com.google.common.collect.Maps.EntryTransformer; +import com.spectralogic.ds3client.models.bulk.BulkObject; + +import java.util.Collection; +import java.util.HashMap; + +class JobPartTrackerFactory { + public static JobPartTracker buildPartTracker(final Collection objects) { + final ArrayListMultimap multimap = ArrayListMultimap.create(); + for (final BulkObject bulkObject : objects) { + multimap.put(bulkObject.getName(), new ObjectPart(bulkObject.getOffset(), bulkObject.getLength())); + } + return new JobPartTracker(new HashMap<>(Maps.transformEntries( + multimap.asMap(), + new BuildObjectPartTrackerFromObjectPartGroup() + ))); + } + + private static final class BuildObjectPartTrackerFromObjectPartGroup + implements EntryTransformer, ObjectPartTracker> { + @Override + public ObjectPartTracker transformEntry(final String key, final Collection value) { + return new ObjectPartTrackerImpl(key, value); + } + } +} diff --git a/sdk/src/test/java/com/spectralogic/ds3client/helpers/JobPartTrackerFactory_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/helpers/JobPartTrackerFactory_Test.java new file mode 100644 index 000000000..996dc8566 --- /dev/null +++ b/sdk/src/test/java/com/spectralogic/ds3client/helpers/JobPartTrackerFactory_Test.java @@ -0,0 +1,62 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import com.spectralogic.ds3client.models.bulk.BulkObject; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +public class JobPartTrackerFactory_Test { + @Test + public void createdTrackerTracksParts() { + final JobPartTracker tracker = JobPartTrackerFactory.buildPartTracker(Arrays.asList( + new BulkObject("foo", 10L, false, 0L), + new BulkObject("bar", 13L, false, 0L), + new BulkObject("foo", 11L, true, 10L), + new BulkObject("baz", 12L, true, 0L) + )); + + final List transfers = new ArrayList<>(); + final List objects = new ArrayList<>(); + tracker.attachDataTransferredListener(new DataTransferredListener() { + @Override + public void dataTransferred(final long size) { + transfers.add(size); + } + }); + tracker.attachObjectCompletedListener(new ObjectCompletedListener() { + @Override + public void objectCompleted(final String name) { + objects.add(name); + } + }); + + tracker.completePart("baz", new ObjectPart(0L, 12L)); + tracker.completePart("foo", new ObjectPart(10L, 11L)); + tracker.completePart("bar", new ObjectPart(0L, 13L)); + tracker.completePart("foo", new ObjectPart(0L, 10L)); + + assertThat(transfers, is(Arrays.asList(12L, 11L, 13L, 10L))); + assertThat(objects, is(Arrays.asList("baz", "bar", "foo"))); + } +} From 8cd03e203831acaf2fafec8499bd9b8b5a61f164 Mon Sep 17 00:00:00 2001 From: Hans-Peter Klett Date: Wed, 17 Sep 2014 10:24:17 -0600 Subject: [PATCH 12/16] Set up the job system to use the new requests. --- .../com/spectralogic/ds3client/Ds3Client.java | 6 + .../{helpers => }/Ds3ClientFactory.java | 6 +- .../spectralogic/ds3client/Ds3ClientImpl.java | 13 ++ .../ds3client/commands/AbstractResponse.java | 2 +- .../helpers/ChunkTransferExecutor.java | 116 ++++++++++++++++ .../helpers/Ds3ClientFactoryImpl.java | 34 ----- .../ds3client/helpers/Ds3ClientHelpers.java | 2 +- .../helpers/Ds3ClientHelpersImpl.java | 7 +- .../ds3client/helpers/JobImpl.java | 117 ++--------------- .../helpers/JobPartTrackerFactory.java | 2 +- .../ds3client/helpers/JobState.java | 97 ++++++++++++++ .../ds3client/helpers/ReadJobImpl.java | 80 ++++++++--- .../ds3client/helpers/WriteJobImpl.java | 124 +++++++++++++++--- 13 files changed, 417 insertions(+), 189 deletions(-) rename sdk/src/main/java/com/spectralogic/ds3client/{helpers => }/Ds3ClientFactory.java (87%) create mode 100644 sdk/src/main/java/com/spectralogic/ds3client/helpers/ChunkTransferExecutor.java delete mode 100644 sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientFactoryImpl.java create mode 100644 sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java diff --git a/sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java b/sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java index ae71cd755..25d8950d4 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/Ds3Client.java @@ -16,6 +16,7 @@ package com.spectralogic.ds3client; import com.spectralogic.ds3client.commands.*; +import com.spectralogic.ds3client.models.bulk.Node; import java.io.IOException; import java.security.SignatureException; @@ -236,4 +237,9 @@ public abstract CancelJobResponse cancelJob(CancelJobRequest request) */ public abstract ModifyJobResponse modifyJob(ModifyJobRequest request) throws IOException, SignatureException; + + /** + * Creates a factory based on a set of nodes that can return clients by node id. + */ + public abstract Ds3ClientFactory buildFactory(Iterable nodes); } diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientFactory.java b/sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientFactory.java similarity index 87% rename from sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientFactory.java rename to sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientFactory.java index 80da252fb..4807d3636 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientFactory.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientFactory.java @@ -13,12 +13,10 @@ * **************************************************************************** */ -package com.spectralogic.ds3client.helpers; - -import com.spectralogic.ds3client.Ds3Client; +package com.spectralogic.ds3client; import java.util.UUID; -interface Ds3ClientFactory { +public interface Ds3ClientFactory { Ds3Client getClientForNodeId(UUID nodeId); } diff --git a/sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java index 7088aa743..91cca3654 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/Ds3ClientImpl.java @@ -16,10 +16,12 @@ package com.spectralogic.ds3client; import com.spectralogic.ds3client.commands.*; +import com.spectralogic.ds3client.models.bulk.Node; import com.spectralogic.ds3client.networking.NetworkClient; import java.io.IOException; import java.security.SignatureException; +import java.util.UUID; class Ds3ClientImpl implements Ds3Client { private final NetworkClient netClient; @@ -115,4 +117,15 @@ public CancelJobResponse cancelJob(final CancelJobRequest request) throws IOExce public ModifyJobResponse modifyJob(final ModifyJobRequest request) throws IOException, SignatureException { return new ModifyJobResponse(this.netClient.getResponse(request)); } + + @Override + public Ds3ClientFactory buildFactory(final Iterable nodes) { + return new Ds3ClientFactory() { + @Override + public Ds3Client getClientForNodeId(final UUID nodeId) { + //TODO: pay attention to actual nodes. + return Ds3ClientImpl.this; + } + }; + } } diff --git a/sdk/src/main/java/com/spectralogic/ds3client/commands/AbstractResponse.java b/sdk/src/main/java/com/spectralogic/ds3client/commands/AbstractResponse.java index 7aaddf014..3f5f477b1 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/commands/AbstractResponse.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/commands/AbstractResponse.java @@ -96,4 +96,4 @@ private String readResponseString() throws IOException { public String getMd5() { return md5; } -} \ No newline at end of file +} diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/ChunkTransferExecutor.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ChunkTransferExecutor.java new file mode 100644 index 000000000..da7253f59 --- /dev/null +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ChunkTransferExecutor.java @@ -0,0 +1,116 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.spectralogic.ds3client.Ds3Client; +import com.spectralogic.ds3client.Ds3ClientFactory; +import com.spectralogic.ds3client.models.bulk.BulkObject; +import com.spectralogic.ds3client.models.bulk.Node; +import com.spectralogic.ds3client.models.bulk.Objects; +import com.spectralogic.ds3client.serializer.XmlProcessingException; + +import java.io.IOException; +import java.security.SignatureException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; + +class ChunkTransferExecutor { + private final Transferrer transferrer; + private final Ds3Client mainClient; + private final JobPartTracker partTracker; + private final ListeningExecutorService executor; + + public static interface Transferrer { + void transferItem(Ds3Client client, BulkObject ds3Object) throws SignatureException, IOException; + } + + public ChunkTransferExecutor( + final Transferrer transferrer, + final Ds3Client mainClient, + final JobPartTracker partTracker, + final int maxParallelRequests) { + this.transferrer = transferrer; + this.mainClient = mainClient; + this.partTracker = partTracker; + this.executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(maxParallelRequests)); + } + + public void transferChunks( + final Iterable nodes, + final Iterable chunks) + throws SignatureException, IOException, XmlProcessingException { + try { + final List> tasks = new ArrayList<>(); + final Ds3ClientFactory clientFactory = this.mainClient.buildFactory(nodes); + for (final Objects chunk : chunks) { + final Ds3Client client = clientFactory.getClientForNodeId(chunk.getNodeId()); + for (final BulkObject ds3Object : chunk) { + final ObjectPart part = new ObjectPart(ds3Object.getOffset(), ds3Object.getLength()); + if (this.partTracker.containsPart(ds3Object.getName(), part)) { + tasks.add(this.executor.submit(new Callable() { + @Override + public Object call() throws Exception { + ChunkTransferExecutor.this.transferrer.transferItem(client, ds3Object); + ChunkTransferExecutor.this.partTracker.completePart(ds3Object.getName(), part); + return null; + } + })); + } + } + } + executeWithExceptionHandling(tasks); + } finally { + this.executor.shutdown(); + } + } + + private static void executeWithExceptionHandling(final List> tasks) + throws IOException, SignatureException, XmlProcessingException { + try { + // Block on the asynchronous result. + Futures.allAsList(tasks).get(); + } catch (final InterruptedException e) { + // This is a checked exception, but we don't want to expose that this is implemented with futures. + throw new RuntimeException(e); + } catch (final ExecutionException e) { + // The future throws a wrapper exception, but we want don't want to expose that this was implemented with futures. + final Throwable cause = e.getCause(); + + // Throw each of the advertised thrown exceptions. + if (cause instanceof IOException) { + throw (IOException)cause; + } else if (cause instanceof SignatureException) { + throw (SignatureException)cause; + } else if (cause instanceof XmlProcessingException) { + throw (XmlProcessingException)cause; + } + + // The rest we don't know about, so we'll just forward them. + if (cause instanceof RuntimeException) { + throw (RuntimeException)cause; + } else { + throw new RuntimeException(cause); + } + } + } +} diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientFactoryImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientFactoryImpl.java deleted file mode 100644 index 13099753a..000000000 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientFactoryImpl.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * ****************************************************************************** - * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. - * Licensed under the Apache License, Version 2.0 (the "License"). You may not use - * this file except in compliance with the License. A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. - * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR - * CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * **************************************************************************** - */ - -package com.spectralogic.ds3client.helpers; - -import com.spectralogic.ds3client.Ds3Client; - -import java.util.UUID; - -class Ds3ClientFactoryImpl implements Ds3ClientFactory { - private final Ds3Client client; - - Ds3ClientFactoryImpl(final Ds3Client client) { - this.client = client; - } - - //TODO: need to actually return a client that points to a particular server id. - @Override - public Ds3Client getClientForNodeId(final UUID nodeId) { - return this.client; - } -} diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java index cf9fea26e..bc2d57ccc 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers.java @@ -64,7 +64,7 @@ public interface Job { * @throws IOException * @throws XmlProcessingException */ - public void transfer(final ObjectChannelBuilder transferrer) + public void transfer(final ObjectChannelBuilder channelBuilder) throws SignatureException, IOException, XmlProcessingException; } diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java index c2894ab94..75a58cee6 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java @@ -23,7 +23,6 @@ import com.spectralogic.ds3client.models.Contents; import com.spectralogic.ds3client.models.ListBucketResult; import com.spectralogic.ds3client.models.bulk.Ds3Object; -import com.spectralogic.ds3client.models.bulk.MasterObjectList; import com.spectralogic.ds3client.serializer.XmlProcessingException; import java.io.IOException; @@ -70,8 +69,7 @@ private Ds3ClientHelpers.Job innerStartWriteJob(final String bucket, final BulkPutResponse prime = this.client.bulkPut(new BulkPutRequest(bucket, Lists.newArrayList(objectsToWrite)) .withPriority(options.getPriority()) .withWriteOptimization(options.getWriteOptimization())); - final MasterObjectList result = prime.getResult(); - return new WriteJobImpl(new Ds3ClientFactoryImpl(this.client), result.getJobId(), bucket, result.getObjects()); + return new WriteJobImpl(this.client, prime.getResult()); } @Override @@ -93,8 +91,7 @@ private Ds3ClientHelpers.Job innerStartReadJob(final String bucket, final Iterab throws SignatureException, IOException, XmlProcessingException { final BulkGetResponse prime = this.client.bulkGet(new BulkGetRequest(bucket, Lists.newArrayList(objectsToRead)) .withPriority(options.getPriority())); - final MasterObjectList result = prime.getResult(); - return new ReadJobImpl(new Ds3ClientFactoryImpl(this.client), result.getJobId(), bucket, result.getObjects()); + return new ReadJobImpl(this.client, prime.getResult()); } @Override diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java index 2f3145f6e..57766f257 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobImpl.java @@ -15,51 +15,30 @@ package com.spectralogic.ds3client.helpers; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; import com.spectralogic.ds3client.Ds3Client; import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.Job; -import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.ObjectChannelBuilder; -import com.spectralogic.ds3client.models.bulk.BulkObject; -import com.spectralogic.ds3client.models.bulk.Objects; -import com.spectralogic.ds3client.serializer.XmlProcessingException; +import com.spectralogic.ds3client.models.bulk.MasterObjectList; -import java.io.IOException; -import java.security.SignatureException; -import java.util.*; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; +import java.util.UUID; abstract class JobImpl implements Job { - private int maxParallelRequests = 20; - - private final Ds3ClientFactory clientFactory; - private final UUID jobId; - private final String bucketName; - private final Iterable objectLists; + protected int maxParallelRequests = 20; + protected final Ds3Client client; + protected final MasterObjectList masterObjectList; - public JobImpl( - final Ds3ClientFactory clientFactory, - final UUID jobId, - final String bucketName, - final Iterable objectLists) { - this.clientFactory = clientFactory; - this.jobId = jobId; - this.bucketName = bucketName; - this.objectLists = objectLists; + public JobImpl(final Ds3Client client, final MasterObjectList masterObjectList) { + this.client = client; + this.masterObjectList = masterObjectList; } @Override public UUID getJobId() { - return this.jobId; + return this.masterObjectList.getJobId(); } @Override public String getBucketName() { - return this.bucketName; + return this.masterObjectList.getBucketName(); } @Override @@ -67,80 +46,4 @@ public Job withMaxParallelRequests(final int maxParallelRequests) { this.maxParallelRequests = maxParallelRequests; return this; } - - @Override - public void transfer(final ObjectTransferrer transferrer) - throws SignatureException, IOException, XmlProcessingException { - for (final Objects chunk : this.objectLists) { - this.transferChunks(transferrer, Arrays.asList(chunk)); - } - } - - protected abstract void transferItem( - final Ds3Client client, - final UUID jobId, - final String bucketName, - final BulkObject ds3Object, - final ObjectTransferrer transferrer) throws SignatureException, IOException; - - private void transferChunks(final ObjectTransferrer transferrer, final Collection chunkList) - throws SignatureException, IOException, XmlProcessingException { - final ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(this.maxParallelRequests)); - final List> tasks = new ArrayList<>(); - try { - for (final Objects chunk : chunkList) { - final Ds3Client client = this.clientFactory.getClientForNodeId(chunk.getNodeId()); - for (final BulkObject ds3Object : chunk) { - tasks.add(this.createTransferTask(transferrer, service, client, ds3Object)); - } - } - this.executeWithExceptionHandling(tasks); - } finally { - service.shutdown(); - } - } - - private ListenableFuture createTransferTask( - final ObjectTransferrer transferrer, - final ListeningExecutorService service, - final Ds3Client client, - final BulkObject ds3Object) { - return service.submit(new Callable() { - @Override - public Object call() throws Exception { - transferItem(client, JobImpl.this.jobId, JobImpl.this.bucketName, ds3Object, transferrer); - return null; - } - }); - } - - private void executeWithExceptionHandling(final List> tasks) - throws IOException, SignatureException, XmlProcessingException { - try { - // Block on the asynchronous result. - Futures.allAsList(tasks).get(); - } catch (final InterruptedException e) { - // This is a checked exception, but we don't want to expose that this is implemented with futures. - throw new RuntimeException(e); - } catch (final ExecutionException e) { - // The future throws a wrapper exception, but we want don't want to expose that this was implemented with futures. - final Throwable cause = e.getCause(); - - // Throw each of the advertised thrown exceptions. - if (cause instanceof IOException) { - throw (IOException)cause; - } else if (cause instanceof SignatureException) { - throw (SignatureException)cause; - } else if (cause instanceof XmlProcessingException) { - throw (XmlProcessingException)cause; - } - - // The rest we don't know about, so we'll just forward them. - if (cause instanceof RuntimeException) { - throw (RuntimeException)cause; - } else { - throw new RuntimeException(cause); - } - } - } } diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTrackerFactory.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTrackerFactory.java index 4ad568f58..8a6e40ea4 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTrackerFactory.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTrackerFactory.java @@ -24,7 +24,7 @@ import java.util.HashMap; class JobPartTrackerFactory { - public static JobPartTracker buildPartTracker(final Collection objects) { + public static JobPartTracker buildPartTracker(final Iterable objects) { final ArrayListMultimap multimap = ArrayListMultimap.create(); for (final BulkObject bulkObject : objects) { multimap.put(bulkObject.getName(), new ObjectPart(bulkObject.getOffset(), bulkObject.getLength())); diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java new file mode 100644 index 000000000..f0500cd6b --- /dev/null +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java @@ -0,0 +1,97 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import com.google.common.collect.Iterables; +import com.spectralogic.ds3client.helpers.AutoCloseableCache.ValueBuilder; +import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.ObjectChannelBuilder; +import com.spectralogic.ds3client.models.bulk.BulkObject; +import com.spectralogic.ds3client.models.bulk.Objects; + +import java.io.IOException; +import java.nio.channels.SeekableByteChannel; +import java.util.Collection; +import java.util.HashSet; + +//TODO: concurrency +class JobState implements AutoCloseable { + private int objectsRemaining; + private final AutoCloseableCache channelCache; + private final JobPartTracker partTracker; + + public JobState(final ObjectChannelBuilder channelBuilder, final Collection filteredChunks) { + this.objectsRemaining = getObjectCount(filteredChunks); + this.channelCache = buildCache(channelBuilder); + this.partTracker = JobPartTrackerFactory + .buildPartTracker(Iterables.concat(filteredChunks)) + .attachObjectCompletedListener(new ObjectCompletedListenerImplementation()); + } + + public boolean hasObjects() { + return this.objectsRemaining > 0; + } + + private static int getObjectCount(final Collection chunks) { + final HashSet result = new HashSet<>(); + for (final Objects chunk : chunks) { + for (final BulkObject bulkObject : chunk.getObjects()) { + result.add(bulkObject.getName()); + } + } + return result.size(); + } + + private static AutoCloseableCache buildCache( + final ObjectChannelBuilder channelBuilder) { + return new AutoCloseableCache( + new ValueBuilder() { + @Override + public WindowedChannelFactory get(final String key) { + try { + return new WindowedChannelFactory(channelBuilder.buildChannel(key)); + } catch (final IOException e) { + throw new RuntimeException(e); + } + } + } + ); + } + + @Override + public void close() throws Exception { + this.channelCache.close(); + } + + public JobPartTracker getPartTracker() { + return partTracker; + } + + private final class ObjectCompletedListenerImplementation implements ObjectCompletedListener { + @Override + public void objectCompleted(final String name) { + JobState.this.objectsRemaining--; + try { + JobState.this.channelCache.close(name); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + } + + public SeekableByteChannel getChannel(final String name, final long offset, final long length) { + return this.channelCache.get(name).get(offset, length); + } +} diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java index ea2dbdf81..f3cf25230 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java @@ -16,39 +16,81 @@ package com.spectralogic.ds3client.helpers; import com.spectralogic.ds3client.Ds3Client; +import com.spectralogic.ds3client.commands.GetAvailableJobChunksRequest; +import com.spectralogic.ds3client.commands.GetAvailableJobChunksResponse; import com.spectralogic.ds3client.commands.GetObjectRequest; +import com.spectralogic.ds3client.helpers.ChunkTransferExecutor.Transferrer; import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.ObjectChannelBuilder; import com.spectralogic.ds3client.models.bulk.BulkObject; -import com.spectralogic.ds3client.models.bulk.Objects; +import com.spectralogic.ds3client.models.bulk.MasterObjectList; +import com.spectralogic.ds3client.serializer.XmlProcessingException; import java.io.IOException; import java.security.SignatureException; -import java.util.UUID; class ReadJobImpl extends JobImpl { - public ReadJobImpl( - final Ds3ClientFactory clientFactory, - final UUID jobId, - final String bucketName, - final Iterable objectLists) { - super(clientFactory, jobId, bucketName, objectLists); + public ReadJobImpl(final Ds3Client client, final MasterObjectList masterObjectList) { + super(client, masterObjectList); } @Override - protected void transferItem( - final Ds3Client client, - final UUID jobId, - final String bucketName, - final BulkObject ds3Object, - final ObjectChannelBuilder transferrer) - throws SignatureException, IOException { - client - .getObject(new GetObjectRequest( - bucketName, + public void transfer(final ObjectChannelBuilder channelBuilder) + throws SignatureException, IOException, XmlProcessingException { + try (final JobState jobState = new JobState(channelBuilder, this.masterObjectList.getObjects())) { + final ChunkTransferExecutor executor = new ChunkTransferExecutor( + new GetObjectTransferrer(jobState), + this.client, + jobState.getPartTracker(), + this.maxParallelRequests + ); + while (jobState.hasObjects()) { + transferNextChunks(executor); + } + } catch (final SignatureException | IOException | XmlProcessingException e) { + throw e; + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + private void transferNextChunks(final ChunkTransferExecutor executor) + throws IOException, SignatureException, XmlProcessingException, + InterruptedException { + final GetAvailableJobChunksResponse availableJobChunks = + this.client.getAvailableJobChunks(new GetAvailableJobChunksRequest(this.masterObjectList.getJobId())); + switch(availableJobChunks.getStatus()) { + case AVAILABLE: + final MasterObjectList availableMol = availableJobChunks.getMasterObjectList(); + executor.transferChunks(availableMol.getNodes(), availableMol.getObjects()); + break; + case NOTFOUND: + //TODO: this is a job cancellation condition. + throw new IllegalStateException(); + case RETRYLATER: + Thread.sleep(availableJobChunks.getRetryAfterSeconds() * 1000); + break; + default: + assert false : "This line of code should be impossible to hit."; + } + } + + private final class GetObjectTransferrer implements Transferrer { + private final JobState jobState; + + private GetObjectTransferrer(final JobState jobState) { + this.jobState = jobState; + } + + @Override + public void transferItem(final Ds3Client client, final BulkObject ds3Object) + throws SignatureException, IOException { + client.getObject(new GetObjectRequest( + ReadJobImpl.this.masterObjectList.getBucketName(), ds3Object.getName(), ds3Object.getOffset(), ReadJobImpl.this.getJobId(), - transferrer.buildChannel(ds3Object.getName()) + jobState.getChannel(ds3Object.getName(), ds3Object.getOffset(), ds3Object.getLength()) )); + } } } diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java index c568f2df2..615955b5d 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java @@ -16,39 +16,129 @@ package com.spectralogic.ds3client.helpers; import com.spectralogic.ds3client.Ds3Client; +import com.spectralogic.ds3client.commands.AllocateJobChunkRequest; +import com.spectralogic.ds3client.commands.AllocateJobChunkResponse; import com.spectralogic.ds3client.commands.PutObjectRequest; +import com.spectralogic.ds3client.helpers.ChunkTransferExecutor.Transferrer; import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.ObjectChannelBuilder; import com.spectralogic.ds3client.models.bulk.BulkObject; +import com.spectralogic.ds3client.models.bulk.MasterObjectList; import com.spectralogic.ds3client.models.bulk.Objects; +import com.spectralogic.ds3client.serializer.XmlProcessingException; + +import sun.reflect.generics.reflectiveObjects.NotImplementedException; import java.io.IOException; import java.security.SignatureException; -import java.util.UUID; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; class WriteJobImpl extends JobImpl { public WriteJobImpl( - final Ds3ClientFactory clientFactory, - final UUID jobId, - final String bucketName, - final Iterable objectLists) { - super(clientFactory, jobId, bucketName, objectLists); + final Ds3Client client, + final MasterObjectList masterObjectList) { + super(client, masterObjectList); } @Override - protected void transferItem( - final Ds3Client client, - final UUID jobId, - final String bucketName, - final BulkObject ds3Object, - final ObjectChannelBuilder transferrer) throws SignatureException, IOException { - client - .putObject(new PutObjectRequest( - bucketName, + public void transfer(final ObjectChannelBuilder channelBuilder) + throws SignatureException, IOException, XmlProcessingException { + final List filteredChunks = filterChunks(this.masterObjectList.getObjects()); + try (final JobState jobState = new JobState(channelBuilder, filteredChunks)) { + final ChunkTransferExecutor executor = new ChunkTransferExecutor( + new PutObjectTransferrer(jobState), + this.client, + jobState.getPartTracker(), + this.maxParallelRequests + ); + for (final Objects chunk : filteredChunks) { + executor.transferChunks(this.masterObjectList.getNodes(), Arrays.asList(filterChunk(allocateChunk(chunk)))); + } + } catch (final SignatureException | IOException | XmlProcessingException e) { + throw e; + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + private Objects allocateChunk(final Objects filtered) throws IOException, SignatureException { + Objects chunk = null; + while (chunk == null) { + chunk = tryAllocateChunk(filtered); + } + return chunk; + } + + private Objects tryAllocateChunk(final Objects filtered) throws IOException, SignatureException { + final AllocateJobChunkResponse response = + this.client.allocateJobChunk(new AllocateJobChunkRequest(filtered.getChunkId())); + switch (response.getStatus()) { + case ALLOCATED: + return response.getObjects(); + case NOTFOUND: + //TODO: But why is the rum gone? + throw new NotImplementedException(); + case RETRYLATER: + try { + Thread.sleep(response.getRetryAfterSeconds() * 1000); + return null; + } catch (final InterruptedException e) { + throw new RuntimeException(e); + } + default: + assert false : "This line of code should be impossible to hit."; return null; + } + } + + private static List filterChunks(final List chunks) { + final List filteredChunks = new ArrayList<>(); + for (final Objects chunk : chunks) { + final Objects filteredChunk = filterChunk(chunk); + if (filteredChunk.getObjects().size() > 0) { + filteredChunks.add(filteredChunk); + } + } + return filteredChunks; + } + + private static Objects filterChunk(final Objects chunk) { + final Objects newChunk = new Objects(); + newChunk.setChunkId(chunk.getChunkId()); + newChunk.setChunkNumber(chunk.getChunkNumber()); + newChunk.setNodeId(chunk.getNodeId()); + newChunk.setObjects(filterObjects(chunk.getObjects())); + return newChunk; + } + + private static List filterObjects(final List list) { + final List filtered = new ArrayList<>(); + for (final BulkObject obj : list) { + if (!obj.isInCache()) { + filtered.add(obj); + } + } + return filtered; + } + + private final class PutObjectTransferrer implements Transferrer { + private final JobState jobState; + + private PutObjectTransferrer(final JobState jobState) { + this.jobState = jobState; + } + + @Override + public void transferItem(final Ds3Client client, final BulkObject ds3Object) + throws SignatureException, IOException { + client.putObject(new PutObjectRequest( + WriteJobImpl.this.masterObjectList.getBucketName(), ds3Object.getName(), - jobId, + WriteJobImpl.this.getJobId(), ds3Object.getLength(), ds3Object.getOffset(), - transferrer.buildChannel(ds3Object.getName()) + jobState.getChannel(ds3Object.getName(), ds3Object.getOffset(), ds3Object.getLength()) )); + } } } From 2b8f4364eb56729124772f922a0228249dd3ea7e Mon Sep 17 00:00:00 2001 From: Hans-Peter Klett Date: Thu, 18 Sep 2014 14:04:22 -0600 Subject: [PATCH 13/16] Re-wrote tests for the helper functions. --- sdk/build.gradle | 1 + .../commands/AllocateJobChunkResponse.java | 7 +- .../GetAvailableJobChunksResponse.java | 7 +- .../helpers/ChunkTransferExecutor.java | 9 +- .../ds3client/helpers/ReadJobImpl.java | 8 +- .../ds3client/helpers/WriteJobImpl.java | 7 +- .../ds3client/Ds3Client_Test.java | 30 +- .../helpers/Ds3ClientHelpers_Test.java | 456 ++++++++++-------- .../ds3client/helpers/RequestMatchers.java | 198 ++++++++ .../ds3client/helpers/ResponseBuilders.java | 186 +++++++ 10 files changed, 643 insertions(+), 266 deletions(-) create mode 100644 sdk/src/test/java/com/spectralogic/ds3client/helpers/RequestMatchers.java create mode 100644 sdk/src/test/java/com/spectralogic/ds3client/helpers/ResponseBuilders.java diff --git a/sdk/build.gradle b/sdk/build.gradle index 2c3d3b1f6..9f40ada7a 100644 --- a/sdk/build.gradle +++ b/sdk/build.gradle @@ -9,4 +9,5 @@ dependencies { compile 'org.codehaus.woodstox:woodstox-core-asl:4.2.0' compile 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.2.3' compile 'com.google.guava:guava:16.0.1' + compile 'org.hamcrest:hamcrest-library:1.3' } diff --git a/sdk/src/main/java/com/spectralogic/ds3client/commands/AllocateJobChunkResponse.java b/sdk/src/main/java/com/spectralogic/ds3client/commands/AllocateJobChunkResponse.java index 4e635e2e7..a553e0b2a 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/commands/AllocateJobChunkResponse.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/commands/AllocateJobChunkResponse.java @@ -31,7 +31,7 @@ public class AllocateJobChunkResponse extends AbstractResponse { private int retryAfterSeconds; static public enum Status { - ALLOCATED, RETRYLATER, NOTFOUND + ALLOCATED, RETRYLATER } public AllocateJobChunkResponse(final WebResponse response) throws IOException { @@ -53,7 +53,7 @@ public int getRetryAfterSeconds() { @Override protected void processResponse() throws IOException { try (final WebResponse response = this.getResponse()) { - checkStatusCode(200, 503, 404); + checkStatusCode(200, 503); switch (this.getStatusCode()) { case 200: this.status = Status.ALLOCATED; @@ -63,9 +63,6 @@ protected void processResponse() throws IOException { this.status = Status.RETRYLATER; this.retryAfterSeconds = parseRetryAfter(response); break; - case 404: - this.status = Status.NOTFOUND; - break; default: assert false : "checkStatusCode should have made it impossible to reach this line."; } diff --git a/sdk/src/main/java/com/spectralogic/ds3client/commands/GetAvailableJobChunksResponse.java b/sdk/src/main/java/com/spectralogic/ds3client/commands/GetAvailableJobChunksResponse.java index 0721c9ea7..d7790ac88 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/commands/GetAvailableJobChunksResponse.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/commands/GetAvailableJobChunksResponse.java @@ -31,7 +31,7 @@ public class GetAvailableJobChunksResponse extends AbstractResponse { private int retryAfterSeconds; static public enum Status { - AVAILABLE, RETRYLATER, NOTFOUND + AVAILABLE, RETRYLATER } public Status getStatus() { @@ -53,7 +53,7 @@ public GetAvailableJobChunksResponse(final WebResponse response) throws IOExcept @Override protected void processResponse() throws IOException { try (final WebResponse webResponse = this.getResponse()) { - this.checkStatusCode(200, 404); + this.checkStatusCode(200); switch (this.getStatusCode()) { case 200: this.masterObjectList = parseMasterObjectList(webResponse); @@ -64,9 +64,6 @@ protected void processResponse() throws IOException { this.status = Status.AVAILABLE; } break; - case 404: - this.status = Status.NOTFOUND; - break; default: assert false : "checkStatusCode should have made it impossible to reach this line."; } diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/ChunkTransferExecutor.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ChunkTransferExecutor.java index da7253f59..3ccfc8378 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/ChunkTransferExecutor.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ChunkTransferExecutor.java @@ -38,7 +38,7 @@ class ChunkTransferExecutor { private final Transferrer transferrer; private final Ds3Client mainClient; private final JobPartTracker partTracker; - private final ListeningExecutorService executor; + private final int maxParallelRequests; public static interface Transferrer { void transferItem(Ds3Client client, BulkObject ds3Object) throws SignatureException, IOException; @@ -52,13 +52,14 @@ public ChunkTransferExecutor( this.transferrer = transferrer; this.mainClient = mainClient; this.partTracker = partTracker; - this.executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(maxParallelRequests)); + this.maxParallelRequests = maxParallelRequests; } public void transferChunks( final Iterable nodes, final Iterable chunks) throws SignatureException, IOException, XmlProcessingException { + final ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(maxParallelRequests)); try { final List> tasks = new ArrayList<>(); final Ds3ClientFactory clientFactory = this.mainClient.buildFactory(nodes); @@ -67,7 +68,7 @@ public void transferChunks( for (final BulkObject ds3Object : chunk) { final ObjectPart part = new ObjectPart(ds3Object.getOffset(), ds3Object.getLength()); if (this.partTracker.containsPart(ds3Object.getName(), part)) { - tasks.add(this.executor.submit(new Callable() { + tasks.add(executor.submit(new Callable() { @Override public Object call() throws Exception { ChunkTransferExecutor.this.transferrer.transferItem(client, ds3Object); @@ -80,7 +81,7 @@ public Object call() throws Exception { } executeWithExceptionHandling(tasks); } finally { - this.executor.shutdown(); + executor.shutdown(); } } diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java index f3cf25230..b695016a3 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ReadJobImpl.java @@ -48,14 +48,15 @@ public void transfer(final ObjectChannelBuilder channelBuilder) } } catch (final SignatureException | IOException | XmlProcessingException e) { throw e; + } catch (final RuntimeException e) { + throw e; } catch (final Exception e) { throw new RuntimeException(e); } } private void transferNextChunks(final ChunkTransferExecutor executor) - throws IOException, SignatureException, XmlProcessingException, - InterruptedException { + throws IOException, SignatureException, XmlProcessingException, InterruptedException { final GetAvailableJobChunksResponse availableJobChunks = this.client.getAvailableJobChunks(new GetAvailableJobChunksRequest(this.masterObjectList.getJobId())); switch(availableJobChunks.getStatus()) { @@ -63,9 +64,6 @@ private void transferNextChunks(final ChunkTransferExecutor executor) final MasterObjectList availableMol = availableJobChunks.getMasterObjectList(); executor.transferChunks(availableMol.getNodes(), availableMol.getObjects()); break; - case NOTFOUND: - //TODO: this is a job cancellation condition. - throw new IllegalStateException(); case RETRYLATER: Thread.sleep(availableJobChunks.getRetryAfterSeconds() * 1000); break; diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java index 615955b5d..fb26f8fd7 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/WriteJobImpl.java @@ -26,8 +26,6 @@ import com.spectralogic.ds3client.models.bulk.Objects; import com.spectralogic.ds3client.serializer.XmlProcessingException; -import sun.reflect.generics.reflectiveObjects.NotImplementedException; - import java.io.IOException; import java.security.SignatureException; import java.util.ArrayList; @@ -57,6 +55,8 @@ public void transfer(final ObjectChannelBuilder channelBuilder) } } catch (final SignatureException | IOException | XmlProcessingException e) { throw e; + } catch (final RuntimeException e) { + throw e; } catch (final Exception e) { throw new RuntimeException(e); } @@ -76,9 +76,6 @@ private Objects tryAllocateChunk(final Objects filtered) throws IOException, Sig switch (response.getStatus()) { case ALLOCATED: return response.getObjects(); - case NOTFOUND: - //TODO: But why is the rum gone? - throw new NotImplementedException(); case RETRYLATER: try { Thread.sleep(response.getRetryAfterSeconds() * 1000); diff --git a/sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java index a0df1a85e..87395cea6 100644 --- a/sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java +++ b/sdk/src/test/java/com/spectralogic/ds3client/Ds3Client_Test.java @@ -40,7 +40,7 @@ import java.util.*; import static org.hamcrest.CoreMatchers.*; -import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.MatcherAssert.*; public class Ds3Client_Test { private static final UUID MASTER_OBJECT_LIST_JOB_ID = UUID.fromString("1a85e743-ec8f-4789-afec-97e587a26936"); @@ -387,19 +387,6 @@ public void allocateJobChunk() throws SignatureException, IOException { assertThat(object3.getLength(), is(5368709120L)); } - @Test - public void allocateJobChunkReturnsChunkNotFound() throws SignatureException, IOException { - final Map queryParams = new HashMap<>(); - queryParams.put("operation", "allocate"); - final AllocateJobChunkResponse response = MockNetwork - .expecting(HttpVerb.PUT, "/_rest_/job_chunk/203f6886-b058-4f7c-a012-8779176453b1", queryParams, null) - .returning(404, "") - .asClient() - .allocateJobChunk(new AllocateJobChunkRequest(UUID.fromString("203f6886-b058-4f7c-a012-8779176453b1"))); - - assertThat(response.getStatus(), is(AllocateJobChunkResponse.Status.NOTFOUND)); - } - @Test public void allocateJobChunkReturnsRetryAfter() throws SignatureException, IOException { final Map queryParams = new HashMap<>(); @@ -457,21 +444,6 @@ public void getAvailableJobChunksReturnsRetryLater() throws SignatureException, assertThat(response.getRetryAfterSeconds(), is(300)); } - @Test - public void getAvailableJobChunksReturnsNotFound() throws SignatureException, IOException { - final UUID jobId = UUID.fromString("1a85e743-ec8f-4789-afec-97e587a26936"); - - final Map queryParams = new HashMap<>(); - queryParams.put("job", jobId.toString()); - final GetAvailableJobChunksResponse response = MockNetwork - .expecting(HttpVerb.GET, "/_rest_/job_chunk", queryParams, null) - .returning(404, "") - .asClient() - .getAvailableJobChunks(new GetAvailableJobChunksRequest(jobId)); - - assertThat(response.getStatus(), is(GetAvailableJobChunksResponse.Status.NOTFOUND)); - } - @Test public void getJobs() throws SignatureException, IOException { final String responseString = diff --git a/sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java index 39d4901fe..6c2ee4e1e 100644 --- a/sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java +++ b/sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java @@ -15,170 +15,220 @@ package com.spectralogic.ds3client.helpers; +import com.google.common.base.Stopwatch; import com.google.common.collect.Lists; import com.spectralogic.ds3client.Ds3Client; +import com.spectralogic.ds3client.Ds3ClientFactory; import com.spectralogic.ds3client.commands.*; import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.Job; import com.spectralogic.ds3client.helpers.Ds3ClientHelpers.ObjectChannelBuilder; import com.spectralogic.ds3client.models.Contents; import com.spectralogic.ds3client.models.ListBucketResult; import com.spectralogic.ds3client.models.Owner; -import com.spectralogic.ds3client.models.bulk.BulkObject; -import com.spectralogic.ds3client.models.bulk.Ds3Object; -import com.spectralogic.ds3client.models.bulk.MasterObjectList; +import com.spectralogic.ds3client.models.bulk.*; import com.spectralogic.ds3client.models.bulk.Objects; import com.spectralogic.ds3client.serializer.XmlProcessingException; import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel; import org.junit.Assert; import org.junit.Test; -import org.mockito.ArgumentMatcher; -import org.mockito.InOrder; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; +import org.mockito.Mockito; import java.io.IOException; -import java.io.Writer; -import java.nio.channels.Channels; import java.nio.channels.SeekableByteChannel; -import java.nio.channels.WritableByteChannel; import java.security.SignatureException; import java.util.*; +import java.util.concurrent.TimeUnit; -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.argThat; -import static org.mockito.Mockito.*; - +import static com.spectralogic.ds3client.helpers.RequestMatchers.*; +import static com.spectralogic.ds3client.helpers.ResponseBuilders.*; +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.*; public class Ds3ClientHelpers_Test { private static final String MYBUCKET = "mybucket"; - + @Test public void testReadObjects() throws SignatureException, IOException, XmlProcessingException { - final Ds3Client ds3Client = mock(Ds3Client.class); - when(ds3Client.bulkGet(any(BulkGetRequest.class))).thenReturn(new StubBulkGetResponse()); - when(ds3Client.getObject(getRequestHas("foo"))).then(new GetObjectAnswer("foo")); - when(ds3Client.getObject(getRequestHas("bar"))).then(new GetObjectAnswer("bar")); - when(ds3Client.getObject(getRequestHas("baz"))).then(new GetObjectAnswer("baz")); + final Ds3Client ds3Client = buildDs3ClientForBulk(); + + final BulkGetResponse buildBulkGetResponse = buildBulkGetResponse(); + Mockito.when(ds3Client.bulkGet(Mockito.any(BulkGetRequest.class))).thenReturn(buildBulkGetResponse); + + Mockito.when(ds3Client.getObject(getRequestHas(MYBUCKET, "foo", jobId, 0))).then(getObjectAnswer("foo co")); + Mockito.when(ds3Client.getObject(getRequestHas(MYBUCKET, "bar", jobId, 0))).then(getObjectAnswer("bar contents")); + Mockito.when(ds3Client.getObject(getRequestHas(MYBUCKET, "baz", jobId, 0))).then(getObjectAnswer("baz co")); + Mockito.when(ds3Client.getObject(getRequestHas(MYBUCKET, "foo", jobId, 6))).then(getObjectAnswer("ntents")); + Mockito.when(ds3Client.getObject(getRequestHas(MYBUCKET, "baz", jobId, 6))).then(getObjectAnswer("ntents")); - final List objectsToGet = Lists.newArrayList( + final Job job = Ds3ClientHelpers.wrap(ds3Client).startReadJob(MYBUCKET, Lists.newArrayList( new Ds3Object("foo"), new Ds3Object("bar"), new Ds3Object("baz") - ); - final Job readJob = Ds3ClientHelpers.wrap(ds3Client).startReadJob(MYBUCKET, objectsToGet); + )); - assertThat(readJob.getJobId(), is(jobId)); - assertThat(readJob.getBucketName(), is(MYBUCKET)); + assertThat(job.getJobId(), is(jobId)); + assertThat(job.getBucketName(), is(MYBUCKET)); final HashMap channelMap = new HashMap<>(); channelMap.put("foo", new ByteArraySeekableByteChannel()); channelMap.put("bar", new ByteArraySeekableByteChannel()); channelMap.put("baz", new ByteArraySeekableByteChannel()); - readJob.transfer(new ObjectChannelBuilder() { + final Stopwatch stopwatch = Stopwatch.createStarted(); + job.transfer(new ObjectChannelBuilder() { @Override public SeekableByteChannel buildChannel(final String key) throws IOException { return channelMap.get(key); } }); + assertThat(stopwatch.stop().elapsed(TimeUnit.MILLISECONDS), is(both(greaterThan(1000L)).and(lessThan(1250L)))); for (final Map.Entry channelEntry : channelMap.entrySet()) { assertThat(channelEntry.getValue().toString(), is(channelEntry.getKey() + " contents")); } + } + + @Test + public void testReadObjectsWithFailedGet() throws SignatureException, IOException, XmlProcessingException { + final Ds3Client ds3Client = Mockito.mock(Ds3Client.class); + + final Ds3ClientFactory ds3ClientFactory = ds3ClientFactory(ds3Client); + Mockito.when(ds3Client.buildFactory(Mockito.>any())).thenReturn(ds3ClientFactory); + + final BulkGetResponse buildBulkGetResponse = buildBulkGetResponse(); + Mockito.when(ds3Client.bulkGet(Mockito.any(BulkGetRequest.class))).thenReturn(buildBulkGetResponse); + + final GetAvailableJobChunksResponse jobChunksResponse = buildJobChunksResponse2(); + Mockito.when(ds3Client.getAvailableJobChunks(hasJobId(jobId))).thenReturn(jobChunksResponse); + + Mockito.when(ds3Client.getObject(getRequestHas(MYBUCKET, "foo", jobId, 6))).thenThrow(new StubException()); + Mockito.when(ds3Client.getObject(getRequestHas(MYBUCKET, "baz", jobId, 6))).then(getObjectAnswer("ntents")); - final InOrder clientInOrder = inOrder(ds3Client); - clientInOrder.verify(ds3Client).bulkGet(any(BulkGetRequest.class)); - clientInOrder.verify(ds3Client).getObject(getRequestHas("baz")); - clientInOrder.verify(ds3Client).getObject(getRequestHas("foo")); - clientInOrder.verify(ds3Client).getObject(getRequestHas("bar")); - clientInOrder.verifyNoMoreInteractions(); + final Job job = Ds3ClientHelpers.wrap(ds3Client).startReadJob(MYBUCKET, Lists.newArrayList( + new Ds3Object("foo"), + new Ds3Object("bar"), + new Ds3Object("baz") + )); + + try { + job.transfer(new ObjectChannelBuilder() { + @Override + public SeekableByteChannel buildChannel(final String key) throws IOException { + // We don't care about the contents since we just want to know that the exception handling works correctly. + return new ByteArraySeekableByteChannel(); + } + }); + Assert.fail("Should have failed with an exception before we got here."); + } catch (final StubException e) { + } } @Test public void testWriteObjects() throws SignatureException, IOException, XmlProcessingException { - final Ds3Client ds3Client = mock(Ds3Client.class); - when(ds3Client.bulkPut(any(BulkPutRequest.class))).thenReturn(new StubBulkPutResponse()); - when(ds3Client.putObject(any(PutObjectRequest.class))).thenReturn(new StubPutObjectResponse()); + final Ds3Client ds3Client = buildDs3ClientForBulk(); + + final BulkPutResponse bulkPutResponse = buildBulkPutResponse(); + Mockito.when(ds3Client.bulkPut(Mockito.any(BulkPutRequest.class))).thenReturn(bulkPutResponse); + + final AllocateJobChunkResponse allocateResponse1 = buildAllocateResponse1(); + final AllocateJobChunkResponse allocateResponse2 = buildAllocateResponse2(); + final AllocateJobChunkResponse allocateResponse3 = buildAllocateResponse3(); + Mockito.when(ds3Client.allocateJobChunk(hasChunkId(CHUNK_ID_1))) + .thenReturn(allocateResponse1) + .thenReturn(allocateResponse2); + Mockito.when(ds3Client.allocateJobChunk(hasChunkId(CHUNK_ID_2))) + .thenReturn(allocateResponse3); + + final PutObjectResponse response = Mockito.mock(PutObjectResponse.class); + Mockito.when(ds3Client.putObject(putRequestHas(MYBUCKET, "foo", jobId, 0, "foo co"))).thenReturn(response); + Mockito.when(ds3Client.putObject(putRequestHas(MYBUCKET, "bar", jobId, 0, "bar contents"))).thenReturn(response); + Mockito.when(ds3Client.putObject(putRequestHas(MYBUCKET, "baz", jobId, 0, "baz co"))).thenReturn(response); + Mockito.when(ds3Client.putObject(putRequestHas(MYBUCKET, "foo", jobId, 6, "ntents"))).thenReturn(response); + Mockito.when(ds3Client.putObject(putRequestHas(MYBUCKET, "baz", jobId, 6, "ntents"))).thenReturn(response); - final Iterable objectsToPut = Lists.newArrayList( + final Job job = Ds3ClientHelpers.wrap(ds3Client).startWriteJob(MYBUCKET, Lists.newArrayList( new Ds3Object("foo", 12), new Ds3Object("bar", 12), new Ds3Object("baz", 12) - ); - final Job job = Ds3ClientHelpers.wrap(ds3Client).startWriteJob(MYBUCKET, objectsToPut); + )); assertThat(job.getJobId(), is(jobId)); assertThat(job.getBucketName(), is(MYBUCKET)); - final HashMap channels = new HashMap<>(); - channels.put("baz", mock(SeekableByteChannel.class)); - channels.put("foo", mock(SeekableByteChannel.class)); - channels.put("bar", mock(SeekableByteChannel.class)); + final HashMap channelMap = new HashMap<>(); + channelMap.put("foo", channelWithContents("foo contents")); + channelMap.put("bar", channelWithContents("bar contents")); + channelMap.put("baz", channelWithContents("baz contents")); + final Stopwatch stopwatch = Stopwatch.createStarted(); job.transfer(new ObjectChannelBuilder() { @Override public SeekableByteChannel buildChannel(final String key) throws IOException { - return channels.get(key); + return channelMap.get(key); } }); - - final InOrder clientInOrder = inOrder(ds3Client); - clientInOrder.verify(ds3Client).bulkPut(any(BulkPutRequest.class)); - clientInOrder.verify(ds3Client).putObject(putRequestHas("baz", channels.get("baz"))); - clientInOrder.verify(ds3Client).putObject(putRequestHas("foo", channels.get("foo"))); - clientInOrder.verify(ds3Client).putObject(putRequestHas("bar", channels.get("bar"))); - clientInOrder.verifyNoMoreInteractions(); + assertThat(stopwatch.stop().elapsed(TimeUnit.MILLISECONDS), is(both(greaterThan(1000L)).and(lessThan(1250L)))); } - + @Test - public void testListObjects() throws SignatureException, IOException, XmlProcessingException { - final Ds3Client ds3Client = mock(Ds3Client.class); - when(ds3Client.getBucket(getBucketHas(null))).thenReturn(new StubGetBucketResponse(0)); - when(ds3Client.getBucket(getBucketHas("baz"))).thenReturn(new StubGetBucketResponse(1)); - - // Call the list objects method. - final List contentList = Lists.newArrayList(Ds3ClientHelpers.wrap(ds3Client).listObjects(MYBUCKET)); - - // Check the results. - assertThat(contentList.size(), is(3)); - checkContents(contentList.get(0), "foo", "2cde576e5f5a613e6cee466a681f4929", "2009-10-12T17:50:30.000Z", 12); - checkContents(contentList.get(1), "bar", "f3f98ff00be128139332bcf4b772be43", "2009-10-14T17:50:31.000Z", 12); - checkContents(contentList.get(2), "baz", "802d45fcb9a3f7d00f1481362edc0ec9", "2009-10-18T17:50:35.000Z", 12); - } + public void testWriteObjectsWithFailedPut() throws SignatureException, IOException, XmlProcessingException { + final Ds3Client ds3Client = Mockito.mock(Ds3Client.class); - @Test - public void testReadObjectsWithFailedPut() throws SignatureException, IOException, XmlProcessingException { - final Ds3Client ds3Client = mock(Ds3Client.class); - when(ds3Client.bulkGet(any(BulkGetRequest.class))).thenReturn(new StubBulkGetResponse()); - when(ds3Client.getObject(getRequestHas("foo"))).thenThrow(new StubException()); - when(ds3Client.getObject(getRequestHas("bar"))).thenThrow(new StubException()); - when(ds3Client.getObject(getRequestHas("baz"))).then(new GetObjectAnswer("baz")); + final Ds3ClientFactory ds3ClientFactory = ds3ClientFactory(ds3Client); + Mockito.when(ds3Client.buildFactory(Mockito.>any())).thenReturn(ds3ClientFactory); + + final BulkPutResponse buildBulkPutResponse = buildBulkPutResponse(); + Mockito.when(ds3Client.bulkPut(Mockito.any(BulkPutRequest.class))).thenReturn(buildBulkPutResponse); + + final AllocateJobChunkResponse allocateResponse = buildAllocateResponse2(); + Mockito.when(ds3Client.allocateJobChunk(hasChunkId(CHUNK_ID_1))).thenReturn(allocateResponse); + + final PutObjectResponse putResponse = Mockito.mock(PutObjectResponse.class); + Mockito.when(ds3Client.putObject(putRequestHas(MYBUCKET, "foo", jobId, 0, "foo co"))).thenThrow(new StubException()); + Mockito.when(ds3Client.putObject(putRequestHas(MYBUCKET, "baz", jobId, 0, "baz co"))).thenReturn(putResponse); - // Build input list. - final ArrayList objectsToGet = Lists.newArrayList( + final Job job = Ds3ClientHelpers.wrap(ds3Client).startWriteJob(MYBUCKET, Lists.newArrayList( new Ds3Object("foo"), new Ds3Object("bar"), new Ds3Object("baz") - ); - - final Job job = Ds3ClientHelpers.wrap(ds3Client).startReadJob(MYBUCKET, objectsToGet); + )); + final HashMap channelMap = new HashMap<>(); + channelMap.put("foo", channelWithContents("foo contents")); + channelMap.put("bar", channelWithContents("bar contents")); + channelMap.put("baz", channelWithContents("baz contents")); + try { job.transfer(new ObjectChannelBuilder() { @Override public SeekableByteChannel buildChannel(final String key) throws IOException { - // We don't care about the contents since we just want to know that the exception handling works correctly. - return new ByteArraySeekableByteChannel(); + return channelMap.get(key); } }); + Assert.fail("Should have failed with an exception before we got here."); } catch (final StubException e) { - // This is what we want. - return; } - Assert.fail("Should have failed with an exception before we got here."); + } + + private static final class StubException extends RuntimeException { + private static final long serialVersionUID = 5121719894916333278L; + } + + @Test + public void testListObjects() throws SignatureException, IOException, XmlProcessingException { + final Ds3Client ds3Client = Mockito.mock(Ds3Client.class); + Mockito.when(ds3Client.getBucket(getBucketHas(MYBUCKET, null))).thenReturn(new StubGetBucketResponse(0)); + Mockito.when(ds3Client.getBucket(getBucketHas(MYBUCKET, "baz"))).thenReturn(new StubGetBucketResponse(1)); + + // Call the list objects method. + final List contentList = Lists.newArrayList(Ds3ClientHelpers.wrap(ds3Client).listObjects(MYBUCKET)); + + // Check the results. + assertThat(contentList.size(), is(3)); + checkContents(contentList.get(0), "foo", "2cde576e5f5a613e6cee466a681f4929", "2009-10-12T17:50:30.000Z", 12); + checkContents(contentList.get(1), "bar", "f3f98ff00be128139332bcf4b772be43", "2009-10-14T17:50:31.000Z", 12); + checkContents(contentList.get(2), "baz", "802d45fcb9a3f7d00f1481362edc0ec9", "2009-10-18T17:50:35.000Z", 12); } private static void checkContents( @@ -192,74 +242,7 @@ private static void checkContents( assertThat(contents.getLastModified(), is(lastModified)); assertThat(contents.getSize(), is(size)); } - - private static GetObjectRequest getRequestHas(final String key) { - return argThat(new ArgumentMatcher() { - @Override - public boolean matches(final Object argument) { - if (!(argument instanceof GetObjectRequest)) { - return false; - } - final GetObjectRequest getObjectRequest = (GetObjectRequest)argument; - return - getObjectRequest.getBucketName().equals(MYBUCKET) - && getObjectRequest.getObjectName().equals(key); - } - }); - } - - private static PutObjectRequest putRequestHas(final String key, final SeekableByteChannel channel) { - return argThat(new ArgumentMatcher() { - @Override - public boolean matches(final Object argument) { - if (!(argument instanceof PutObjectRequest)) { - return false; - } - final PutObjectRequest putObjectRequest = (PutObjectRequest)argument; - return - putObjectRequest.getBucketName().equals(MYBUCKET) - && putObjectRequest.getObjectName().equals(key) - && putObjectRequest.getChannel() == channel; - } - }); - } - - private static GetBucketRequest getBucketHas(final String marker) { - return argThat(new ArgumentMatcher() { - @Override - public boolean matches(final Object argument) { - if (!(argument instanceof GetBucketRequest)) { - return false; - } - final GetBucketRequest getBucketRequest = ((GetBucketRequest)argument); - return - getBucketRequest.getBucket().equals(MYBUCKET) - && (marker == null - ? null == getBucketRequest.getNextMarker() - : marker.equals(getBucketRequest.getNextMarker())); - - } - }); - } - private static final class GetObjectAnswer implements Answer { - private final String key; - - private GetObjectAnswer(final String key) { - this.key = key; - } - - @Override - public GetObjectResponse answer(final InvocationOnMock invocation) throws Throwable { - final WritableByteChannel channel = ((GetObjectRequest)invocation.getArguments()[0]).getDestinationChannel(); - final Writer writer = Channels.newWriter(channel, "UTF-8"); - writer.write(key); - writer.write(" contents"); - writer.flush(); - return new StubGetObjectResponse(); - } - } - private static final class StubGetBucketResponse extends GetBucketResponse { private final int invocationIndex; @@ -333,82 +316,129 @@ private static Contents buildContents( } } - private static final class StubBulkGetResponse extends BulkGetResponse { - public StubBulkGetResponse() throws IOException { - super(null); - } + private static final UUID jobId = new UUID(0x0123456789abcdefl, 0xfedcba9876543210l); + private static final UUID nodeId = UUID.fromString("29bf5a53-d891-407f-8f3f-749ee7e636f3"); - @Override - protected void processResponse() throws IOException { - } - - @Override - public MasterObjectList getResult() { - return buildMasterObjectList(); - } - } - - private static final class StubBulkPutResponse extends BulkPutResponse { - public StubBulkPutResponse() throws IOException { - super(null); - } + private static Ds3Client buildDs3ClientForBulk() throws IOException, + SignatureException { + final Ds3Client ds3Client = Mockito.mock(Ds3Client.class); - @Override - protected void processResponse() throws IOException { - } - - @Override - public MasterObjectList getResult() { - return buildMasterObjectList(); - } - } + final Ds3ClientFactory ds3ClientFactory = ds3ClientFactory(ds3Client); + Mockito.when(ds3Client.buildFactory(Mockito.>any())).thenReturn(ds3ClientFactory); - private static UUID jobId = new UUID(0x0123456789abcdefl, 0xfedcba9876543210l); + final GetAvailableJobChunksResponse jobChunksResponse1 = buildJobChunksResponse1(); + final GetAvailableJobChunksResponse jobChunksResponse2 = buildJobChunksResponse2(); + final GetAvailableJobChunksResponse jobChunksResponse3 = buildJobChunksResponse3(); + Mockito.when(ds3Client.getAvailableJobChunks(hasJobId(jobId))) + .thenReturn(jobChunksResponse1) + .thenReturn(jobChunksResponse2) + .thenReturn(jobChunksResponse3); - private static MasterObjectList buildMasterObjectList() { - final MasterObjectList masterObjectList = new MasterObjectList(); - masterObjectList.setJobId(jobId); - masterObjectList.setObjects(Arrays.asList( - makeObjects("020c860d-9bb4-4e42-86ff-a1779da0bf16", Arrays.asList(new BulkObject("baz", 12, false, 0))), - makeObjects("47b83776-7cfd-4f2c-b439-c264bba354cc", Arrays.asList(new BulkObject("foo", 12, false, 0))), - makeObjects("66a2414a-6690-410b-86c9-dd79be4d72ae", Arrays.asList(new BulkObject("bar", 12, false, 0))) - )); - return masterObjectList; + return ds3Client; } - private static Objects makeObjects(final String nodeId, final List objectList) { - final Objects objects = new Objects(); - objects.setNodeId(UUID.fromString(nodeId)); - objects.setObjects(objectList); - return objects; + private static BulkGetResponse buildBulkGetResponse() { + return bulkGetResponse(buildJobResponse( + RequestType.GET, + ChunkClientProcessingOrderGuarantee.NONE, + 0L, + 0L, + chunk1(false), + chunk2(false) + )); } - private static final class StubException extends RuntimeException { - private static final long serialVersionUID = 5121719894916333278L; + private static GetAvailableJobChunksResponse buildJobChunksResponse1() { + return retryGetAvailableAfter(1); } - private static final class StubGetObjectResponse extends GetObjectResponse { - public StubGetObjectResponse() throws IOException { - super(null, null, 0); - } + private static GetAvailableJobChunksResponse buildJobChunksResponse2() { + return availableJobChunks(buildJobResponse( + RequestType.GET, + ChunkClientProcessingOrderGuarantee.NONE, + 12L, + 0L, + chunk2(true) + )); + } + + private static GetAvailableJobChunksResponse buildJobChunksResponse3() { + return availableJobChunks(buildJobResponse( + RequestType.GET, + ChunkClientProcessingOrderGuarantee.NONE, + 24L, + 12L, + chunk1(true) + )); + } - @Override - protected void processResponse() throws IOException { - } - - @Override - protected void download(final WritableByteChannel destinationChannel, final int bufferSize) throws IOException { - } + private static BulkPutResponse buildBulkPutResponse() { + return bulkPutResponse(buildJobResponse( + RequestType.PUT, + ChunkClientProcessingOrderGuarantee.IN_ORDER, + 0L, + 0L, + chunk1(false), + chunk2(false) + )); + } + + private static AllocateJobChunkResponse buildAllocateResponse1() { + return retryAllocateLater(1); } - private static final class StubPutObjectResponse extends PutObjectResponse { + private static AllocateJobChunkResponse buildAllocateResponse2() { + return allocated(chunk1(false)); + } + + private static AllocateJobChunkResponse buildAllocateResponse3() { + return allocated(chunk2(false)); + } - public StubPutObjectResponse() throws IOException { - super(null); - } + private static MasterObjectList buildJobResponse( + final RequestType requestType, + final ChunkClientProcessingOrderGuarantee chunkOrdering, + final long cachedSizeInBytes, + final long completedSizeInBytes, + final Objects ... chunks) { + return ResponseBuilders.jobResponse( + jobId, + MYBUCKET, + requestType, + 36L, + cachedSizeInBytes, + completedSizeInBytes, + chunkOrdering, + Priority.VERY_HIGH, + "9/17/2014 1:03:54 PM", + UUID.fromString("57919d2d-448c-4e2a-8886-0413af22243e"), + "spectra", + WriteOptimization.CAPACITY, + Arrays.asList(basicNode(nodeId, "black-pearl")), + Arrays.asList(chunks) + ); + } - @Override - protected void processResponse() throws IOException { - } + private static final UUID CHUNK_ID_1 = UUID.fromString("f44f1aab-f365-4814-883f-037d6afa6bcf"); + private static Objects chunk1(final boolean inCache) { + return chunk( + 1, + CHUNK_ID_1, + nodeId, + object("bar", 0, 12, inCache), + object("baz", 0, 6, inCache), + object("foo", 0, 6, inCache) + ); + } + + private static final UUID CHUNK_ID_2 = UUID.fromString("7cda9f1a-3a7d-44a5-813e-29535228c40c"); + private static Objects chunk2(final boolean inCache) { + return chunk( + 2, + CHUNK_ID_2, + nodeId, + object("foo", 6, 6, inCache), + object("baz", 6, 6, inCache) + ); } } diff --git a/sdk/src/test/java/com/spectralogic/ds3client/helpers/RequestMatchers.java b/sdk/src/test/java/com/spectralogic/ds3client/helpers/RequestMatchers.java new file mode 100644 index 000000000..e594fec41 --- /dev/null +++ b/sdk/src/test/java/com/spectralogic/ds3client/helpers/RequestMatchers.java @@ -0,0 +1,198 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import com.spectralogic.ds3client.commands.*; + +import org.apache.commons.io.IOUtils; +import org.hamcrest.Description; +import org.hamcrest.TypeSafeMatcher; +import org.mockito.ArgumentMatcher; + +import java.io.IOException; +import java.nio.channels.Channels; +import java.nio.channels.SeekableByteChannel; +import java.util.UUID; + +import static org.mockito.Matchers.*; + +public class RequestMatchers { + public static GetAvailableJobChunksRequest hasJobId(final UUID jobId) { + return argThat(new TypeSafeMatcher() { + @Override + public void describeTo(final Description description) { + describeRequest(jobId, description); + } + + @Override + protected boolean matchesSafely(final GetAvailableJobChunksRequest item) { + return jobId.equals(item.getJobId()); + } + + @Override + protected void describeMismatchSafely( + final GetAvailableJobChunksRequest item, + final Description mismatchDescription) { + describeRequest(item.getJobId(), mismatchDescription); + } + + private void describeRequest(final UUID jobIdValue, final Description description) { + description + .appendText("GetAvailableJobChunksRequest with job id: ") + .appendValue(jobIdValue); + } + }); + } + + public static AllocateJobChunkRequest hasChunkId(final UUID chunkId) { + return argThat(new TypeSafeMatcher() { + @Override + public void describeTo(final Description description) { + describeRequest(chunkId, description); + } + + @Override + protected boolean matchesSafely(final AllocateJobChunkRequest item) { + return chunkId.equals(item.getChunkId()); + } + + @Override + protected void describeMismatchSafely( + final AllocateJobChunkRequest item, + final Description mismatchDescription) { + describeRequest(item.getChunkId(), mismatchDescription); + } + + private void describeRequest(final UUID chunkIdValue, final Description description) { + description + .appendText("AllocateJobChunkResponse with chunk id: ") + .appendValue(chunkIdValue); + } + }); + } + + public static GetObjectRequest getRequestHas(final String bucket, final String key, final UUID jobId, final long offset) { + return argThat(new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(final GetObjectRequest item) { + return + item.getBucketName().equals(bucket) + && item.getObjectName().equals(key) + && (item.getJobId() == null ? jobId == null : item.getJobId().equals(jobId)) + && item.getOffset() == offset; + } + + @Override + public void describeTo(final Description description) { + describeTransferRequest(bucket, key, jobId, offset, description); + } + + @Override + protected void describeMismatchSafely(final GetObjectRequest item, final Description mismatchDescription) { + describeTransferRequest( + item.getBucketName(), + item.getObjectName(), + item.getJobId(), + item.getOffset(), + mismatchDescription + ); + } + }); + } + + public static PutObjectRequest putRequestHas( + final String bucket, + final String key, + final UUID jobId, + final long offset, + final String expectedContents) { + return argThat(new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(final PutObjectRequest item) { + return + item.getBucketName().equals(bucket) + && item.getObjectName().equals(key) + && (item.getJobId() == null ? jobId == null : item.getJobId().equals(jobId)) + && item.getOffset() == offset + && channelToString(item.getChannel()).equals(expectedContents); + } + + @Override + public void describeTo(final Description description) { + describeTransferRequest(bucket, key, jobId, offset, description) + .appendText(", contents: ") + .appendValue(expectedContents); + } + + @Override + protected void describeMismatchSafely(final PutObjectRequest item, final Description mismatchDescription) { + describeTransferRequest( + item.getBucketName(), + item.getObjectName(), + item.getJobId(), + item.getOffset(), + mismatchDescription + ) + .appendText(", contents: ") + .appendValue(channelToString(item.getChannel())); + } + }); + } + + private static Description describeTransferRequest( + final String bucket, + final String key, + final UUID jobId, + final long offset, + final Description description) { + return description + .appendText("Get request with bucket: ") + .appendValue(bucket) + .appendText(", object: ") + .appendValue(key) + .appendText(", job id: ") + .appendValue(jobId) + .appendText(", offset: ") + .appendValue(offset); + } + + private static String channelToString(final SeekableByteChannel channel) { + try { + channel.position(0); + return IOUtils.toString(Channels.newReader(channel, "UTF-8")); + } catch (final IOException e) { + throw new RuntimeException(e); + } + } + + public static GetBucketRequest getBucketHas(final String bucket, final String marker) { + return argThat(new ArgumentMatcher() { + @Override + public boolean matches(final Object argument) { + if (!(argument instanceof GetBucketRequest)) { + return false; + } + final GetBucketRequest getBucketRequest = ((GetBucketRequest)argument); + return + getBucketRequest.getBucket().equals(bucket) + && (marker == null + ? null == getBucketRequest.getNextMarker() + : marker.equals(getBucketRequest.getNextMarker())); + + } + }); + } +} diff --git a/sdk/src/test/java/com/spectralogic/ds3client/helpers/ResponseBuilders.java b/sdk/src/test/java/com/spectralogic/ds3client/helpers/ResponseBuilders.java new file mode 100644 index 000000000..ab2b65da5 --- /dev/null +++ b/sdk/src/test/java/com/spectralogic/ds3client/helpers/ResponseBuilders.java @@ -0,0 +1,186 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.helpers; + +import com.spectralogic.ds3client.Ds3Client; +import com.spectralogic.ds3client.Ds3ClientFactory; +import com.spectralogic.ds3client.commands.*; +import com.spectralogic.ds3client.models.bulk.*; +import com.spectralogic.ds3client.utils.ByteArraySeekableByteChannel; + +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import java.io.IOException; +import java.io.Writer; +import java.nio.channels.Channels; +import java.nio.channels.SeekableByteChannel; +import java.nio.channels.WritableByteChannel; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +import static org.mockito.Mockito.*; + +public class ResponseBuilders { + public static BulkGetResponse bulkGetResponse(final MasterObjectList masterObjectList) { + final BulkGetResponse response = mock(BulkGetResponse.class); + when(response.getResult()).thenReturn(masterObjectList); + return response; + } + + public static BulkPutResponse bulkPutResponse(final MasterObjectList masterObjectList) { + final BulkPutResponse response = mock(BulkPutResponse.class); + when(response.getResult()).thenReturn(masterObjectList); + return response; + } + + public static GetAvailableJobChunksResponse availableJobChunks(final MasterObjectList masterObjectList) { + final GetAvailableJobChunksResponse response = mock(GetAvailableJobChunksResponse.class); + when(response.getStatus()).thenReturn(GetAvailableJobChunksResponse.Status.AVAILABLE); + when(response.getMasterObjectList()).thenReturn(masterObjectList); + return response; + } + + public static GetAvailableJobChunksResponse retryGetAvailableAfter(final int retryAfterInSeconds) { + final GetAvailableJobChunksResponse response = mock(GetAvailableJobChunksResponse.class); + when(response.getStatus()).thenReturn(GetAvailableJobChunksResponse.Status.RETRYLATER); + when(response.getRetryAfterSeconds()).thenReturn(retryAfterInSeconds); + return response; + } + + public static AllocateJobChunkResponse allocated(final Objects chunk) { + final AllocateJobChunkResponse response = mock(AllocateJobChunkResponse.class); + when(response.getStatus()).thenReturn(AllocateJobChunkResponse.Status.ALLOCATED); + when(response.getObjects()).thenReturn(chunk); + return response; + } + + public static AllocateJobChunkResponse retryAllocateLater(final int retryAfterInSeconds) { + final AllocateJobChunkResponse response = mock(AllocateJobChunkResponse.class); + when(response.getStatus()).thenReturn(AllocateJobChunkResponse.Status.RETRYLATER); + when(response.getRetryAfterSeconds()).thenReturn(retryAfterInSeconds); + return response; + } + + public static MasterObjectList jobResponse( + final UUID jobId, + final String bucketName, + final RequestType requestType, + final long originalSizeInBytes, + final long cachedSizeInBytes, + final long completedSizeInBytes, + final ChunkClientProcessingOrderGuarantee chunkClientProcessingOrderGuarantee, + final Priority priority, + final String startDate, + final UUID userId, + final String userName, + final WriteOptimization writeOptimization, + final List nodes, + final List objects) { + final MasterObjectList masterObjectList = new MasterObjectList(); + masterObjectList.setJobId(jobId); + masterObjectList.setBucketName(bucketName); + masterObjectList.setRequestType(requestType); + masterObjectList.setOriginalSizeInBytes(originalSizeInBytes); + masterObjectList.setCachedSizeInBytes(cachedSizeInBytes); + masterObjectList.setCompletedSizeInBytes(completedSizeInBytes); + masterObjectList.setChunkClientProcessingOrderGuarantee(chunkClientProcessingOrderGuarantee); + masterObjectList.setPriority(priority); + masterObjectList.setStartDate(startDate); + masterObjectList.setUserId(userId); + masterObjectList.setUserName(userName); + masterObjectList.setWriteOptimization(writeOptimization); + masterObjectList.setNodes(nodes); + masterObjectList.setObjects(objects); + return masterObjectList; + } + + public static Node basicNode(final UUID nodeId, final String endpoint) { + return node(nodeId, endpoint, 80, 443); + } + + public static Node node( + final UUID nodeId, + final String endpoint, + final int httpPort, + final int httpsPort) { + final Node node = new Node(); + node.setId(nodeId); + node.setEndpoint(endpoint); + node.setHttpPort(httpPort); + node.setHttpsPort(httpsPort); + return node; + } + + public static Objects chunk( + final long chunkNumber, + final UUID chunkId, + final UUID nodeId, + final BulkObject ... chunkList) { + final Objects objects = new Objects(); + objects.setChunkNumber(chunkNumber); + objects.setChunkId(chunkId); + objects.setNodeId(nodeId); + objects.setObjects(Arrays.asList(chunkList)); + return objects; + } + + public static BulkObject object( + final String name, + final long offset, + final long length, + final boolean inCache) { + final BulkObject bulkObject = new BulkObject(); + bulkObject.setName(name); + bulkObject.setOffset(offset); + bulkObject.setLength(length); + bulkObject.setInCache(inCache); + return bulkObject; + } + + public static Answer getObjectAnswer(final String result) { + return new Answer() { + @Override + public GetObjectResponse answer(final InvocationOnMock invocation) throws Throwable { + writeToChannel(result, ((GetObjectRequest)invocation.getArguments()[0]).getDestinationChannel()); + return mock(GetObjectResponse.class); + } + }; + } + + public static Ds3ClientFactory ds3ClientFactory(final Ds3Client client) { + final Ds3ClientFactory factory = mock(Ds3ClientFactory.class); + when(factory.getClientForNodeId(Mockito.any())).thenReturn(client); + return factory; + } + + public static SeekableByteChannel channelWithContents(final String contents) { + return writeToChannel(contents, new ByteArraySeekableByteChannel()); + } + + private static T writeToChannel(final String contents, final T channel) { + final Writer writer = Channels.newWriter(channel, "UTF-8"); + try { + writer.write(contents); + writer.flush(); + } catch (final IOException e) { + throw new RuntimeException(); + } + return channel; + } +} From 5e9c280edb796aada4bdb590170274f7b00a332a Mon Sep 17 00:00:00 2001 From: Hans-Peter Klett Date: Thu, 18 Sep 2014 14:09:04 -0600 Subject: [PATCH 14/16] Added thread safety to the objects remaining. --- .../com/spectralogic/ds3client/helpers/JobState.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java index f0500cd6b..55fff6e16 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java @@ -25,15 +25,15 @@ import java.nio.channels.SeekableByteChannel; import java.util.Collection; import java.util.HashSet; +import java.util.concurrent.atomic.AtomicInteger; -//TODO: concurrency class JobState implements AutoCloseable { - private int objectsRemaining; + private final AtomicInteger objectsRemaining; private final AutoCloseableCache channelCache; private final JobPartTracker partTracker; public JobState(final ObjectChannelBuilder channelBuilder, final Collection filteredChunks) { - this.objectsRemaining = getObjectCount(filteredChunks); + this.objectsRemaining = new AtomicInteger(getObjectCount(filteredChunks)); this.channelCache = buildCache(channelBuilder); this.partTracker = JobPartTrackerFactory .buildPartTracker(Iterables.concat(filteredChunks)) @@ -41,7 +41,7 @@ public JobState(final ObjectChannelBuilder channelBuilder, final Collection 0; + return this.objectsRemaining.get() > 0; } private static int getObjectCount(final Collection chunks) { @@ -82,7 +82,7 @@ public JobPartTracker getPartTracker() { private final class ObjectCompletedListenerImplementation implements ObjectCompletedListener { @Override public void objectCompleted(final String name) { - JobState.this.objectsRemaining--; + JobState.this.objectsRemaining.decrementAndGet(); try { JobState.this.channelCache.close(name); } catch (final Exception e) { From 29c2269f24376798aa737f97588a907bbe26cd3e Mon Sep 17 00:00:00 2001 From: Hans-Peter Klett Date: Thu, 18 Sep 2014 17:03:14 -0600 Subject: [PATCH 15/16] Used ChunkClientProcessingOrderGuarantee * Updated the BulkGetRequest to include ChunkClientProcessingOrderGuarantee. * Updated the read helpers to set ChunkClientProcessingOrderGuarantee to NONE. --- .../commands/AllocateJobChunkResponse.java | 8 +++-- .../ds3client/commands/BulkGetRequest.java | 10 +++++++ .../ds3client/commands/BulkRequest.java | 7 ++--- .../GetAvailableJobChunksResponse.java | 6 +++- .../commands/RetryAfterExpectedException.java | 24 +++++++++++++++ .../helpers/Ds3ClientHelpersImpl.java | 2 ++ .../ds3client/models/bulk/Ds3ObjectList.java | 13 +++++++- .../helpers/Ds3ClientHelpers_Test.java | 4 +-- .../ds3client/helpers/RequestMatchers.java | 30 +++++++++++++++++++ 9 files changed, 94 insertions(+), 10 deletions(-) create mode 100644 sdk/src/main/java/com/spectralogic/ds3client/commands/RetryAfterExpectedException.java diff --git a/sdk/src/main/java/com/spectralogic/ds3client/commands/AllocateJobChunkResponse.java b/sdk/src/main/java/com/spectralogic/ds3client/commands/AllocateJobChunkResponse.java index a553e0b2a..ffa9f5c62 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/commands/AllocateJobChunkResponse.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/commands/AllocateJobChunkResponse.java @@ -77,7 +77,11 @@ private static Objects parseChunk(final WebResponse webResponse) throws IOExcept } } - private static int parseRetryAfter(final WebResponse response) { - return Integer.parseInt(response.getHeaders().get("Retry-After")); + private static int parseRetryAfter(final WebResponse webResponse) { + final String retryAfter = webResponse.getHeaders().get("Retry-After"); + if (retryAfter == null) { + throw new RetryAfterExpectedException(); + } + return Integer.parseInt(retryAfter); } } diff --git a/sdk/src/main/java/com/spectralogic/ds3client/commands/BulkGetRequest.java b/sdk/src/main/java/com/spectralogic/ds3client/commands/BulkGetRequest.java index 482dfd674..a848ac1ce 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/commands/BulkGetRequest.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/commands/BulkGetRequest.java @@ -16,6 +16,7 @@ package com.spectralogic.ds3client.commands; import com.spectralogic.ds3client.BulkCommand; +import com.spectralogic.ds3client.models.bulk.ChunkClientProcessingOrderGuarantee; import com.spectralogic.ds3client.models.bulk.Ds3Object; import com.spectralogic.ds3client.models.bulk.Priority; import com.spectralogic.ds3client.models.bulk.WriteOptimization; @@ -40,6 +41,15 @@ public BulkGetRequest withWriteOptimization(final WriteOptimization writeOptimiz super.withWriteOptimization(writeOptimization); return this; } + + public BulkGetRequest withChunkOrdering(final ChunkClientProcessingOrderGuarantee chunkOrdering) { + this.chunkOrdering = chunkOrdering; + return this; + } + + public ChunkClientProcessingOrderGuarantee getChunkOrdering() { + return this.chunkOrdering; + } @Override public BulkCommand getCommand() { diff --git a/sdk/src/main/java/com/spectralogic/ds3client/commands/BulkRequest.java b/sdk/src/main/java/com/spectralogic/ds3client/commands/BulkRequest.java index 6661c2720..fb4e41d9b 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/commands/BulkRequest.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/commands/BulkRequest.java @@ -17,10 +17,7 @@ import com.spectralogic.ds3client.BulkCommand; import com.spectralogic.ds3client.HttpVerb; -import com.spectralogic.ds3client.models.bulk.Ds3Object; -import com.spectralogic.ds3client.models.bulk.Ds3ObjectList; -import com.spectralogic.ds3client.models.bulk.Priority; -import com.spectralogic.ds3client.models.bulk.WriteOptimization; +import com.spectralogic.ds3client.models.bulk.*; import com.spectralogic.ds3client.serializer.XmlOutput; import java.io.ByteArrayInputStream; @@ -35,6 +32,7 @@ abstract class BulkRequest extends AbstractRequest { private long size; private Priority priority; private WriteOptimization writeOptimization; + protected ChunkClientProcessingOrderGuarantee chunkOrdering; public BulkRequest(final String bucket, final List objects) { this.bucket = bucket; @@ -57,6 +55,7 @@ private InputStream generateStream() { objects.setObjects(this.ds3Objects); objects.setPriority(this.priority); objects.setWriteOptimization(this.writeOptimization); + objects.setChunkClientProcessingOrderGuarantee(this.chunkOrdering); final String xmlOutput = XmlOutput.toXml(objects, this.getCommand()); final byte[] stringBytes = xmlOutput.getBytes(); diff --git a/sdk/src/main/java/com/spectralogic/ds3client/commands/GetAvailableJobChunksResponse.java b/sdk/src/main/java/com/spectralogic/ds3client/commands/GetAvailableJobChunksResponse.java index d7790ac88..491a6a864 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/commands/GetAvailableJobChunksResponse.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/commands/GetAvailableJobChunksResponse.java @@ -79,6 +79,10 @@ private static MasterObjectList parseMasterObjectList(final WebResponse webRespo } private static int parseRetryAfter(final WebResponse webResponse) { - return Integer.parseInt(webResponse.getHeaders().get("Retry-After")); + final String retryAfter = webResponse.getHeaders().get("Retry-After"); + if (retryAfter == null) { + throw new RetryAfterExpectedException(); + } + return Integer.parseInt(retryAfter); } } diff --git a/sdk/src/main/java/com/spectralogic/ds3client/commands/RetryAfterExpectedException.java b/sdk/src/main/java/com/spectralogic/ds3client/commands/RetryAfterExpectedException.java new file mode 100644 index 000000000..89cdd164f --- /dev/null +++ b/sdk/src/main/java/com/spectralogic/ds3client/commands/RetryAfterExpectedException.java @@ -0,0 +1,24 @@ +/* + * ****************************************************************************** + * Copyright 2014 Spectra Logic Corporation. All Rights Reserved. + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use + * this file except in compliance with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. + * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR + * CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * **************************************************************************** + */ + +package com.spectralogic.ds3client.commands; + +public class RetryAfterExpectedException extends RuntimeException { + private static final long serialVersionUID = 6193215224073981762L; + + public RetryAfterExpectedException() { + super("Based on the response the server should have returned a Retry-After HTTP header."); + } +} diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java index 75a58cee6..5c90f8ba0 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpersImpl.java @@ -22,6 +22,7 @@ import com.spectralogic.ds3client.helpers.options.WriteJobOptions; import com.spectralogic.ds3client.models.Contents; import com.spectralogic.ds3client.models.ListBucketResult; +import com.spectralogic.ds3client.models.bulk.ChunkClientProcessingOrderGuarantee; import com.spectralogic.ds3client.models.bulk.Ds3Object; import com.spectralogic.ds3client.serializer.XmlProcessingException; @@ -90,6 +91,7 @@ public Ds3ClientHelpers.Job startReadJob(final String bucket, final Iterable objectsToRead, final ReadJobOptions options) throws SignatureException, IOException, XmlProcessingException { final BulkGetResponse prime = this.client.bulkGet(new BulkGetRequest(bucket, Lists.newArrayList(objectsToRead)) + .withChunkOrdering(ChunkClientProcessingOrderGuarantee.NONE) .withPriority(options.getPriority())); return new ReadJobImpl(this.client, prime.getResult()); } diff --git a/sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3ObjectList.java b/sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3ObjectList.java index 62ce21f65..170936e8d 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3ObjectList.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/models/bulk/Ds3ObjectList.java @@ -17,6 +17,9 @@ public class Ds3ObjectList { @JacksonXmlProperty(isAttribute = true, namespace = "", localName = "WriteOptimization") private WriteOptimization writeOptimization; + @JacksonXmlProperty(isAttribute = true, namespace = "", localName = "ChunkClientProcessingOrderGuarantee") + private ChunkClientProcessingOrderGuarantee chunkClientProcessingOrderGuarantee; + public Ds3ObjectList() { } @@ -41,10 +44,18 @@ public void setPriority(final Priority priority) { } public WriteOptimization getWriteOptimization() { - return writeOptimization; + return this.writeOptimization; } public void setWriteOptimization(final WriteOptimization writeOptimization) { this.writeOptimization = writeOptimization; } + + public ChunkClientProcessingOrderGuarantee getChunkClientProcessingOrderGuarantee() { + return this.chunkClientProcessingOrderGuarantee; + } + + public void setChunkClientProcessingOrderGuarantee(final ChunkClientProcessingOrderGuarantee chunkClientProcessingOrderGuarantee) { + this.chunkClientProcessingOrderGuarantee = chunkClientProcessingOrderGuarantee; + } } diff --git a/sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java b/sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java index 6c2ee4e1e..831dee6ac 100644 --- a/sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java +++ b/sdk/src/test/java/com/spectralogic/ds3client/helpers/Ds3ClientHelpers_Test.java @@ -53,7 +53,7 @@ public void testReadObjects() throws SignatureException, IOException, XmlProcess final Ds3Client ds3Client = buildDs3ClientForBulk(); final BulkGetResponse buildBulkGetResponse = buildBulkGetResponse(); - Mockito.when(ds3Client.bulkGet(Mockito.any(BulkGetRequest.class))).thenReturn(buildBulkGetResponse); + Mockito.when(ds3Client.bulkGet(hasChunkOrdering(ChunkClientProcessingOrderGuarantee.NONE))).thenReturn(buildBulkGetResponse); Mockito.when(ds3Client.getObject(getRequestHas(MYBUCKET, "foo", jobId, 0))).then(getObjectAnswer("foo co")); Mockito.when(ds3Client.getObject(getRequestHas(MYBUCKET, "bar", jobId, 0))).then(getObjectAnswer("bar contents")); @@ -97,7 +97,7 @@ public void testReadObjectsWithFailedGet() throws SignatureException, IOExceptio Mockito.when(ds3Client.buildFactory(Mockito.>any())).thenReturn(ds3ClientFactory); final BulkGetResponse buildBulkGetResponse = buildBulkGetResponse(); - Mockito.when(ds3Client.bulkGet(Mockito.any(BulkGetRequest.class))).thenReturn(buildBulkGetResponse); + Mockito.when(ds3Client.bulkGet(hasChunkOrdering(ChunkClientProcessingOrderGuarantee.NONE))).thenReturn(buildBulkGetResponse); final GetAvailableJobChunksResponse jobChunksResponse = buildJobChunksResponse2(); Mockito.when(ds3Client.getAvailableJobChunks(hasJobId(jobId))).thenReturn(jobChunksResponse); diff --git a/sdk/src/test/java/com/spectralogic/ds3client/helpers/RequestMatchers.java b/sdk/src/test/java/com/spectralogic/ds3client/helpers/RequestMatchers.java index e594fec41..8945bac48 100644 --- a/sdk/src/test/java/com/spectralogic/ds3client/helpers/RequestMatchers.java +++ b/sdk/src/test/java/com/spectralogic/ds3client/helpers/RequestMatchers.java @@ -16,6 +16,7 @@ package com.spectralogic.ds3client.helpers; import com.spectralogic.ds3client.commands.*; +import com.spectralogic.ds3client.models.bulk.ChunkClientProcessingOrderGuarantee; import org.apache.commons.io.IOUtils; import org.hamcrest.Description; @@ -30,6 +31,35 @@ import static org.mockito.Matchers.*; public class RequestMatchers { + public static BulkGetRequest hasChunkOrdering(final ChunkClientProcessingOrderGuarantee chunkOrdering) { + return argThat(new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(final BulkGetRequest item) { + return item.getChunkOrdering() == null + ? chunkOrdering == null + : item.getChunkOrdering().equals(chunkOrdering); + } + + @Override + public void describeTo(final Description description) { + describe(chunkOrdering, description); + } + + @Override + protected void describeMismatchSafely(final BulkGetRequest item, final Description mismatchDescription) { + describe(item.getChunkOrdering(), mismatchDescription); + } + + private void describe( + final ChunkClientProcessingOrderGuarantee chunkOrdering, + final Description description) { + description + .appendText("BulkGetRequest with Chunk Ordering: ") + .appendValue(chunkOrdering); + } + }); + } + public static GetAvailableJobChunksRequest hasJobId(final UUID jobId) { return argThat(new TypeSafeMatcher() { @Override From 3eebfb9a5135c23b2175cdaf855ab02a4de3cd28 Mon Sep 17 00:00:00 2001 From: Hans-Peter Klett Date: Mon, 22 Sep 2014 14:18:02 -0600 Subject: [PATCH 16/16] Updated some naming and comments per review. --- ...ansferExecutor.java => ChunkTransferrer.java} | 16 ++++++++-------- .../ds3client/helpers/JobPartTracker.java | 5 +++++ .../spectralogic/ds3client/helpers/JobState.java | 4 ++-- .../ds3client/helpers/ReadJobImpl.java | 12 ++++++------ .../ds3client/helpers/WriteJobImpl.java | 8 ++++---- 5 files changed, 25 insertions(+), 20 deletions(-) rename sdk/src/main/java/com/spectralogic/ds3client/helpers/{ChunkTransferExecutor.java => ChunkTransferrer.java} (91%) diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/ChunkTransferExecutor.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ChunkTransferrer.java similarity index 91% rename from sdk/src/main/java/com/spectralogic/ds3client/helpers/ChunkTransferExecutor.java rename to sdk/src/main/java/com/spectralogic/ds3client/helpers/ChunkTransferrer.java index 3ccfc8378..a77139733 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/ChunkTransferExecutor.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/ChunkTransferrer.java @@ -34,22 +34,22 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; -class ChunkTransferExecutor { - private final Transferrer transferrer; +class ChunkTransferrer { + private final ItemTransferrer itemTransferrer; private final Ds3Client mainClient; private final JobPartTracker partTracker; private final int maxParallelRequests; - public static interface Transferrer { + public static interface ItemTransferrer { void transferItem(Ds3Client client, BulkObject ds3Object) throws SignatureException, IOException; } - public ChunkTransferExecutor( - final Transferrer transferrer, + public ChunkTransferrer( + final ItemTransferrer transferrer, final Ds3Client mainClient, final JobPartTracker partTracker, final int maxParallelRequests) { - this.transferrer = transferrer; + this.itemTransferrer = transferrer; this.mainClient = mainClient; this.partTracker = partTracker; this.maxParallelRequests = maxParallelRequests; @@ -71,8 +71,8 @@ public void transferChunks( tasks.add(executor.submit(new Callable() { @Override public Object call() throws Exception { - ChunkTransferExecutor.this.transferrer.transferItem(client, ds3Object); - ChunkTransferExecutor.this.partTracker.completePart(ds3Object.getName(), part); + ChunkTransferrer.this.itemTransferrer.transferItem(client, ds3Object); + ChunkTransferrer.this.partTracker.completePart(ds3Object.getName(), part); return null; } })); diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTracker.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTracker.java index d5b5a136c..2c3477f3a 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTracker.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobPartTracker.java @@ -17,6 +17,11 @@ import java.util.Map; +/** + * This class manages parts for all of the objects in the job. It aggregates + * ObjectPartTracker implementations, which manage the parts for a single + * object. + */ public class JobPartTracker { private final Map trackers; diff --git a/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java index 55fff6e16..5179e6bf1 100644 --- a/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java +++ b/sdk/src/main/java/com/spectralogic/ds3client/helpers/JobState.java @@ -37,7 +37,7 @@ public JobState(final ObjectChannelBuilder channelBuilder, final Collection filteredChunks = filterChunks(this.masterObjectList.getObjects()); try (final JobState jobState = new JobState(channelBuilder, filteredChunks)) { - final ChunkTransferExecutor executor = new ChunkTransferExecutor( + final ChunkTransferrer chunkTransferrer = new ChunkTransferrer( new PutObjectTransferrer(jobState), this.client, jobState.getPartTracker(), this.maxParallelRequests ); for (final Objects chunk : filteredChunks) { - executor.transferChunks(this.masterObjectList.getNodes(), Arrays.asList(filterChunk(allocateChunk(chunk)))); + chunkTransferrer.transferChunks(this.masterObjectList.getNodes(), Arrays.asList(filterChunk(allocateChunk(chunk)))); } } catch (final SignatureException | IOException | XmlProcessingException e) { throw e; @@ -118,7 +118,7 @@ private static List filterObjects(final List list) { return filtered; } - private final class PutObjectTransferrer implements Transferrer { + private final class PutObjectTransferrer implements ItemTransferrer { private final JobState jobState; private PutObjectTransferrer(final JobState jobState) {