Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DURACLOUD-1313: Stops synctool retrying hundreds of times #154

Merged
merged 5 commits into from
Nov 11, 2021
Merged

DURACLOUD-1313: Stops synctool retrying hundreds of times #154

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4555e4a
Bump hibernate version
Jul 23, 2021
985caa8
Stop syncclient from retrying hundreds of times
Nov 10, 2021
f4df17d
Revert ContentIterator and change spaceExists method
Nov 11, 2021
90e5da4
Don't capture acls in a variable
Nov 11, 2021
2ff0213
Do not create space if it does not exist
Nov 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@
<org.springframework.version>4.2.5.RELEASE</org.springframework.version>
<org.springframework.security.version>4.0.4.RELEASE</org.springframework.security.version>
<org.springframework.webflow.version>2.4.2.RELEASE</org.springframework.webflow.version>
<hibernate.version>5.1.0.Final</hibernate.version>
<hibernate.version>5.4.3.Final</hibernate.version>
<aws.sdk.version>1.12.24</aws.sdk.version>
<jersey.version>2.5.1</jersey.version>
<jackson.version>2.12.3</jackson.version>
Expand All @@ -252,7 +252,7 @@
<slf4j.version>1.7.6</slf4j.version>
<jetty.version>9.4.31.v20200723</jetty.version>
<jaxb.api.version>2.3.1</jaxb.api.version>
<jaxb.runtime.version>2.3.3</jaxb.runtime.version>
<jaxb.runtime.version>2.3.1</jaxb.runtime.version>
</properties>

<distributionManagement>
Expand Down
44 changes: 3 additions & 41 deletions synctool/src/main/java/org/duracloud/sync/endpoint/DuraStoreSyncEndpoint.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import org.duracloud.common.util.ContentIdUtil;
import org.duracloud.common.util.DateUtil;
import org.duracloud.error.ContentStoreException;
import org.duracloud.error.NotFoundException;
import org.duracloud.storage.util.StorageProviderUtil;
import org.duracloud.sync.config.SyncToolConfig;
import org.slf4j.Logger;
Expand Down Expand Up @@ -101,48 +100,11 @@ protected String getUsername() {
}

private void ensureSpaceExists() {
boolean spaceExists = false;
for (int i = 0; i < 10; i++) {
if (spaceExists()) {
spaceExists = true;
break;
}
sleep(300);
}
if (!spaceExists) {
throw new RuntimeException("Could not connect to space with ID '" +
spaceId + "'.");
}
}

private void sleep(long millis) {
try {
Thread.sleep(millis);
} catch (InterruptedException e) {
// Exit sleep on interruption
}
}

private boolean spaceExists() {
try {
try {
Iterator<String> contents =
contentStore.getSpaceContents(spaceId);
if (contents.hasNext()) {
logger.warn("The specified space '" + spaceId +
"' is not empty. If this space is being used for an " +
"activity other than sync there is the possibility " +
"of data loss.");
}
return true;
} catch (NotFoundException e) {
contentStore.createSpace(spaceId);
return false;
}
contentStore.getSpaceACLs(spaceId);
} catch (ContentStoreException e) {
logger.warn("Could not connect to space with ID '" + spaceId +
"' due to error: " + e.getMessage(), e);
return false;
throw new RuntimeException("Could not connect to space with ID '" +
spaceId + "'.");
}
}

Expand Down
12 changes: 4 additions & 8 deletions synctool/src/test/java/org/duracloud/sync/endpoint/DuraStoreChunkSyncEndpointTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,9 +104,11 @@ public void testGetFilesList() throws Exception {
private void createGetFilesListMocks(List<String> contents)
throws ContentStoreException {

EasyMock.expect(contentStore.getSpaceACLs(spaceId))
.andReturn(new HashMap<String, AclType>());

EasyMock.expect(contentStore.getSpaceContents(spaceId))
.andReturn(contents.iterator())
.times(2);
.andReturn(contents.iterator());

}

Expand Down Expand Up @@ -139,8 +141,6 @@ public void testAddNewContent() throws Exception {
new ChecksumUtil(ChecksumUtil.Algorithm.MD5);
String checksum = checksumUtil.generateChecksum(contentFile);

EasyMock.expect(contentStore.getSpaceContents(spaceId))
.andReturn(new ArrayList<String>().iterator());
EasyMock.expect(contentStore.getSpaceACLs(spaceId))
.andReturn(new HashMap<String, AclType>())
.anyTimes();
Expand Down Expand Up @@ -182,8 +182,6 @@ public void testUpdateChunkedContentWithUnchunked() throws Exception {
new ChecksumUtil(ChecksumUtil.Algorithm.MD5);
String checksum = checksumUtil.generateChecksum(contentFile);

EasyMock.expect(contentStore.getSpaceContents(spaceId))
.andReturn(new ArrayList<String>().iterator());
EasyMock.expect(contentStore.getSpaceACLs(spaceId))
.andReturn(new HashMap<String, AclType>())
.anyTimes();
Expand Down Expand Up @@ -256,8 +254,6 @@ protected void testAddChunkedFile(int chunkCount, int chunkSize, int threadCount
File tmpFile = File.createTempFile("test", "txt");
tmpFile.deleteOnExit();

EasyMock.expect(contentStore.getSpaceContents(spaceId))
.andReturn(new ArrayList<String>().iterator()).times(1);
EasyMock.expect(contentStore.getSpaceACLs(spaceId))
.andReturn(new HashMap<String, AclType>())
.anyTimes();
Expand Down
7 changes: 4 additions & 3 deletions synctool/src/test/java/org/duracloud/sync/endpoint/DuraStoreSyncEndpointTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,12 @@

import java.io.File;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;

import org.apache.commons.io.FileUtils;
import org.duracloud.client.ContentStore;
import org.duracloud.common.model.AclType;
import org.duracloud.common.util.ChecksumUtil;
import org.easymock.Capture;
import org.easymock.CaptureType;
Expand All @@ -42,8 +43,8 @@ public void setUp() throws Exception {
spaceId = "spaceId";
contentStore = EasyMock.createMock(ContentStore.class);

EasyMock.expect(contentStore.getSpaceContents(EasyMock.isA(String.class)))
.andReturn(new ArrayList<String>().iterator())
EasyMock.expect(contentStore.getSpaceACLs(EasyMock.isA(String.class)))
.andReturn(new HashMap<String, AclType>())
.anyTimes();

EasyMock.expect(contentStore.getStoreId())
Expand Down
6 changes: 6 additions & 0 deletions synctoolui/src/test/java/org/duracloud/syncui/service/SyncProcessManagerImplTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.CountDownLatch;
Expand All @@ -25,6 +26,7 @@
import org.apache.commons.io.FileUtils;
import org.duracloud.client.ContentStore;
import org.duracloud.client.ContentStoreManager;
import org.duracloud.common.model.AclType;
import org.duracloud.common.model.Credential;
import org.duracloud.error.ContentStoreException;
import org.duracloud.sync.endpoint.MonitoredFile;
Expand Down Expand Up @@ -136,6 +138,10 @@ protected void setupStart(int times) throws ContentStoreException {
.andReturn(contentStore).times(times);
expect(this.contentStore.getStoreId()).andReturn("0").anyTimes();

expect(contentStore.getSpaceACLs(isA(String.class)))
.andReturn(new HashMap<String, AclType>())
.anyTimes();

expect(this.contentStore.getSpaceContents(isA(String.class)))
.andAnswer(new IAnswer<Iterator<String>>() {
@Override
Expand Down