Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ACS-7587 Implement bulk cancellations #2683

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
a22e7d2
ACS-7557 Add bulk API design
damianujma Apr 25, 2024
502c996
ACS-7557 Fix [ags]
damianujma Apr 29, 2024
8986d03
ACS-7557 Add permissions checks [ags]
damianujma May 6, 2024
34925b4
ACS-7557 Add IT tests
damianujma May 9, 2024
b5ce847
ACS-7557 Add comments + logging
damianujma May 9, 2024
0735233
ACS-7557 Refactor
damianujma May 9, 2024
d7f9ed1
ACS-7557 Reimplement task container
damianujma May 10, 2024
07dcba9
ACS-7557 Refactor code
damianujma May 15, 2024
82e9f04
Merge branch 'feature/ACS-7557_design_bulk_api' into feature/ACS-7587…
damianujma May 15, 2024
c31ae1f
ACS-7587 Remove merge leftovers
damianujma Apr 25, 2024
c8c1102
ACS-7587 Refactor [ags]
damianujma May 15, 2024
73724a8
ACS-7587 Tests [ags][tas]
damianujma May 15, 2024
592dc35
ACS-7587 Change DefaultHoldBulkMonitor [ags][tas]
damianujma May 15, 2024
0de1aca
ACS-7587 Reimplement BulkStatusUpdater [ags][tas]
damianujma May 16, 2024
7b516f2
ACS-7587 Fix PMD issues
damianujma May 17, 2024
66aef18
ACS-7587 Fix PMD isues
damianujma May 17, 2024
785bdb7
ACS-7587 Refactor
damianujma May 20, 2024
8009a6d
ACS-7587 Add test files alternately
damianujma May 20, 2024
ed0bbb6
ACS-7587 Refactor code
damianujma May 24, 2024
fa6e0de
ACS-7587 Improve search query
damianujma May 24, 2024
a69be86
ACS-7587 Fix PMD issues
damianujma May 27, 2024
72e8507
ACS-7587 Fix PMD issue
damianujma May 27, 2024
818bcb0
ACS-7587 Fix intermittent failure
damianujma May 27, 2024
33976e8
ACS-7587 Implement bulk cancellation
damianujma May 27, 2024
098dea5
ACS-7587 Refactor HoldsAPI
damianujma May 28, 2024
47bf098
ACS-7587 Fix cache
damianujma May 29, 2024
c0dd88a
ACS-7587 Implement IT test for cancellation
damianujma Jun 3, 2024
3c79caf
ACS-7587 Add cancellation + extend bulk status
damianujma Jun 4, 2024
bec806f
Merge branch 'feature/ACS-7556_bulk_update_in_legal_holds' into featu…
damianujma Jun 4, 2024
787b0b1
ACS-7587 Add bulk cancellation [ags][tas]
damianujma Jun 6, 2024
4e0d5b7
ACS-7587 Add private constructor [tas][ags]
damianujma Jun 6, 2024
4292ec4
ACS-7587 Increase delay [tas][ags]
damianujma Jun 6, 2024
56ff090
ACS-7587 Wait to cancel [tas][ags]
damianujma Jun 6, 2024
8aba752
ACS-7587 Add newFixedThreadPool [tas][ags]
damianujma Jun 6, 2024
f53f1a4
ACS-7587 Refactor code [tas][ags]
damianujma Jun 6, 2024
c6ab5ea
ACS-7587 Fix PMD issues [tas][ags]
damianujma Jun 7, 2024
5e1f3d5
ACS-7587 Remove moved HoldBulkStatus
damianujma Jun 7, 2024
92eab9c
ACS-7587 Set limit
damianujma Jun 10, 2024
dd76283
ACS-7587 Add equals and hashcode to RestRequestQueryModel
damianujma Jun 11, 2024
23409ba
ACS-7587 Fix PMD issue
damianujma Jun 11, 2024
d6c76a4
ACS-7587 Set timeToLiveSeconds + reimplement caches
damianujma Jun 12, 2024
24e3379
ACS-7587 Reimplement cancellation cache
damianujma Jun 13, 2024
43433d9
ACS-7587 Reformat code
damianujma Jun 13, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*-
* #%L
* Alfresco Records Management Module
* %%
* Copyright (C) 2005 - 2024 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* -
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
* -
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* -
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
* -
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.rest.rm.community.model.hold;

import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;

@Builder
@Data
@NoArgsConstructor
@AllArgsConstructor
public class BulkBodyCancel
{
private String reason;
}
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import org.alfresco.rest.search.RestRequestQueryModel;
import org.alfresco.utility.model.TestModel;
Expand All @@ -41,7 +40,6 @@
*
* @author Damian Ujma
*/
@EqualsAndHashCode(callSuper = true)
@Builder
@Data
@NoArgsConstructor
Expand All @@ -57,4 +55,5 @@ public enum HoldBulkOperationType
private RestRequestQueryModel query;
@JsonProperty(required = true)
private HoldBulkOperationType op;

}
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,11 @@ public class HoldBulkStatus extends TestModel

private String lastError;

private Status status;
private String status;
dsibilio marked this conversation as resolved.
Show resolved Hide resolved

public enum Status
{
PENDING,
IN_PROGRESS,
DONE
}
private boolean isCancelled;

private String cancellationReason;

private HoldBulkOperation holdBulkOperation;
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import static org.springframework.http.HttpMethod.PUT;

import org.alfresco.rest.core.RMRestWrapper;
import org.alfresco.rest.rm.community.model.hold.BulkBodyCancel;
import org.alfresco.rest.rm.community.model.hold.Hold;
import org.alfresco.rest.rm.community.model.hold.HoldBulkOperation;
import org.alfresco.rest.rm.community.model.hold.HoldBulkOperationEntry;
Expand Down Expand Up @@ -400,4 +401,46 @@ public HoldBulkStatusCollection getBulkStatuses(String holdId)
return getBulkStatuses(holdId, EMPTY);
}

/**
* Cancels a bulk operation for a hold.
*
* @param holdId The identifier of a hold
* @param bulkStatusId The identifier of a bulk status operation
* @param bulkBodyCancel The bulk body cancel model
* @param parameters The URL parameters to add
* @throws RuntimeException for the following cases:
* <ul>
* <li>{@code holdId}, {@code bulkStatusId} or {@code bulkBodyCancel} is invalid</li>
* <li>authentication fails</li>
* <li>current user does not have permission to cancel the bulk operation for {@code bulkStatusId}</li>
* <li>{@code holdId} or {@code bulkStatusId} does not exist</li>
* </ul>
*/
public void cancelBulkOperation(String holdId, String bulkStatusId, BulkBodyCancel bulkBodyCancel, String parameters)
{
mandatoryString("holdId", holdId);
mandatoryString("bulkStatusId", bulkStatusId);
mandatoryObject("bulkBodyCancel", bulkBodyCancel);

getRmRestWrapper().processEmptyModel(requestWithBody(
POST,
toJson(bulkBodyCancel),
"holds/{holdId}/bulk-statuses/{bulkStatusId}/cancel",
holdId,
bulkStatusId,
parameters
));
}

/**
* See {@link #cancelBulkOperation(String, String, BulkBodyCancel, String)}
*/
public void cancelBulkOperation(String holdId, String bulkStatusId, BulkBodyCancel bulkBodyCancel)
{
mandatoryString("holdId", holdId);
mandatoryString("bulkStatusId", bulkStatusId);
mandatoryObject("bulkBodyCancel", bulkBodyCancel);

cancelBulkOperation(holdId, bulkStatusId, bulkBodyCancel, EMPTY);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -41,22 +41,24 @@
import static org.springframework.http.HttpStatus.BAD_REQUEST;
import static org.springframework.http.HttpStatus.FORBIDDEN;
import static org.springframework.http.HttpStatus.NOT_FOUND;
import static org.springframework.http.HttpStatus.OK;
import static org.springframework.http.HttpStatus.UNAUTHORIZED;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.TimeUnit;

import org.alfresco.dataprep.CMISUtil;
import org.alfresco.dataprep.ContentActions;
import org.alfresco.rest.rm.community.base.BaseRMRestTest;
import org.alfresco.rest.rm.community.model.hold.BulkBodyCancel;
import org.alfresco.rest.rm.community.model.hold.Hold;
import org.alfresco.rest.rm.community.model.hold.HoldBulkOperation;
import org.alfresco.rest.rm.community.model.hold.HoldBulkOperation.HoldBulkOperationType;
import org.alfresco.rest.rm.community.model.hold.HoldBulkOperationEntry;
import org.alfresco.rest.rm.community.model.hold.HoldBulkStatus;
import org.alfresco.rest.rm.community.model.hold.HoldBulkStatus.Status;
import org.alfresco.rest.rm.community.model.hold.HoldBulkStatusCollection;
import org.alfresco.rest.rm.community.model.hold.HoldBulkStatusEntry;
import org.alfresco.rest.rm.community.model.hold.HoldChild;
Expand Down Expand Up @@ -167,12 +169,13 @@ public void addContentFromTestSiteToHoldUsingBulkAPI()
STEP("Check the bulk status.");
HoldBulkStatus holdBulkStatus = getRestAPIFactory().getHoldsAPI(userAddHoldPermission)
.getBulkStatus(hold.getId(), bulkOperationEntry.getBulkStatusId());
assertBulkProcessStatus(holdBulkStatus, NUMBER_OF_FILES, 0, null);
assertBulkProcessStatus(holdBulkStatus, NUMBER_OF_FILES, 0, null, holdBulkOperation);

STEP("Check the bulk statuses.");
HoldBulkStatusCollection holdBulkStatusCollection = getRestAPIFactory().getHoldsAPI(userAddHoldPermission)
.getBulkStatuses(hold.getId());
assertEquals(Arrays.asList(holdBulkStatus), holdBulkStatusCollection.getEntries().stream().map(HoldBulkStatusEntry::getEntry).toList());
assertEquals(Arrays.asList(holdBulkStatus),
holdBulkStatusCollection.getEntries().stream().map(HoldBulkStatusEntry::getEntry).toList());
}

/**
Expand Down Expand Up @@ -217,12 +220,13 @@ public void addContentFromFolderAndAllSubfoldersToHoldUsingBulkAPI()
STEP("Check the bulk status.");
HoldBulkStatus holdBulkStatus = getRestAPIFactory().getHoldsAPI(userAddHoldPermission)
.getBulkStatus(hold3.getId(), bulkOperationEntry.getBulkStatusId());
assertBulkProcessStatus(holdBulkStatus, NUMBER_OF_FILES, 0, null);
assertBulkProcessStatus(holdBulkStatus, NUMBER_OF_FILES, 0, null, bulkOperation);

STEP("Check the bulk statuses.");
HoldBulkStatusCollection holdBulkStatusCollection = getRestAPIFactory().getHoldsAPI(userAddHoldPermission)
.getBulkStatuses(hold3.getId());
assertEquals(Arrays.asList(holdBulkStatus), holdBulkStatusCollection.getEntries().stream().map(HoldBulkStatusEntry::getEntry).toList());
assertEquals(List.of(holdBulkStatus),
holdBulkStatusCollection.getEntries().stream().map(HoldBulkStatusEntry::getEntry).toList());
}

/**
Expand Down Expand Up @@ -300,12 +304,13 @@ public void testBulkProcessWithUserWithoutWritePermissionOnTheContent()
assertStatusCode(ACCEPTED);

await().atMost(20, TimeUnit.SECONDS).until(() ->
getRestAPIFactory().getHoldsAPI(userWithoutPermission)
.getBulkStatus(hold.getId(), bulkOperationEntry.getBulkStatusId()).getStatus() == Status.DONE);
Objects.equals(getRestAPIFactory().getHoldsAPI(userWithoutPermission)
.getBulkStatus(hold.getId(), bulkOperationEntry.getBulkStatusId()).getStatus(), "DONE"));

HoldBulkStatus holdBulkStatus = getRestAPIFactory().getHoldsAPI(userWithoutPermission)
.getBulkStatus(hold.getId(), bulkOperationEntry.getBulkStatusId());
assertBulkProcessStatus(holdBulkStatus, NUMBER_OF_FILES, NUMBER_OF_FILES, ACCESS_DENIED_ERROR_MESSAGE);
assertBulkProcessStatus(holdBulkStatus, NUMBER_OF_FILES, NUMBER_OF_FILES, ACCESS_DENIED_ERROR_MESSAGE,
holdBulkOperation);
}

/**
Expand Down Expand Up @@ -344,7 +349,8 @@ public void testBulkProcessWithUserWithoutWritePermissionOnOneFile()
== NUMBER_OF_FILES - 1);
await().atMost(30, TimeUnit.SECONDS).until(
() -> getRestAPIFactory().getHoldsAPI(userAddHoldPermission)
.getBulkStatus(hold2.getId(), bulkOperationEntry.getBulkStatusId()).getProcessedItems() == NUMBER_OF_FILES);
.getBulkStatus(hold2.getId(), bulkOperationEntry.getBulkStatusId()).getProcessedItems()
== NUMBER_OF_FILES);
List<String> holdChildrenNodeRefs = getRestAPIFactory().getHoldsAPI(userAddHoldPermission)
.getChildren(hold2.getId()).getEntries().stream().map(HoldChildEntry::getEntry).map(
HoldChild::getId).toList();
Expand All @@ -354,12 +360,13 @@ public void testBulkProcessWithUserWithoutWritePermissionOnOneFile()
STEP("Check the bulk status.");
HoldBulkStatus holdBulkStatus = getRestAPIFactory().getHoldsAPI(userAddHoldPermission)
.getBulkStatus(hold2.getId(), bulkOperationEntry.getBulkStatusId());
assertBulkProcessStatus(holdBulkStatus, NUMBER_OF_FILES, 1, ACCESS_DENIED_ERROR_MESSAGE);
assertBulkProcessStatus(holdBulkStatus, NUMBER_OF_FILES, 1, ACCESS_DENIED_ERROR_MESSAGE, holdBulkOperation);

STEP("Check the bulk statuses.");
HoldBulkStatusCollection holdBulkStatusCollection = getRestAPIFactory().getHoldsAPI(userAddHoldPermission)
.getBulkStatuses(hold2.getId());
assertEquals(Arrays.asList(holdBulkStatus), holdBulkStatusCollection.getEntries().stream().map(HoldBulkStatusEntry::getEntry).toList());
assertEquals(List.of(holdBulkStatus),
holdBulkStatusCollection.getEntries().stream().map(HoldBulkStatusEntry::getEntry).toList());

// Revert the permissions
contentActions.setPermissionForUser(getAdminUser().getUsername(), getAdminUser().getPassword(),
Expand Down Expand Up @@ -489,14 +496,89 @@ public void testExceedingBulkOperationLimit()
assertStatusCode(BAD_REQUEST);
}

/**
* Given a user with the add to hold capability and hold filing permission
* When the user adds content from a site to a hold using the bulk API
* And then the user cancels the bulk operation
* Then the user receives OK status code
*/
@Test
public void testBulkProcessCancellationWithAllowedUser()
{
Hold hold4 = getRestAPIFactory().getFilePlansAPI(getAdminUser()).createHold(
Hold.builder().name("HOLD" + generateTestPrefix(AddToHoldsV1Tests.class)).description(HOLD_DESCRIPTION)
.reason(HOLD_REASON).build(), FILE_PLAN_ALIAS);
holds.add(hold4);

UserModel userAddHoldPermission = roleService.createUserWithSiteRoleRMRoleAndPermission(testSite,
UserRole.SiteCollaborator, hold4.getId(), UserRoles.ROLE_RM_MANAGER, PERMISSION_FILING);
users.add(userAddHoldPermission);

STEP("Add content from the site to the hold using the bulk API.");
HoldBulkOperationEntry bulkOperationEntry = getRestAPIFactory().getHoldsAPI(userAddHoldPermission)
.startBulkProcess(holdBulkOperation, hold4.getId());

// Verify the status code
assertStatusCode(ACCEPTED);
assertEquals(NUMBER_OF_FILES, bulkOperationEntry.getTotalItems());

dsibilio marked this conversation as resolved.
Show resolved Hide resolved
STEP("Cancel the bulk operation.");
getRestAPIFactory().getHoldsAPI(userAddHoldPermission)
.cancelBulkOperation(hold4.getId(), bulkOperationEntry.getBulkStatusId(), new BulkBodyCancel());

// Verify the status code
assertStatusCode(OK);
}

/**
* Given a user with the add to hold capability and hold filing permission
* When the user adds content from a site to a hold using the bulk API
* And a 2nd user without the add to hold capability cancels the bulk operation
* Then the 2nd user receives access denied error
*/
@Test
public void testBulkProcessCancellationWithUserWithoutAddToHoldCapability()
{
Hold hold5 = getRestAPIFactory().getFilePlansAPI(getAdminUser()).createHold(
Hold.builder().name("HOLD" + generateTestPrefix(AddToHoldsV1Tests.class)).description(HOLD_DESCRIPTION)
.reason(HOLD_REASON).build(), FILE_PLAN_ALIAS);
holds.add(hold5);

UserModel userAddHoldPermission = roleService.createUserWithSiteRoleRMRoleAndPermission(testSite,
UserRole.SiteCollaborator, hold5.getId(), UserRoles.ROLE_RM_MANAGER, PERMISSION_FILING);
users.add(userAddHoldPermission);

STEP("Add content from the site to the hold using the bulk API.");
HoldBulkOperationEntry bulkOperationEntry = getRestAPIFactory().getHoldsAPI(userAddHoldPermission)
.startBulkProcess(holdBulkOperation, hold5.getId());

// Verify the status code
assertStatusCode(ACCEPTED);
assertEquals(NUMBER_OF_FILES, bulkOperationEntry.getTotalItems());

UserModel userWithoutAddToHoldCapability = roleService.createUserWithSiteRoleRMRoleAndPermission(testSite,
UserRole
.SiteCollaborator,
hold5.getId(), UserRoles.ROLE_RM_POWER_USER, PERMISSION_FILING);
users.add(userWithoutAddToHoldCapability);

STEP("Cancel the bulk operation.");
getRestAPIFactory().getHoldsAPI(userWithoutAddToHoldCapability)
.cancelBulkOperation(hold5.getId(), bulkOperationEntry.getBulkStatusId(), new BulkBodyCancel());

STEP("Verify the response status code and the error message.");
assertStatusCode(FORBIDDEN);
getRestAPIFactory().getRmRestWrapper().assertLastError().containsSummary(ACCESS_DENIED_ERROR_MESSAGE);
}

private void assertBulkProcessStatus(HoldBulkStatus holdBulkStatus, long expectedProcessedItems,
int expectedErrorsCount, String expectedErrorMessage)
int expectedErrorsCount, String expectedErrorMessage, HoldBulkOperation holdBulkOperation)
{
assertEquals(Status.DONE, holdBulkStatus.getStatus());
assertEquals("DONE", holdBulkStatus.getStatus());
assertEquals(expectedProcessedItems, holdBulkStatus.getTotalItems());
assertEquals(expectedProcessedItems, holdBulkStatus.getProcessedItems());
assertEquals(expectedErrorsCount, holdBulkStatus.getErrorsCount());
assertEquals(holdBulkStatus.getHoldBulkOperation(), holdBulkOperation);
assertNotNull(holdBulkStatus.getStartTime());
assertNotNull(holdBulkStatus.getEndTime());

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,9 @@ serverHealth.showTenants=false
# testManagement.username=<username>
# testManagement.apiKey=<api-key>
# testManagement.project=<id-of-your-project
# testManagement.testRun=<test-run-name>
# testManagement.testRun=<test-run-name>
# testManagement.includeOnlyTestCasesExecuted=true #if you want to include in your run ONLY the test cases that you run, then set this value to false
# testManagement.rateLimitInSeconds=1 #is the default rate limit after what minimum time, should we upload the next request. http://docs.gurock.com/testrail-api2/introduction #Rate Limit
# testManagement.rateLimitInSeconds=1 #is the default rate limit after what minimum time, should we upload the next request. http://docs.gurock.com/testrail-api2/introduction #Rate Limit
# testManagement.suiteId=23 (the id of the Master suite)
# ------------------------------------------------------
testManagement.enabled=false
Expand All @@ -72,7 +72,7 @@ reports.path=./target/reports
#
# MySQL:
# db.url = jdbc:mysql://${alfresco.server}:3306/alfresco
#
#
# PostgreSQL:
# db.url = jdbc:postgresql://<your-DB-IP>:3306/alfresco
#
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,12 @@ rm.hold.bulk.batchSize=100
rm.hold.bulk.logging.interval=100
# The number of entries we process at a time in a transaction.
rm.hold.bulk.itemsPerTransaction=1
# The maximum number of bulk requests we can process in parallel.
rm.hold.bulk.maxParallelRequests=10

cache.bulkHoldStatusCache.cluster.type=fully-distributed
cache.bulkHoldStatusCache.timeToLiveSeconds=2592000
cache.bulkHoldRegistryCache.cluster.type=fully-distributed

cache.bulkHoldRegistryCache.timeToLiveSeconds=2592000
cache.bulkCancellationsCache.cluster.type=fully-distributed
cache.bulkCancellationsCache.timeToLiveSeconds=2592000
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,15 @@
<property name="itemsPerTransaction">
<value>${rm.hold.bulk.itemsPerTransaction}</value>
</property>
<property name="maxParallelRequests">
<value>${rm.hold.bulk.maxParallelRequests}</value>
</property>
</bean>

<bean id="holdBulkMonitor" class="org.alfresco.module.org_alfresco_module_rm.bulk.hold.DefaultHoldBulkMonitor">
<property name="holdProgressCache" ref="holdProgressCache" />
<property name="holdProcessRegistry" ref="holdProcessRegistry" />
<property name="bulkCancellationsCache" ref="bulkCancellationsCache" />
</bean>


Expand All @@ -48,4 +52,8 @@
<constructor-arg value="cache.bulkHoldRegistryCache" />
</bean>

<bean name="bulkCancellationsCache" factory-bean="cacheFactory" factory-method="createCache">
<constructor-arg value="cache.bulkCancellationsCache" />
</bean>

</beans>
Loading
Loading