Skip to content

Commit

Permalink
Merge pull request DSpace#9269 from atmire/w2p-90830-104654_issue-812…
Browse files Browse the repository at this point in the history 
…5_pr-8267_Issue-stale-record-cleanup-7.x

7.x - Issue predb status cleanup
  • Loading branch information
@tdonohue
tdonohue committed Apr 10, 2024
2 parents 2afc9b1 + 4fc19da commit 68bb3c5
Show file tree
Hide file tree
Showing 2 changed files with 111 additions and 28 deletions.
70 changes: 42 additions & 28 deletions dspace-api/src/main/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncCli.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,42 +105,56 @@ private void performStatusUpdate(Context context) throws SearchServiceException,
solrQuery.addFilterQuery(dateRangeFilter);
solrQuery.addField(SearchUtils.RESOURCE_ID_FIELD);
solrQuery.addField(SearchUtils.RESOURCE_UNIQUE_ID);
solrQuery.setRows(0);
QueryResponse response = solrSearchCore.getSolr().query(solrQuery, solrSearchCore.REQUEST_METHOD);

if (response != null) {
logInfoAndOut(response.getResults().size() + " items found to process");

for (SolrDocument doc : response.getResults()) {
String uuid = (String) doc.getFirstValue(SearchUtils.RESOURCE_ID_FIELD);
String uniqueId = (String) doc.getFirstValue(SearchUtils.RESOURCE_UNIQUE_ID);
logDebugAndOut("Processing item with UUID: " + uuid);

Optional<IndexableObject> indexableObject = Optional.empty();
try {
indexableObject = indexObjectServiceFactory
.getIndexableObjectFactory(uniqueId).findIndexableObject(context, uuid);
} catch (SQLException e) {
log.warn("An exception occurred when attempting to retrieve item with UUID \"" + uuid +
"\" from the database, removing related solr document", e);
}

try {
if (indexableObject.isPresent()) {
logDebugAndOut("Item exists in DB, updating solr document");
updateItem(context, indexableObject.get());
} else {
logDebugAndOut("Item doesn't exist in DB, removing solr document");
removeItem(context, uniqueId);
}
} catch (SQLException | IOException e) {
log.error(e.getMessage(), e);
if (response != null && response.getResults() != null) {
long nrOfPreDBResults = response.getResults().getNumFound();
if (nrOfPreDBResults > 0) {
logInfoAndOut(nrOfPreDBResults + " items found to process");
int batchSize = configurationService.getIntProperty("script.solr-database-resync.batch-size", 100);
for (int start = 0; start < nrOfPreDBResults; start += batchSize) {
solrQuery.setStart(start);
solrQuery.setRows(batchSize);
performStatusUpdateOnNextBatch(context, solrQuery);
}
}
}

indexingService.commit();
}

private void performStatusUpdateOnNextBatch(Context context, SolrQuery solrQuery)
throws SolrServerException, IOException {
QueryResponse response = solrSearchCore.getSolr().query(solrQuery, solrSearchCore.REQUEST_METHOD);

for (SolrDocument doc : response.getResults()) {
String uuid = (String) doc.getFirstValue(SearchUtils.RESOURCE_ID_FIELD);
String uniqueId = (String) doc.getFirstValue(SearchUtils.RESOURCE_UNIQUE_ID);
logDebugAndOut("Processing item with UUID: " + uuid);

Optional<IndexableObject> indexableObject = Optional.empty();
try {
indexableObject = indexObjectServiceFactory
.getIndexableObjectFactory(uniqueId).findIndexableObject(context, uuid);
} catch (SQLException e) {
log.warn("An exception occurred when attempting to retrieve item with UUID \"" + uuid +
"\" from the database, removing related solr document", e);
}

try {
if (indexableObject.isPresent()) {
logDebugAndOut("Item exists in DB, updating solr document");
updateItem(context, indexableObject.get());
} else {
logDebugAndOut("Item doesn't exist in DB, removing solr document");
removeItem(context, uniqueId);
}
} catch (SQLException | IOException e) {
log.error(e.getMessage(), e);
}
}
}

private void updateItem(Context context, IndexableObject indexableObject) throws SolrServerException, IOException {
Map<String,Object> fieldModifier = new HashMap<>(1);
fieldModifier.put("remove", STATUS_FIELD_PREDB);
Expand Down
69 changes: 69 additions & 0 deletions dspace-api/src/test/java/org/dspace/app/solrdatabaseresync/SolrDatabaseResyncIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@
import org.junit.Before;
import org.junit.Test;

/**
* IT for {@link org.dspace.app.solrdatabaseresync.SolrDatabaseResyncIT}
*/
public class SolrDatabaseResyncIT extends AbstractIntegrationTestWithDatabase {

private final ConfigurationService configurationService =
Expand All @@ -48,11 +51,21 @@ public class SolrDatabaseResyncIT extends AbstractIntegrationTestWithDatabase {
private Collection col;
private Item item1;
private Item item2;
private Item item3;
private Item item4;
private Item item5;
private Item item6;
private Item item7;
private Item item8;
private Item item9;
private Item item10;
private Item item11;

@Before
public void setUp() throws Exception {
super.setUp();
configurationService.setProperty("solr-database-resync.time-until-reindex", 1);
configurationService.setProperty("script.solr-database-resync.batch-size", 5);

ServiceManager serviceManager = DSpaceServicesFactory.getInstance().getServiceManager();
searchService = serviceManager.getServiceByName(null, MockSolrSearchCore.class);
Expand All @@ -75,6 +88,16 @@ public void setUp() throws Exception {
.withSubject("TestingForMore")
.build();

item3 = ItemBuilder.createItem(context, col).withTitle("Public item 3").build();
item4 = ItemBuilder.createItem(context, col).withTitle("Public item 4").build();
item5 = ItemBuilder.createItem(context, col).withTitle("Public item 5").build();
item6 = ItemBuilder.createItem(context, col).withTitle("Public item 6").build();
item7 = ItemBuilder.createItem(context, col).withTitle("Public item 7").build();
item8 = ItemBuilder.createItem(context, col).withTitle("Public item 8").build();
item9 = ItemBuilder.createItem(context, col).withTitle("Public item 9").build();
item10 = ItemBuilder.createItem(context, col).withTitle("Public item 10").build();
item11 = ItemBuilder.createItem(context, col).withTitle("Public item 11").build();

context.setDispatcher("noindex");
}

Expand All @@ -83,12 +106,30 @@ public void solrPreDBStatusExistingItemTest() throws Exception {
// Items were created, they should contain a predb status in solr
assertHasPreDBStatus(item1);
assertHasPreDBStatus(item2);
assertHasPreDBStatus(item3);
assertHasPreDBStatus(item4);
assertHasPreDBStatus(item5);
assertHasPreDBStatus(item6);
assertHasPreDBStatus(item7);
assertHasPreDBStatus(item8);
assertHasPreDBStatus(item9);
assertHasPreDBStatus(item10);
assertHasPreDBStatus(item11);

performSolrDatabaseResyncScript();

// Database status script was performed, their predb status should be removed
assertHasNoPreDBStatus(item1);
assertHasNoPreDBStatus(item2);
assertHasNoPreDBStatus(item3);
assertHasNoPreDBStatus(item4);
assertHasNoPreDBStatus(item5);
assertHasNoPreDBStatus(item6);
assertHasNoPreDBStatus(item7);
assertHasNoPreDBStatus(item8);
assertHasNoPreDBStatus(item9);
assertHasNoPreDBStatus(item10);
assertHasNoPreDBStatus(item11);

context.restoreAuthSystemState();
}
Expand All @@ -98,22 +139,50 @@ public void solrPreDBStatusRemovedItemTest() throws Exception {
// Items were created, they should contain a predb status in solr
assertHasPreDBStatus(item1);
assertHasPreDBStatus(item2);
assertHasPreDBStatus(item3);
assertHasPreDBStatus(item4);
assertHasPreDBStatus(item5);
assertHasPreDBStatus(item6);
assertHasPreDBStatus(item7);
assertHasPreDBStatus(item8);
assertHasPreDBStatus(item9);
assertHasPreDBStatus(item10);
assertHasPreDBStatus(item11);

collectionService.delete(context, col);

// Items were deleted, they should still contain a predb status in solr for now
assertHasPreDBStatus(item1);
assertHasPreDBStatus(item2);
assertHasPreDBStatus(item3);
assertHasPreDBStatus(item4);
assertHasPreDBStatus(item5);
assertHasPreDBStatus(item6);
assertHasPreDBStatus(item7);
assertHasPreDBStatus(item8);
assertHasPreDBStatus(item9);
assertHasPreDBStatus(item10);
assertHasPreDBStatus(item11);

performSolrDatabaseResyncScript();

// Database status script was performed, their solr document should have been removed
assertNoSolrDocument(item1);
assertNoSolrDocument(item2);
assertNoSolrDocument(item3);
assertNoSolrDocument(item4);
assertNoSolrDocument(item5);
assertNoSolrDocument(item6);
assertNoSolrDocument(item7);
assertNoSolrDocument(item8);
assertNoSolrDocument(item9);
assertNoSolrDocument(item10);
assertNoSolrDocument(item11);

context.restoreAuthSystemState();
}


public void assertHasNoPreDBStatus(Item item) throws Exception {
assertNotEquals(STATUS_FIELD_PREDB, getStatus(item));
}
Expand Down

0 comments on commit 68bb3c5

Please sign in to comment.