Skip to content

Commit

Permalink
Checking a few experimental changes into the branch:
Browse files Browse the repository at this point in the history 
Jim's soft commit fixes from 10547;
A quick experiment, replacing join on public objects with a boolean publicObject_b:true
for logged-in users as well (with a join added for just for their own personal documents;
groups are ignored for now). #10554
  • Loading branch information
@landreev
landreev committed May 21, 2024
1 parent 070b3c0 commit bb33111
Show file tree
Hide file tree
Showing 3 changed files with 41 additions and 20 deletions.
29 changes: 16 additions & 13 deletions src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ public Future<String> indexDataverse(Dataverse dataverse, boolean processPaths)
String status;
try {
if (dataverse.getId() != null) {
solrClientService.getSolrClient().add(docs);
solrClientService.getSolrClient().add(docs, COMMIT_WITHIN);
} else {
logger.info("WARNING: indexing of a dataverse with no id attempted");
}
Expand All @@ -321,13 +321,13 @@ public Future<String> indexDataverse(Dataverse dataverse, boolean processPaths)
logger.info(status);
return new AsyncResult<>(status);
}
try {
/*try {
solrClientService.getSolrClient().commit();
} catch (SolrServerException | IOException ex) {
status = ex.toString();
logger.info(status);
return new AsyncResult<>(status);
}
}*/

dvObjectService.updateContentIndexTime(dataverse);
IndexResponse indexResponse = solrIndexService.indexPermissionsForOneDvObject(dataverse);
Expand All @@ -353,6 +353,7 @@ public void indexDatasetInNewTransaction(Long datasetId) { //Dataset dataset) {
private static final Map<Long, Boolean> INDEXING_NOW = new ConcurrentHashMap<>();
// semaphore for async indexing
private static final Semaphore ASYNC_INDEX_SEMAPHORE = new Semaphore(JvmSettings.MAX_ASYNC_INDEXES.lookupOptional(Integer.class).orElse(4), true);
static final int COMMIT_WITHIN = 5000;

@Inject
@Metric(name = "index_permit_wait_time", absolute = true, unit = MetricUnits.NANOSECONDS,
Expand Down Expand Up @@ -1535,8 +1536,8 @@ private String addOrUpdateDataset(IndexableDataset indexableDataset, Set<Long> d
final SolrInputDocuments docs = toSolrDocs(indexableDataset, datafilesInDraftVersion);

try {
solrClientService.getSolrClient().add(docs.getDocuments());
solrClientService.getSolrClient().commit();
solrClientService.getSolrClient().add(docs.getDocuments(), COMMIT_WITHIN);
//solrClientService.getSolrClient().commit();
} catch (SolrServerException | IOException ex) {
if (ex.getCause() instanceof SolrServerException) {
throw new SolrServerException(ex);
Expand Down Expand Up @@ -1788,8 +1789,8 @@ private void updatePathForExistingSolrDocs(DvObject object) throws SolrServerExc

sid.removeField(SearchFields.SUBTREE);
sid.addField(SearchFields.SUBTREE, paths);
UpdateResponse addResponse = solrClientService.getSolrClient().add(sid);
UpdateResponse commitResponse = solrClientService.getSolrClient().commit();
UpdateResponse addResponse = solrClientService.getSolrClient().add(sid, COMMIT_WITHIN);
//UpdateResponse commitResponse = solrClientService.getSolrClient().commit();
if (object.isInstanceofDataset()) {
for (DataFile df : dataset.getFiles()) {
solrQuery.setQuery(SearchUtil.constructQuery(SearchFields.ENTITY_ID, df.getId().toString()));
Expand All @@ -1802,8 +1803,8 @@ private void updatePathForExistingSolrDocs(DvObject object) throws SolrServerExc
}
sid.removeField(SearchFields.SUBTREE);
sid.addField(SearchFields.SUBTREE, paths);
addResponse = solrClientService.getSolrClient().add(sid);
commitResponse = solrClientService.getSolrClient().commit();
addResponse = solrClientService.getSolrClient().add(sid, COMMIT_WITHIN);
//commitResponse = solrClientService.getSolrClient().commit();
}
}
}
Expand Down Expand Up @@ -1845,15 +1846,16 @@ public String delete(Dataverse doomed) {
logger.fine("deleting Solr document for dataverse " + doomed.getId());
UpdateResponse updateResponse;
try {
updateResponse = solrClientService.getSolrClient().deleteById(solrDocIdentifierDataverse + doomed.getId());
updateResponse = solrClientService.getSolrClient().deleteById(solrDocIdentifierDataverse + doomed.getId(), COMMIT_WITHIN);
} catch (SolrServerException | IOException ex) {
return ex.toString();
}
try {
/*try {
solrClientService.getSolrClient().commit();
} catch (SolrServerException | IOException ex) {
return ex.toString();
}
*/
String response = "Successfully deleted dataverse " + doomed.getId() + " from Solr index. updateReponse was: " + updateResponse.toString();
logger.fine(response);
return response;
Expand All @@ -1870,15 +1872,16 @@ public String removeSolrDocFromIndex(String doomed) {
logger.fine("deleting Solr document: " + doomed);
UpdateResponse updateResponse;
try {
updateResponse = solrClientService.getSolrClient().deleteById(doomed);
updateResponse = solrClientService.getSolrClient().deleteById(doomed, COMMIT_WITHIN);
} catch (SolrServerException | IOException ex) {
return ex.toString();
}
try {
/*try {
solrClientService.getSolrClient().commit();
} catch (SolrServerException | IOException ex) {
return ex.toString();
}
*/
String response = "Attempted to delete " + doomed + " from Solr index. updateReponse was: " + updateResponse.toString();
logger.fine(response);
return response;
Expand Down
18 changes: 18 additions & 0 deletions src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1079,6 +1079,24 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ
// ----------------------------------------------------
// (5) Work with Authenticated User who is not a Superuser
// ----------------------------------------------------

// A quick speedup experiment (L.A.)
// - This ignores the group membership completely;
// so, not a production-ready fix, just a proof of concept,
// an attempt to replace an uber-expensive join on ALL the public
// documents with the "publicObject:true" flag, similarly to what we
// are doing for guest users, above, and only using
// the join to explicitly look up the (few) documents the user is
// directly authorized to see (once again, groups are ignored for now)
if (FeatureFlags.AVOID_EXPENSIVE_SOLR_JOIN.enabled()) {
StringBuilder sb = new StringBuilder();

sb.append(SearchFields.PUBLIC_OBJECT + ":" + true + " OR ");
sb.append("{!join from=" + SearchFields.DEFINITION_POINT + " to=id}" + SearchFields.DISCOVERABLE_BY + ":" + IndexServiceBean.getGroupPerUserPrefix() + au.getId());
String ret = sb.toString();
logger.info("Returning experimental query: "+ret);
return ret;
}
/**
* @todo all this code needs cleanup and clarification.
*/
Expand Down
14 changes: 7 additions & 7 deletions src/main/java/edu/harvard/iq/dataverse/search/SolrIndexServiceBean.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -356,8 +356,8 @@ private void persistToSolr(Collection<SolrInputDocument> docs) throws SolrServer
/**
* @todo Do something with these responses from Solr.
*/
UpdateResponse addResponse = solrClientService.getSolrClient().add(docs);
UpdateResponse commitResponse = solrClientService.getSolrClient().commit();
UpdateResponse addResponse = solrClientService.getSolrClient().add(docs, IndexServiceBean.COMMIT_WITHIN);
//UpdateResponse commitResponse = solrClientService.getSolrClient().commit();
}

public IndexResponse indexPermissionsOnSelfAndChildren(long definitionPointId) {
Expand Down Expand Up @@ -497,26 +497,26 @@ public IndexResponse deleteMultipleSolrIds(List<String> solrIdsToDelete) {
return new IndexResponse("nothing to delete");
}
try {
solrClientService.getSolrClient().deleteById(solrIdsToDelete);
solrClientService.getSolrClient().deleteById(solrIdsToDelete, IndexServiceBean.COMMIT_WITHIN);
} catch (SolrServerException | IOException ex) {
/**
* @todo mark these for re-deletion
*/
return new IndexResponse("problem deleting the following documents from Solr: " + solrIdsToDelete);
}
try {
/*try {
solrClientService.getSolrClient().commit();
} catch (SolrServerException | IOException ex) {
return new IndexResponse("problem committing deletion of the following documents from Solr: " + solrIdsToDelete);
}
}*/
return new IndexResponse("no known problem deleting the following documents from Solr:" + solrIdsToDelete);
}

public JsonObjectBuilder deleteAllFromSolrAndResetIndexTimes() throws SolrServerException, IOException {
JsonObjectBuilder response = Json.createObjectBuilder();
logger.info("attempting to delete all Solr documents before a complete re-index");
solrClientService.getSolrClient().deleteByQuery("*:*");
solrClientService.getSolrClient().commit();
solrClientService.getSolrClient().deleteByQuery("*:*", IndexServiceBean.COMMIT_WITHIN);
//solrClientService.getSolrClient().commit();
int numRowsAffected = dvObjectService.clearAllIndexTimes();
response.add(numRowsClearedByClearAllIndexTimes, numRowsAffected);
response.add(messageString, "Solr index and database index timestamps cleared.");
Expand Down

0 comments on commit bb33111

Please sign in to comment.