Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

TRUNK-5557: Improve performance of ConceptService.getConceptsByMapping #2961

Merged
merged 3 commits into from Dec 10, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
22 changes: 22 additions & 0 deletions api/src/main/java/org/openmrs/api/ConceptService.java
Expand Up @@ -1025,6 +1025,28 @@ public interface ConceptService extends OpenmrsService {
*/
@Authorized(PrivilegeConstants.GET_CONCEPTS)
public List<Concept> getConceptsByMapping(String code, String sourceName, boolean includeRetired) throws APIException;

/**
* Looks up concepts via {@link ConceptMap} This will return the list of ids for all
* {@link Concept}s which contain a {@link ConceptMap} entry whose <code>sourceCode</code> is
* equal to the passed <code>conceptCode</code> and whose {@link ConceptSource} has either a
* <code>name</code> or <code>hl7Code</code> that is equal to the passed
* <code>mappingCode</code>
*
* @param code the code associated with a concept within a given {@link ConceptSource}
* @param sourceName the name or hl7Code of the {@link ConceptSource} to check
* @param includeRetired whether or not to include retired concepts
* @return the list ids for all non-voided {@link Concept}s that have the given mapping, or an empty List if none found
* @throws APIException if the specified source+code maps to more than one concept
* @should get concepts with given code and and source hl7 code
* @should get concepts with given code and source name
* @should return empty list if source code does not exist
* @should return empty list if mapping does not exist
* @should include retired concepts
* @since 2.3
*/
@Authorized(PrivilegeConstants.GET_CONCEPTS)
public List<Integer> getConceptIdsByMapping(String code, String sourceName, boolean includeRetired) throws APIException;

/**
* Get all the concept name tags defined in the database, included voided ones
Expand Down
8 changes: 8 additions & 0 deletions api/src/main/java/org/openmrs/api/db/ConceptDAO.java
Expand Up @@ -322,8 +322,16 @@ public Integer getCountOfConcepts(String phrase, List<Locale> locales, boolean i

/**
* @see org.openmrs.api.ConceptService#getConceptsByMapping(java.lang.String, java.lang.String)
*
* @deprecated As of 2.5.0, this method has been deprecated in favor of {@link #getConceptIdsByMapping(String, String, boolean)}
*/
@Deprecated
public List<Concept> getConceptsByMapping(String code, String sourceName, boolean includeRetired);

/**
* @see org.openmrs.api.ConceptService#getConceptIdsByMapping(String, String, boolean)
*/
public List<Integer> getConceptIdsByMapping(String code, String sourceName, boolean includeRetired);

/**
* @param uuid
Expand Down
Expand Up @@ -471,7 +471,8 @@ public List<ConceptDatatype> getAllConceptDatatypes(boolean includeRetired) thro
}

/**
* @see org.openmrs.api.db.ConceptDAO#getConceptDatatypes(java.lang.String)
* @param name the name of the ConceptDatatype
* @return a List of ConceptDatatype whose names start with the passed name
*/
@SuppressWarnings("unchecked")
public List<ConceptDatatype> getConceptDatatypes(String name) throws DAOException {
Expand Down Expand Up @@ -817,7 +818,7 @@ public List<ConceptSet> getSetsContainingConcept(Concept concept) {
/**
* returns a list of n-generations of parents of a concept in a concept set
*
* @param Concept current
* @param current
* @return List&lt;Concept&gt;
* @throws DAOException
*/
Expand Down Expand Up @@ -1016,54 +1017,30 @@ public void remove() {
}

}

/**
* @see org.openmrs.api.db.ConceptDAO#getConceptsByMapping(String, String, boolean)
*/
@Override
@SuppressWarnings("unchecked")
@Deprecated
public List<Concept> getConceptsByMapping(String code, String sourceName, boolean includeRetired) {
Criteria criteria = sessionFactory.getCurrentSession().createCriteria(ConceptMap.class);

// make this criteria return a list of concepts
Criteria criteria = createSearchConceptMapCriteria(code, sourceName, includeRetired);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd make use of cache here as well with:

List<Integer> ids = Context.getConceptService().getConceptIdsByMapping(code, sourceName, includeRetired); 
session.createCriteria(Concept.class).add(Restrictions.in("id",ids)).list(); 
if (includeRetired) {
			// sort retired concepts to the end of the list
			criteria.addOrder(Order.asc("retired"));
}

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried this out, and it was a lot slower (2x) than the way the service method is implemented. I also am not a fan of services calling DAOs which then call different services. I actually think we should leave this dao method implementation alone but Deprecate the method, as it is no longer used by the Service layer at all.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Interesting. Fair point to not call services in DAOs.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume this is due to the fact that by iterating over the concept ids and calling getConcept(id), we also leverage hibernate's use of ehcache in a more efficient manner than by running an HQL query with an "in" clause on ids.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Am in favour of leaving this dao method implementation alone.

criteria.setProjection(Projections.property("concept"));

//join to the conceptReferenceTerm table
criteria.createAlias("conceptReferenceTerm", "term");

// match the source code to the passed code
if (Context.getAdministrationService().isDatabaseStringComparisonCaseSensitive()) {
criteria.add(Restrictions.eq("term.code", code).ignoreCase());
} else {
criteria.add(Restrictions.eq("term.code", code));
}

// join to concept reference source and match to the h17Code or source name
criteria.createAlias("term.conceptSource", "source");
if (Context.getAdministrationService().isDatabaseStringComparisonCaseSensitive()) {
criteria.add(Restrictions.or(Restrictions.eq("source.name", sourceName).ignoreCase(), Restrictions.eq(
"source.hl7Code", sourceName).ignoreCase()));
} else {
criteria.add(Restrictions.or(Restrictions.eq("source.name", sourceName), Restrictions.eq("source.hl7Code",
sourceName)));
}

criteria.createAlias("concept", "concept");

if (!includeRetired) {
// ignore retired concepts
criteria.add(Restrictions.eq("concept.retired", false));
} else {
// sort retired concepts to the end of the list
criteria.addOrder(Order.asc("concept.retired"));
}

// we only want distinct concepts
criteria.setResultTransformer(DistinctRootEntityResultTransformer.INSTANCE);

return (List<Concept>) criteria.list();
}


/**
* @see org.openmrs.api.db.ConceptDAO#getConceptIdsByMapping(String, String, boolean)
*/
@Override
@SuppressWarnings("unchecked")
public List<Integer> getConceptIdsByMapping(String code, String sourceName, boolean includeRetired) {
Criteria criteria = createSearchConceptMapCriteria(code, sourceName, includeRetired);
criteria.setProjection(Projections.property("concept.conceptId"));
return (List<Integer>) criteria.list();
}

/**
* @see org.openmrs.api.db.ConceptDAO#getConceptByUuid(java.lang.String)
*/
Expand Down Expand Up @@ -2074,4 +2051,43 @@ private Criteria createSearchDrugByMappingCriteria(String code, ConceptSource co
}
return searchCriteria;
}

private Criteria createSearchConceptMapCriteria(String code, String sourceName, boolean includeRetired) {
Criteria criteria = sessionFactory.getCurrentSession().createCriteria(ConceptMap.class);

//join to the conceptReferenceTerm table
criteria.createAlias("conceptReferenceTerm", "term");

// match the source code to the passed code
if (Context.getAdministrationService().isDatabaseStringComparisonCaseSensitive()) {
criteria.add(Restrictions.eq("term.code", code).ignoreCase());
} else {
criteria.add(Restrictions.eq("term.code", code));
}

// join to concept reference source and match to the h17Code or source name
criteria.createAlias("term.conceptSource", "source");
if (Context.getAdministrationService().isDatabaseStringComparisonCaseSensitive()) {
criteria.add(Restrictions.or(Restrictions.eq("source.name", sourceName).ignoreCase(), Restrictions.eq(
"source.hl7Code", sourceName).ignoreCase()));
} else {
criteria.add(Restrictions.or(Restrictions.eq("source.name", sourceName), Restrictions.eq("source.hl7Code",
sourceName)));
}

criteria.createAlias("concept", "concept");

if (!includeRetired) {
// ignore retired concepts
criteria.add(Restrictions.eq("concept.retired", false));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note that cache should be evicted also when concept.retired changes.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just pushed a follow-up commit that:

  • Deprecates the getConceptsByMapping DAO method in favor of future use of getConceptIdsByMapping
  • Evicts from cache more liberally on saveConcept, purge/save/retire ConceptSource, updateConceptIndexes
    Thanks for the support and feedback @rkorytkowski - much appreciated.

} else {
// sort retired concepts to the end of the list
criteria.addOrder(Order.asc("concept.retired"));
}

// we only want distinct concepts
criteria.setResultTransformer(DistinctRootEntityResultTransformer.INSTANCE);

return criteria;
}
}
36 changes: 29 additions & 7 deletions api/src/main/java/org/openmrs/api/impl/ConceptServiceImpl.java
Expand Up @@ -67,6 +67,8 @@
import org.openmrs.validator.ValidateUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.StringUtils;

Expand All @@ -89,7 +91,9 @@ public class ConceptServiceImpl extends BaseOpenmrsService implements ConceptSer
private static Concept unknownConcept;

private static final String ERROR_MESSAGE = "Error generated";


private static final String CONCEPT_IDS_BY_MAPPING_CACHE_NAME = "conceptIdsByMapping";

/**
* @see org.openmrs.api.ConceptService#setConceptDAO(org.openmrs.api.db.ConceptDAO)
*/
Expand All @@ -108,6 +112,7 @@ public void setConceptDAO(ConceptDAO dao) {
* <strong>Should</strong> force set flag if set members exist
*/
@Override
@CacheEvict(value = CONCEPT_IDS_BY_MAPPING_CACHE_NAME, allEntries = true)
public Concept saveConcept(Concept concept) throws APIException {
ensureConceptMapTypeIsSet(concept);

Expand Down Expand Up @@ -291,8 +296,7 @@ public Concept retireConcept(Concept concept, String reason) throws APIException

concept.setRetired(true);
concept.setRetireReason(reason);
return dao.saveConcept(concept);

return Context.getConceptService().saveConcept(concept);
}

return concept;
Expand Down Expand Up @@ -949,8 +953,8 @@ public List<ConceptSource> getAllConceptSources(boolean includeRetired) {
* @see org.openmrs.api.ConceptService#purgeConceptSource(org.openmrs.ConceptSource)
*/
@Override
@CacheEvict(value = CONCEPT_IDS_BY_MAPPING_CACHE_NAME, allEntries = true)
public ConceptSource purgeConceptSource(ConceptSource cs) throws APIException {

return dao.deleteConceptSource(cs);
}

Expand All @@ -960,13 +964,14 @@ public ConceptSource purgeConceptSource(ConceptSource cs) throws APIException {
@Override
public ConceptSource retireConceptSource(ConceptSource cs, String reason) throws APIException {
// retireReason is automatically set in BaseRetireHandler
return dao.saveConceptSource(cs);
return Context.getConceptService().saveConceptSource(cs);
}

/**
* @see org.openmrs.api.ConceptService#saveConceptSource(org.openmrs.ConceptSource)
*/
@Override
@CacheEvict(value = CONCEPT_IDS_BY_MAPPING_CACHE_NAME, allEntries = true)
public ConceptSource saveConceptSource(ConceptSource conceptSource) throws APIException {
return dao.saveConceptSource(conceptSource);
}
Expand Down Expand Up @@ -1159,7 +1164,21 @@ public List<Concept> getConceptsByMapping(String code, String sourceName) throws
@Override
@Transactional(readOnly = true)
public List<Concept> getConceptsByMapping(String code, String sourceName, boolean includeRetired) throws APIException {
return dao.getConceptsByMapping(code, sourceName, includeRetired);
List<Concept> concepts = new ArrayList<>();
for (Integer conceptId : Context.getConceptService().getConceptIdsByMapping(code, sourceName, includeRetired)) {
concepts.add(getConcept(conceptId));
}
return concepts;
}

/**
* @see org.openmrs.api.ConceptService#getConceptIdsByMapping(java.lang.String, java.lang.String, boolean)
*/
@Override
@Transactional(readOnly = true)
@Cacheable(value = CONCEPT_IDS_BY_MAPPING_CACHE_NAME)
public List<Integer> getConceptIdsByMapping(String code, String sourceName, boolean includeRetired) throws APIException {
return dao.getConceptIdsByMapping(code, sourceName, includeRetired);
}

/**
Expand Down Expand Up @@ -1484,6 +1503,7 @@ public void updateConceptIndex(Concept concept) throws APIException {
* @see ConceptService#updateConceptIndexes()
*/
@Override
@CacheEvict(value = CONCEPT_IDS_BY_MAPPING_CACHE_NAME, allEntries = true)
public void updateConceptIndexes() throws APIException {
Context.updateSearchIndexForType(ConceptName.class);
}
Expand Down Expand Up @@ -1717,6 +1737,7 @@ public ConceptReferenceTerm getConceptReferenceTermByCode(String code, ConceptSo
* @see org.openmrs.api.ConceptService#saveConceptReferenceTerm(org.openmrs.ConceptReferenceTerm)
*/
@Override
@CacheEvict(value = CONCEPT_IDS_BY_MAPPING_CACHE_NAME, allEntries = true)
public ConceptReferenceTerm saveConceptReferenceTerm(ConceptReferenceTerm conceptReferenceTerm) throws APIException {
return dao.saveConceptReferenceTerm(conceptReferenceTerm);
}
Expand All @@ -1732,7 +1753,7 @@ public ConceptReferenceTerm retireConceptReferenceTerm(ConceptReferenceTerm conc
tmpRetireReason = Context.getMessageSourceService().getMessage("general.default.retireReason");
}
conceptReferenceTerm.setRetireReason(tmpRetireReason);
return dao.saveConceptReferenceTerm(conceptReferenceTerm);
return Context.getConceptService().saveConceptReferenceTerm(conceptReferenceTerm);
}

/**
Expand All @@ -1747,6 +1768,7 @@ public ConceptReferenceTerm unretireConceptReferenceTerm(ConceptReferenceTerm co
* @see org.openmrs.api.ConceptService#purgeConceptReferenceTerm(org.openmrs.ConceptReferenceTerm)
*/
@Override
@CacheEvict(value = CONCEPT_IDS_BY_MAPPING_CACHE_NAME, allEntries = true)
public void purgeConceptReferenceTerm(ConceptReferenceTerm conceptReferenceTerm) throws APIException {
if (dao.isConceptReferenceTermInUse(conceptReferenceTerm)) {
throw new APIException("ConceptRefereceTerm.inUse", (Object[]) null);
Expand Down
9 changes: 8 additions & 1 deletion api/src/main/resources/ehcache-api.xml
Expand Up @@ -33,4 +33,11 @@
<persistence strategy="none"/>
</cache>

</ehcache>
<cache name="conceptIdsByMapping"
maxElementsInMemory="10000"
eternal="true"
memoryStoreEvictionPolicy="LRU">
<persistence strategy="localTempSwap"/>
</cache>

</ehcache>