Skip to content

Commit

Permalink
Add disk caching for faster restart. Do not remove cached data when a…
Browse files Browse the repository at this point in the history 
… service is unavailable during a cache refresh.
  • Loading branch information
@adam-collins
adam-collins committed Feb 19, 2017
1 parent bf6eedb commit 8cf5de4
Show file tree
Hide file tree
Showing 13 changed files with 397 additions and 93 deletions.
8 changes: 8 additions & 0 deletions src/main/java/au/org/ala/biocache/config/AppConfig.java
View file
@@ -1,5 +1,6 @@
package au.org.ala.biocache.config;

import au.org.ala.biocache.service.RestartDataService;
import au.org.ala.biocache.service.SpeciesLookupIndexService;
import au.org.ala.biocache.service.SpeciesLookupRestService;
import au.org.ala.biocache.service.SpeciesLookupService;
Expand Down Expand Up @@ -53,6 +54,13 @@ public class AppConfig {
protected Boolean facetDefault;



@Value("${restart.data.dir:/tmp}")
public void setDatabase(String dir) {
RestartDataService.dir = dir;
}


protected SpeciesLookupService getSpeciesLookupRestService() {
logger.info("Initialising rest-based species lookup services.");
SpeciesLookupRestService service = new SpeciesLookupRestService();
Expand Down
49 changes: 33 additions & 16 deletions src/main/java/au/org/ala/biocache/dao/SearchDAOImpl.java
View file
Expand Up @@ -29,6 +29,7 @@
import au.org.ala.biocache.util.thread.EndemicCallable;
import au.org.ala.biocache.vocab.ErrorCode;
import au.org.ala.biocache.writer.*;
import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.googlecode.ehcache.annotations.Cacheable;
import org.apache.commons.io.output.ByteArrayOutputStream;
Expand Down Expand Up @@ -59,6 +60,7 @@
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;

import javax.annotation.PostConstruct;
import javax.inject.Inject;
import javax.servlet.ServletOutputStream;
import java.io.IOException;
Expand All @@ -85,6 +87,7 @@
* @author "Nick dos Remedios <Nick.dosRemedios@csiro.au>"
* @see au.org.ala.biocache.dao.SearchDAO
*/

@Component("searchDao")
public class SearchDAOImpl implements SearchDAO {

Expand Down Expand Up @@ -291,12 +294,11 @@ public class SearchDAOImpl implements SearchDAO {
@Value("${media.dir:/data/biocache-media/}")
public static String biocacheMediaDir = "/data/biocache-media/";

private volatile Set<IndexFieldDTO> indexFields = null;
private volatile Map<String, IndexFieldDTO> indexFieldMap = null;

private volatile Set<IndexFieldDTO> indexFields = RestartDataService.get(this, "indexFields", new TypeReference<TreeSet<IndexFieldDTO>>(){}, TreeSet.class);
private volatile Map<String, IndexFieldDTO> indexFieldMap = RestartDataService.get(this, "indexFieldMap", new TypeReference<HashMap<String, IndexFieldDTO>>(){}, HashMap.class);
private final Map<String, StatsIndexFieldDTO> rangeFieldCache = new HashMap<String, StatsIndexFieldDTO>();

private Set<String> authIndexFields = null;
private Set<String> authIndexFields = new HashSet<String>();

/**
* SOLR index version for client app caching use.
Expand Down Expand Up @@ -367,7 +369,11 @@ private SolrServer initServer() {
}
// TODO: There was a note about possible issues with the following two lines
Set<IndexFieldDTO> indexedFields = getIndexedFields();
downloadFields = new DownloadFields(indexedFields, messageSource, layersService);
if (downloadFields == null) {
downloadFields = new DownloadFields(indexedFields, messageSource, layersService);
} else {
downloadFields.update(indexedFields);
}
} catch (Exception ex) {
logger.error("Error initialising embedded SOLR server: " + ex.getMessage(), ex);
}
Expand All @@ -377,25 +383,33 @@ private SolrServer initServer() {
return result;
}

@PostConstruct
public void init() {
initServer();
}

public Set<String> getAuthIndexFields() {
if (authIndexFields == null) {
if (authIndexFields.size() == 0) {
//set up the hash set of the fields that need to have the authentication service substitute
if (logger.isDebugEnabled()) {
logger.debug("Auth substitution fields to use: " + authServiceFields);
}
authIndexFields = new java.util.HashSet<String>();
CollectionUtils.mergeArrayIntoCollection(authServiceFields.split(","), authIndexFields);
Set set = new java.util.HashSet<String>();
CollectionUtils.mergeArrayIntoCollection(authServiceFields.split(","), set);
authIndexFields = set;
}
return authIndexFields;
}

public void refreshCaches() {
initServer();

collectionCache.updateCache();
//empties the range cache to allow the settings to be recalculated.
rangeFieldCache.clear();
try {
//update indexed fields
downloadFields = new DownloadFields(getIndexedFields(true), messageSource, layersService);
downloadFields.update(getIndexedFields(true));
} catch (Exception e) {
logger.error("Unable to refresh cache.", e);
}
Expand Down Expand Up @@ -3718,16 +3732,19 @@ public Set<IndexFieldDTO> getIndexedFields() throws Exception {
@Cacheable(cacheName = "getIndexedFields")
public Set<IndexFieldDTO> getIndexedFields(boolean update) throws Exception {
Set<IndexFieldDTO> result = indexFields;
if (result == null || update) {
if (result.size() == 0 || update) {
synchronized (solrIndexVersionLock) {
result = indexFields;
if (result == null || update) {
result = indexFields = getIndexFieldDetails(null);
Map<String, IndexFieldDTO> resultMap = new HashMap<String, IndexFieldDTO>();
for (IndexFieldDTO field : result) {
resultMap.put(field.getName(), field);
if (result.size() == 0 || update) {
result = getIndexFieldDetails(null);
if (result != null && result.size() > 0) {
Map<String, IndexFieldDTO> resultMap = new HashMap<String, IndexFieldDTO>();
for (IndexFieldDTO field : result) {
resultMap.put(field.getName(), field);
}
indexFields = result;
indexFieldMap = resultMap;
}
indexFieldMap = resultMap;
}
}
}
Expand Down
5 changes: 3 additions & 2 deletions src/main/java/au/org/ala/biocache/dto/FacetThemes.java
View file
Expand Up @@ -134,11 +134,12 @@ public static List<FacetTheme> getAllThemes() {
}

private void initAllFacets() {
facetsMap.clear();
LinkedHashMap<String, FacetDTO> map = new LinkedHashMap<String, FacetDTO>();
for (FacetTheme theme : allThemes) {
for(FacetDTO f : theme.getFacets()) {
facetsMap.put(f.getField(), f);
map.put(f.getField(), f);
}
facetsMap = map;
allFacets = facetsMap.keySet().toArray(new String[]{});
allFacetsLimited = allFacets != null && allFacets.length > facetsDefaultMax ? Arrays.copyOfRange(allFacets, 0, facetsDefaultMax) : allFacets;
}
Expand Down
31 changes: 21 additions & 10 deletions src/main/java/au/org/ala/biocache/service/AlaLayersService.java
View file
Expand Up @@ -14,6 +14,7 @@
***************************************************************************/
package au.org.ala.biocache.service;

import com.fasterxml.jackson.core.type.TypeReference;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -41,8 +42,8 @@ public class AlaLayersService implements LayersService {

private final static Logger logger = LoggerFactory.getLogger(AlaLayersService.class);

private Map<String,String> idToNameMap = new HashMap<String, String>();
private List<Map<String,Object>> layers = new ArrayList<Map<String,Object>>();
private Map<String,String> idToNameMap = RestartDataService.get(this, "idToNameMap", new TypeReference<HashMap<String, String>>(){}, HashMap.class);
private List<Map<String,Object>> layers = RestartDataService.get(this, "layers", new TypeReference<ArrayList<Map<String, Object>>>(){}, ArrayList.class);
private Map<String,String> extraLayers = new HashMap<String,String>();

//NC 20131018: Allow cache to be disabled via config (enabled by default)
Expand All @@ -58,9 +59,9 @@ public class AlaLayersService implements LayersService {
@Value("${layers.service.url:http://spatial.ala.org.au/ws}")
protected String layersServiceUrl;

protected Map<String, Integer> distributions = new HashMap<String, Integer>();
protected Map<String, Integer> checklists = new HashMap<String, Integer>();
protected Map<String, Integer> tracks = new HashMap<String, Integer>();
protected Map<String, Integer> distributions = RestartDataService.get(this, "distributions", new TypeReference<HashMap<String, Integer>>(){}, HashMap.class);
protected Map<String, Integer> checklists = RestartDataService.get(this, "checklists", new TypeReference<HashMap<String, Integer>>(){}, HashMap.class);
protected Map<String, Integer> tracks = RestartDataService.get(this, "tracks", new TypeReference<HashMap<String, Integer>>(){}, HashMap.class);

@Inject
private RestOperations restTemplate; // NB MappingJacksonHttpMessageConverter() injected by Spring
Expand All @@ -78,18 +79,28 @@ public Map<String, String> getLayerNameMap() {

@Scheduled(fixedDelay = 43200000)// schedule to run every 12 hours
public void refreshCache(){
if (layers.size() > 0) {
//data exists, no need to wait
wait.countDown();
}

//initialise the cache based on the values at http://spatial.ala.org.au/ws/fields
if(enabled){
//create a tmp map
Map<String,String> tmpMap = new HashMap<String,String>();
layers = restTemplate.getForObject(spatialUrl, List.class);
Map tmpMap = new HashMap<String,String>();
List list = restTemplate.getForObject(spatialUrl, List.class);
if (list != null && list.size() > 0) layers = list;
for(Map<String,Object> values : layers){
tmpMap.put((String)values.get("id"), (String)values.get("desc"));
}
idToNameMap = tmpMap;

distributions = initDistribution("distributions");
checklists = initDistribution("checklists");
if (tmpMap.size() > 0) idToNameMap = tmpMap;

tmpMap = initDistribution("distributions");
if (tmpMap.size() > 0) distributions = tmpMap;

tmpMap = initDistribution("checklists");
if (tmpMap.size() > 0) checklists = tmpMap;

//TODO: initialize tracks only when webservices are available
//tracks = initDistribution("tracks");
Expand Down
45 changes: 30 additions & 15 deletions src/main/java/au/org/ala/biocache/service/AuthService.java
View file
Expand Up @@ -14,7 +14,7 @@
***************************************************************************/
package au.org.ala.biocache.service;

import org.apache.commons.collections.MapUtils;
import com.fasterxml.jackson.core.type.TypeReference;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
Expand Down Expand Up @@ -59,9 +59,9 @@ public class AuthService {
@Value("${caches.auth.enabled:true}")
protected Boolean enabled = true;
// Keep a reference to the output Map in case subsequent web service lookups fail
protected Map<String, String> userNamesById = new HashMap<String, String>();
protected Map<String, String> userNamesByNumericIds = new HashMap<String, String>();
protected Map<String, String> userEmailToId = new HashMap<String, String>();
protected Map<String, String> userNamesById = RestartDataService.get(this, "userNamesById", new TypeReference<HashMap<String, String>>(){}, HashMap.class);
protected Map<String, String> userNamesByNumericIds = RestartDataService.get(this, "userNamesByNumericIds", new TypeReference<HashMap<String, String>>(){}, HashMap.class);
protected Map<String, String> userEmailToId = RestartDataService.get(this, "userEmailToId", new TypeReference<HashMap<String, String>>(){}, HashMap.class);

public AuthService() {
logger.info("Instantiating AuthService: " + this);
Expand Down Expand Up @@ -113,7 +113,8 @@ private void loadMapOfAllUserNamesById() {
final String jsonUri = userDetailsUrl + userNamesForIdPath;
try {
logger.info("authCache requesting: " + jsonUri);
userNamesById = restTemplate.postForObject(jsonUri, null, Map.class);
Map m = restTemplate.postForObject(jsonUri, null, Map.class);
if (m != null && m.size() > 0) userNamesById = m;
} catch (Exception ex) {
logger.error("RestTemplate error for " + jsonUri + ": " + ex.getMessage(), ex);
}
Expand All @@ -123,7 +124,8 @@ private void loadMapOfAllUserNamesByNumericId() {
final String jsonUri = userDetailsUrl + userNamesForNumericIdPath;
try {
logger.info("authCache requesting: " + jsonUri);
userNamesByNumericIds = restTemplate.postForObject(jsonUri, null, Map.class);
Map m = restTemplate.postForObject(jsonUri, null, Map.class);
if (m != null && m.size() > 0) userNamesByNumericIds = m;
} catch (Exception ex) {
logger.error("RestTemplate error for " + jsonUri + ": " + ex.getMessage(), ex);
}
Expand All @@ -133,7 +135,8 @@ private void loadMapOfEmailToUserId() {
final String jsonUri = userDetailsUrl + userNamesFullPath;
try {
logger.info("authCache requesting: " + jsonUri);
userEmailToId = restTemplate.postForObject(jsonUri, null, Map.class);
Map m = restTemplate.postForObject(jsonUri, null, Map.class);
if (m != null && m.size() > 0) userEmailToId = m;
logger.info("authCache userEmail cache: " + userEmailToId.size());
if(userEmailToId.size()>0){
String email = userEmailToId.keySet().iterator().next();
Expand All @@ -148,15 +151,27 @@ private void loadMapOfEmailToUserId() {
@Scheduled(fixedDelay = 600000) // schedule to run every 10 min
//@Async NC 2013-07-29: Disabled the Async so that we don't get bombarded with calls.
public void reloadCaches() {
if(enabled){
logger.info("Triggering reload of auth user names");
loadMapOfAllUserNamesById();
loadMapOfAllUserNamesByNumericId();
loadMapOfEmailToUserId();
logger.info("Finished reload of auth user names");
} else{
logger.info("Authentication Cache has been disabled");
Thread thread = new Thread() {
@Override
public void run() {
if(enabled){
logger.info("Triggering reload of auth user names");
loadMapOfAllUserNamesById();
loadMapOfAllUserNamesByNumericId();
loadMapOfEmailToUserId();
logger.info("Finished reload of auth user names");
} else{
logger.info("Authentication Cache has been disabled");
}
}
};

if (userDetailsPath.length() > 0) {
thread.start();
} else {
thread.run();
}

}

public List getUserRoles(String userId) {
Expand Down
18 changes: 15 additions & 3 deletions src/main/java/au/org/ala/biocache/service/ListsService.java
View file
Expand Up @@ -14,6 +14,7 @@
***************************************************************************/
package au.org.ala.biocache.service;

import com.fasterxml.jackson.core.type.TypeReference;
import org.apache.log4j.Logger;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
Expand Down Expand Up @@ -43,7 +44,7 @@ public class ListsService {
@Value("${list.tool.url:http://lists.ala.org.au}")
private String speciesListUrl;

private Map<String, Map<String, Set<String>>> data = new HashMap();
private Map<String, Map<String, Set<String>>> data = RestartDataService.get(this, "data", new TypeReference<HashMap<String, Map<String, Set<String>>>>(){}, HashMap.class);

@PostConstruct
private void init() {
Expand All @@ -62,14 +63,25 @@ public Map<String, Map<String, Set<String>>> getValues() {

@Scheduled(fixedDelay = 43200000)// schedule to run every 12 hours
public void refreshCache() {
if (data.size() > 0) {
//data exists, no need to wait
wait.countDown();
}

if (enabled) {
try {
HashMap map = new HashMap();

Map threatened = restTemplate.getForObject(new URI(speciesListUrl + "/ws/speciesList/?isThreatened=eq:true&isAuthoritative=eq:true"), Map.class);
Map invasive = restTemplate.getForObject(new URI(speciesListUrl + "/ws/speciesList/?isInvasive=eq:true&isAuthoritative=eq:true"), Map.class);

data.put("Conservation", getItemsMap(threatened));
data.put("Invasive", getItemsMap(invasive));
if ((threatened != null && threatened.size() > 0) ||
(invasive != null && invasive.size() > 0)) {
map.put("Conservation", getItemsMap(threatened));
map.put("Invasive", getItemsMap(invasive));

data = map;
}
} catch (Exception e) {
logger.error("failed to get species lists for threatened or invasive species", e);
}
Expand Down

0 comments on commit 8cf5de4

Please sign in to comment.