Skip to content

Commit

Permalink
Allow metadata table cache to be bypassed by ModelBean (#2359)
Browse files Browse the repository at this point in the history
  • Loading branch information
@keith-ratcliffe
keith-ratcliffe committed May 30, 2024
1 parent 3d27c1e commit b95794e
Show file tree
Hide file tree
Showing 3 changed files with 109 additions and 19 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@
<version.maven-install-plugin>2.5.2</version.maven-install-plugin>
<version.metrics-cdi>1.6.0</version.metrics-cdi>
<version.microservice.accumulo-api>3.0.0</version.microservice.accumulo-api>
<version.microservice.accumulo-utils>3.0.2</version.microservice.accumulo-utils>
<version.microservice.accumulo-utils>3.0.3-SNAPSHOT</version.microservice.accumulo-utils>
<version.microservice.audit-api>3.0.0</version.microservice.audit-api>
<version.microservice.authorization-api>3.0.0</version.microservice.authorization-api>
<version.microservice.base-rest-responses>3.0.0</version.microservice.base-rest-responses>
Expand Down
116 changes: 103 additions & 13 deletions web-services/model/src/main/java/datawave/webservice/query/model/ModelBean.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import javax.interceptor.Interceptors;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.FormParam;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
Expand All @@ -37,6 +38,7 @@
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
Expand All @@ -58,6 +60,8 @@
import datawave.security.util.ScannerHelper;
import datawave.webservice.common.cache.AccumuloTableCache;
import datawave.webservice.common.connection.AccumuloConnectionFactory;
import datawave.webservice.common.connection.WrappedAccumuloClient;
import datawave.webservice.common.connection.WrappedScannerHelper;
import datawave.webservice.common.exception.DatawaveWebApplicationException;
import datawave.webservice.common.exception.NotFoundException;
import datawave.webservice.common.exception.PreConditionFailedException;
Expand Down Expand Up @@ -86,7 +90,7 @@ public class ModelBean {
private static final long BATCH_WRITER_MAX_MEMORY = 10845760;
private static final int BATCH_WRITER_MAX_THREADS = 2;

private static final HashSet<String> RESERVED_COLF_VALUES = Sets.newHashSet("e", "i", "ri", "f", "tf", "m", "desc", "edge", "t", "n", "h");
private static final HashSet<String> RESERVED_COLF_VALUES = Sets.newHashSet("e", "i", "ri", "f", "tf", "m", "desc", "edge", "t", "n", "h", "version");

@Inject
@ConfigProperty(name = "dw.model.defaultTableName", defaultValue = DEFAULT_MODEL_TABLE_NAME)
Expand Down Expand Up @@ -127,6 +131,32 @@ public class ModelBean {
@GZIP
@Interceptors(ResponseInterceptor.class)
public ModelList listModelNames(@QueryParam("modelTableName") String modelTableName) {
return listModelNames(modelTableName, false);
}

/**
* <strong>Administrator credentials required.</strong> Get the names of the models, optionally bypassing the local model cache in order to avoid reading
* stale data
*
*
* @param modelTableName
* name of the table that contains the model
* @param skipCache
* if true, bypasses reads of cached model data, forcing a read from the data source instead
* @return datawave.webservice.model.ModelList
* @RequestHeader X-ProxiedEntitiesChain use when proxying request for user
*
* @HTTP 200 success
* @HTTP 500 internal server error
*/
@GET
@Produces({"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf",
"application/x-protostuff", "text/html"})
@Path("admin/list")
@GZIP
@RolesAllowed({"Administrator", "JBossAdministrator"})
@Interceptors(ResponseInterceptor.class)
public ModelList listModelNames(@QueryParam("modelTableName") String modelTableName, @QueryParam("skipCache") @DefaultValue("true") boolean skipCache) {

if (modelTableName == null) {
modelTableName = defaultModelTableName;
Expand All @@ -152,7 +182,7 @@ public ModelList listModelNames(@QueryParam("modelTableName") String modelTableN
try {
Map<String,String> trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace());
client = connectionFactory.getClient(AccumuloConnectionFactory.Priority.LOW, trackingMap);
try (Scanner scanner = ScannerHelper.createScanner(client, this.checkModelTableName(modelTableName), cbAuths)) {
try (Scanner scanner = createScanner(modelTableName, cbAuths, client, skipCache)) {
for (Entry<Key,Value> entry : scanner) {
String colf = entry.getKey().getColumnFamily().toString();
if (!RESERVED_COLF_VALUES.contains(colf) && !modelNames.contains(colf)) {
Expand Down Expand Up @@ -190,6 +220,8 @@ else if (parts.length == 2)
* the model
* @param modelTableName
* name of the table that contains the model
* @param skipCache
* if true, bypasses reads of cached model data, forcing a read from the data source instead
* @return datawave.webservice.result.VoidResponse
* @RequestHeader X-ProxiedEntitiesChain use when proxying request for user
*
Expand All @@ -205,7 +237,8 @@ else if (parts.length == 2)
@GZIP
@RolesAllowed({"Administrator", "JBossAdministrator"})
@Interceptors(ResponseInterceptor.class)
public VoidResponse importModel(datawave.webservice.model.Model model, @QueryParam("modelTableName") String modelTableName) {
public VoidResponse importModel(datawave.webservice.model.Model model, @QueryParam("modelTableName") String modelTableName,
@QueryParam("skipCache") @DefaultValue("true") boolean skipCache) {

if (modelTableName == null) {
modelTableName = defaultModelTableName;
Expand All @@ -216,7 +249,7 @@ public VoidResponse importModel(datawave.webservice.model.Model model, @QueryPar
}
VoidResponse response = new VoidResponse();

ModelList models = listModelNames(modelTableName);
ModelList models = listModelNames(modelTableName, skipCache);
if (models.getNames().contains(model.getName()))
throw new PreConditionFailedException(null, response);

Expand All @@ -232,6 +265,8 @@ public VoidResponse importModel(datawave.webservice.model.Model model, @QueryPar
* model name to delete
* @param modelTableName
* name of the table that contains the model
* @param skipCache
* if true, bypasses reads of cached model data, forcing a read from the data source instead
* @return datawave.webservice.result.VoidResponse
* @RequestHeader X-ProxiedEntitiesChain use when proxying request for user
*
Expand All @@ -246,27 +281,28 @@ public VoidResponse importModel(datawave.webservice.model.Model model, @QueryPar
@GZIP
@RolesAllowed({"Administrator", "JBossAdministrator"})
@Interceptors({RequiredInterceptor.class, ResponseInterceptor.class})
public VoidResponse deleteModel(@Required("name") @PathParam("name") String name, @QueryParam("modelTableName") String modelTableName) {
public VoidResponse deleteModel(@Required("name") @PathParam("name") String name, @QueryParam("modelTableName") String modelTableName,
@QueryParam("skipCache") @DefaultValue("true") boolean skipCache) {

if (modelTableName == null) {
modelTableName = defaultModelTableName;
}

return deleteModel(name, modelTableName, true);
return deleteModel(name, modelTableName, true, skipCache);
}

private VoidResponse deleteModel(@Required("name") String name, String modelTableName, boolean reloadCache) {
private VoidResponse deleteModel(@Required("name") String name, String modelTableName, boolean reloadCache, boolean skipCache) {
if (log.isDebugEnabled()) {
log.debug("model name: " + name);
log.debug("modelTableName: " + (null == modelTableName ? "" : modelTableName));
}
VoidResponse response = new VoidResponse();

ModelList models = listModelNames(modelTableName);
ModelList models = listModelNames(modelTableName, skipCache);
if (!models.getNames().contains(name))
throw new NotFoundException(null, response);

datawave.webservice.model.Model model = getModel(name, modelTableName);
datawave.webservice.model.Model model = getModel(name, modelTableName, skipCache);
deleteMapping(model, modelTableName, reloadCache);

return response;
Expand All @@ -281,6 +317,8 @@ private VoidResponse deleteModel(@Required("name") String name, String modelTabl
* name of copied model
* @param modelTableName
* name of the table that contains the model
* @param skipCache
* if true, bypasses reads of cached model data, forcing a read from the data source instead
* @return datawave.webservice.result.VoidResponse
* @RequestHeader X-ProxiedEntitiesChain use when proxying request for user
*
Expand All @@ -296,17 +334,17 @@ private VoidResponse deleteModel(@Required("name") String name, String modelTabl
@RolesAllowed({"Administrator", "JBossAdministrator"})
@Interceptors({RequiredInterceptor.class, ResponseInterceptor.class})
public VoidResponse cloneModel(@Required("name") @FormParam("name") String name, @Required("newName") @FormParam("newName") String newName,
@FormParam("modelTableName") String modelTableName) {
@FormParam("modelTableName") String modelTableName, @FormParam("skipCache") @DefaultValue("true") boolean skipCache) {
VoidResponse response = new VoidResponse();

if (modelTableName == null) {
modelTableName = defaultModelTableName;
}

datawave.webservice.model.Model model = getModel(name, modelTableName);
datawave.webservice.model.Model model = getModel(name, modelTableName, skipCache);
// Set the new name
model.setName(newName);
importModel(model, modelTableName);
importModel(model, modelTableName, skipCache);
return response;
}

Expand All @@ -331,6 +369,35 @@ public VoidResponse cloneModel(@Required("name") @FormParam("name") String name,
@GZIP
@Interceptors({RequiredInterceptor.class, ResponseInterceptor.class})
public datawave.webservice.model.Model getModel(@Required("name") @PathParam("name") String name, @QueryParam("modelTableName") String modelTableName) {
return getModel(name, modelTableName, false);
}

/**
* <strong>Administrator credentials required.</strong> Retrieve the model and all of its mappings, optionally bypassing the local model cache in order to
* avoid reading stale data
*
* @param name
* model name
* @param modelTableName
* name of the table that contains the model
* @param skipCache
* if true, bypasses reads of cached model data, forcing a read from the data source instead
* @return datawave.webservice.model.Model
* @RequestHeader X-ProxiedEntitiesChain use when proxying request for user
*
* @HTTP 200 success
* @HTTP 404 model not found
* @HTTP 500 internal server error
*/
@GET
@Produces({"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf",
"application/x-protostuff", "text/html"})
@Path("admin/{name}")
@GZIP
@RolesAllowed({"Administrator", "JBossAdministrator"})
@Interceptors({RequiredInterceptor.class, ResponseInterceptor.class})
public datawave.webservice.model.Model getModel(@Required("name") @PathParam("name") String name, @QueryParam("modelTableName") String modelTableName,
@QueryParam("skipCache") @DefaultValue("true") boolean skipCache) {

if (modelTableName == null) {
modelTableName = defaultModelTableName;
Expand All @@ -355,7 +422,7 @@ public datawave.webservice.model.Model getModel(@Required("name") @PathParam("na
try {
Map<String,String> trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace());
client = connectionFactory.getClient(AccumuloConnectionFactory.Priority.LOW, trackingMap);
try (Scanner scanner = ScannerHelper.createScanner(client, this.checkModelTableName(modelTableName), cbAuths)) {
try (Scanner scanner = createScanner(modelTableName, cbAuths, client, skipCache)) {
IteratorSetting cfg = new IteratorSetting(21, "colfRegex", RegExFilter.class.getName());
cfg.addOption(RegExFilter.COLF_REGEX, "^" + name + "(\\x00.*)?");
scanner.addScanIterator(cfg);
Expand Down Expand Up @@ -543,4 +610,27 @@ private String checkModelTableName(String tableName) {
else
return tableName;
}

/**
* Scanner factory method
*
* @param tableName
* the table name
* @param auths
* the scanner auths
* @param client
* the AccumuloClient instance
* @param skipCache
* if true, forces a read of tableName, bypassing the webservice's internal cache. Ignored when the client is anything other than
* {@link WrappedAccumuloClient}
* @return
* @throws TableNotFoundException
*/
private Scanner createScanner(String tableName, Set<Authorizations> auths, AccumuloClient client, boolean skipCache) throws TableNotFoundException {
if (client instanceof WrappedAccumuloClient) {
return WrappedScannerHelper.createScanner((WrappedAccumuloClient) client, this.checkModelTableName(tableName), auths, skipCache);
} else {
return ScannerHelper.createScanner(client, this.checkModelTableName(tableName), auths);
}
}
}
10 changes: 5 additions & 5 deletions web-services/model/src/test/java/datawave/webservice/query/model/ModelBeanTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ public void testModelImportNoTable() throws Exception {
EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal);
PowerMock.replayAll();

bean.importModel(MODEL_ONE, (String) null);
bean.importModel(MODEL_ONE, (String) null, false);
PowerMock.verifyAll();
}

Expand All @@ -160,7 +160,7 @@ private void importModels() throws Exception {
EasyMock.expect(cache.reloadCache(ModelBean.DEFAULT_MODEL_TABLE_NAME)).andReturn(null);
PowerMock.replayAll();

bean.importModel(MODEL_ONE, (String) null);
bean.importModel(MODEL_ONE, (String) null, false);
PowerMock.verifyAll();
PowerMock.resetAll();

Expand All @@ -178,7 +178,7 @@ private void importModels() throws Exception {
EasyMock.expect(cache.reloadCache(ModelBean.DEFAULT_MODEL_TABLE_NAME)).andReturn(null);
PowerMock.replayAll();

bean.importModel(MODEL_TWO, (String) null);
bean.importModel(MODEL_TWO, (String) null, false);

PowerMock.verifyAll();
}
Expand Down Expand Up @@ -247,7 +247,7 @@ public void testModelDelete() throws Exception {
EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP);
PowerMock.replayAll();

bean.deleteModel(MODEL_TWO.getName(), (String) null);
bean.deleteModel(MODEL_TWO.getName(), (String) null, false);
PowerMock.verifyAll();
PowerMock.resetAll();

Expand Down Expand Up @@ -318,7 +318,7 @@ public void testCloneModel() throws Exception {
EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP);
PowerMock.replayAll();

bean.cloneModel(MODEL_ONE.getName(), "MODEL2", (String) null);
bean.cloneModel(MODEL_ONE.getName(), "MODEL2", (String) null, false);
PowerMock.verifyAll();
PowerMock.resetAll();
EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal);
Expand Down

0 comments on commit b95794e

Please sign in to comment.