Skip to content

Commit

Permalink
Allow table cache to be bypassed by ModelBean (#2359)
Browse files Browse the repository at this point in the history
  • Loading branch information
@keith-ratcliffe
keith-ratcliffe committed Apr 30, 2024
1 parent 0a262b9 commit 38b0321
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 26 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@
<version.maven-install-plugin>2.5.2</version.maven-install-plugin>
<version.metrics-cdi>1.6.0</version.metrics-cdi>
<version.microservice.accumulo-api>3.0.0</version.microservice.accumulo-api>
<version.microservice.accumulo-utils>3.0.2</version.microservice.accumulo-utils>
<version.microservice.accumulo-utils>3.0.3-SNAPSHOT</version.microservice.accumulo-utils>
<version.microservice.audit-api>3.0.0</version.microservice.audit-api>
<version.microservice.authorization-api>3.0.0</version.microservice.authorization-api>
<version.microservice.base-rest-responses>3.0.0</version.microservice.base-rest-responses>
Expand Down
56 changes: 42 additions & 14 deletions web-services/model/src/main/java/datawave/webservice/query/model/ModelBean.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import org.apache.accumulo.core.client.IteratorSetting;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
Expand All @@ -58,6 +59,8 @@
import datawave.security.util.ScannerHelper;
import datawave.webservice.common.cache.AccumuloTableCache;
import datawave.webservice.common.connection.AccumuloConnectionFactory;
import datawave.webservice.common.connection.WrappedAccumuloClient;
import datawave.webservice.common.connection.WrappedScannerHelper;
import datawave.webservice.common.exception.DatawaveWebApplicationException;
import datawave.webservice.common.exception.NotFoundException;
import datawave.webservice.common.exception.PreConditionFailedException;
Expand Down Expand Up @@ -126,7 +129,7 @@ public class ModelBean {
@Path("/list")
@GZIP
@Interceptors(ResponseInterceptor.class)
public ModelList listModelNames(@QueryParam("modelTableName") String modelTableName) {
public ModelList listModelNames(@QueryParam("modelTableName") String modelTableName, @QueryParam("bypassTableCache") boolean bypassTableCache) {

if (modelTableName == null) {
modelTableName = defaultModelTableName;
Expand All @@ -152,7 +155,7 @@ public ModelList listModelNames(@QueryParam("modelTableName") String modelTableN
try {
Map<String,String> trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace());
client = connectionFactory.getClient(AccumuloConnectionFactory.Priority.LOW, trackingMap);
try (Scanner scanner = ScannerHelper.createScanner(client, this.checkModelTableName(modelTableName), cbAuths)) {
try (Scanner scanner = createScanner(modelTableName, cbAuths, client, bypassTableCache)) {
for (Entry<Key,Value> entry : scanner) {
String colf = entry.getKey().getColumnFamily().toString();
if (!RESERVED_COLF_VALUES.contains(colf) && !modelNames.contains(colf)) {
Expand Down Expand Up @@ -205,7 +208,8 @@ else if (parts.length == 2)
@GZIP
@RolesAllowed({"Administrator", "JBossAdministrator"})
@Interceptors(ResponseInterceptor.class)
public VoidResponse importModel(datawave.webservice.model.Model model, @QueryParam("modelTableName") String modelTableName) {
public VoidResponse importModel(datawave.webservice.model.Model model, @QueryParam("modelTableName") String modelTableName,
@QueryParam("bypassTableCache") boolean bypassTableCache) {

if (modelTableName == null) {
modelTableName = defaultModelTableName;
Expand All @@ -216,7 +220,7 @@ public VoidResponse importModel(datawave.webservice.model.Model model, @QueryPar
}
VoidResponse response = new VoidResponse();

ModelList models = listModelNames(modelTableName);
ModelList models = listModelNames(modelTableName, bypassTableCache);
if (models.getNames().contains(model.getName()))
throw new PreConditionFailedException(null, response);

Expand Down Expand Up @@ -246,27 +250,28 @@ public VoidResponse importModel(datawave.webservice.model.Model model, @QueryPar
@GZIP
@RolesAllowed({"Administrator", "JBossAdministrator"})
@Interceptors({RequiredInterceptor.class, ResponseInterceptor.class})
public VoidResponse deleteModel(@Required("name") @PathParam("name") String name, @QueryParam("modelTableName") String modelTableName) {
public VoidResponse deleteModel(@Required("name") @PathParam("name") String name, @QueryParam("modelTableName") String modelTableName,
@QueryParam("bypassTableCache") boolean bypassTableCache) {

if (modelTableName == null) {
modelTableName = defaultModelTableName;
}

return deleteModel(name, modelTableName, true);
return deleteModel(name, modelTableName, true, bypassTableCache);
}

private VoidResponse deleteModel(@Required("name") String name, String modelTableName, boolean reloadCache) {
private VoidResponse deleteModel(@Required("name") String name, String modelTableName, boolean reloadCache, boolean bypassTableCache) {
if (log.isDebugEnabled()) {
log.debug("model name: " + name);
log.debug("modelTableName: " + (null == modelTableName ? "" : modelTableName));
}
VoidResponse response = new VoidResponse();

ModelList models = listModelNames(modelTableName);
ModelList models = listModelNames(modelTableName, bypassTableCache);
if (!models.getNames().contains(name))
throw new NotFoundException(null, response);

datawave.webservice.model.Model model = getModel(name, modelTableName);
datawave.webservice.model.Model model = getModel(name, modelTableName, bypassTableCache);
deleteMapping(model, modelTableName, reloadCache);

return response;
Expand Down Expand Up @@ -296,17 +301,17 @@ private VoidResponse deleteModel(@Required("name") String name, String modelTabl
@RolesAllowed({"Administrator", "JBossAdministrator"})
@Interceptors({RequiredInterceptor.class, ResponseInterceptor.class})
public VoidResponse cloneModel(@Required("name") @FormParam("name") String name, @Required("newName") @FormParam("newName") String newName,
@FormParam("modelTableName") String modelTableName) {
@FormParam("modelTableName") String modelTableName, @FormParam("bypassTableCache") boolean bypassTableCache) {
VoidResponse response = new VoidResponse();

if (modelTableName == null) {
modelTableName = defaultModelTableName;
}

datawave.webservice.model.Model model = getModel(name, modelTableName);
datawave.webservice.model.Model model = getModel(name, modelTableName, bypassTableCache);
// Set the new name
model.setName(newName);
importModel(model, modelTableName);
importModel(model, modelTableName, bypassTableCache);
return response;
}

Expand All @@ -330,7 +335,8 @@ public VoidResponse cloneModel(@Required("name") @FormParam("name") String name,
@Path("/{name}")
@GZIP
@Interceptors({RequiredInterceptor.class, ResponseInterceptor.class})
public datawave.webservice.model.Model getModel(@Required("name") @PathParam("name") String name, @QueryParam("modelTableName") String modelTableName) {
public datawave.webservice.model.Model getModel(@Required("name") @PathParam("name") String name, @QueryParam("modelTableName") String modelTableName,
@QueryParam("bypassTableCache") boolean bypassTableCache) {

if (modelTableName == null) {
modelTableName = defaultModelTableName;
Expand All @@ -355,7 +361,7 @@ public datawave.webservice.model.Model getModel(@Required("name") @PathParam("na
try {
Map<String,String> trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace());
client = connectionFactory.getClient(AccumuloConnectionFactory.Priority.LOW, trackingMap);
try (Scanner scanner = ScannerHelper.createScanner(client, this.checkModelTableName(modelTableName), cbAuths)) {
try (Scanner scanner = createScanner(modelTableName, cbAuths, client, bypassTableCache)) {
IteratorSetting cfg = new IteratorSetting(21, "colfRegex", RegExFilter.class.getName());
cfg.addOption(RegExFilter.COLF_REGEX, "^" + name + "(\\x00.*)?");
scanner.addScanIterator(cfg);
Expand Down Expand Up @@ -543,4 +549,26 @@ private String checkModelTableName(String tableName) {
else
return tableName;
}

/**
* Scanner instance factory method
*
* @param tableName
* the table name
* @param auths
* the scanner auths
* @param client
* the AccumuloClient instance
* @param bypassTableCache
* allows table cache to be bypassed when the client is {@link WrappedAccumuloClient}. Ignored otherwise
* @return
* @throws TableNotFoundException
*/
private Scanner createScanner(String tableName, Set<Authorizations> auths, AccumuloClient client, boolean bypassTableCache) throws TableNotFoundException {
if (client instanceof WrappedAccumuloClient) {
return WrappedScannerHelper.createScanner((WrappedAccumuloClient) client, this.checkModelTableName(tableName), auths, bypassTableCache);
} else {
return ScannerHelper.createScanner(client, this.checkModelTableName(tableName), auths);
}
}
}
22 changes: 11 additions & 11 deletions web-services/model/src/test/java/datawave/webservice/query/model/ModelBeanTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ public void testModelImportNoTable() throws Exception {
EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal);
PowerMock.replayAll();

bean.importModel(MODEL_ONE, (String) null);
bean.importModel(MODEL_ONE, (String) null, false);
PowerMock.verifyAll();
}

Expand All @@ -160,7 +160,7 @@ private void importModels() throws Exception {
EasyMock.expect(cache.reloadCache(ModelBean.DEFAULT_MODEL_TABLE_NAME)).andReturn(null);
PowerMock.replayAll();

bean.importModel(MODEL_ONE, (String) null);
bean.importModel(MODEL_ONE, (String) null, false);
PowerMock.verifyAll();
PowerMock.resetAll();

Expand All @@ -178,7 +178,7 @@ private void importModels() throws Exception {
EasyMock.expect(cache.reloadCache(ModelBean.DEFAULT_MODEL_TABLE_NAME)).andReturn(null);
PowerMock.replayAll();

bean.importModel(MODEL_TWO, (String) null);
bean.importModel(MODEL_TWO, (String) null, false);

PowerMock.verifyAll();
}
Expand All @@ -195,7 +195,7 @@ public void testListModels() throws Exception {
connectionFactory.returnClient(client);
PowerMock.replayAll();

ModelList list = bean.listModelNames((String) null);
ModelList list = bean.listModelNames((String) null, false);
PowerMock.verifyAll();

Assert.assertEquals(2, list.getNames().size());
Expand All @@ -215,7 +215,7 @@ public void testModelGet() throws Exception {
connectionFactory.returnClient(client);
PowerMock.replayAll();

datawave.webservice.model.Model model = bean.getModel(MODEL_ONE.getName(), (String) null);
datawave.webservice.model.Model model = bean.getModel(MODEL_ONE.getName(), (String) null, false);
PowerMock.verifyAll();

Assert.assertEquals(MODEL_ONE, model);
Expand Down Expand Up @@ -247,7 +247,7 @@ public void testModelDelete() throws Exception {
EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP);
PowerMock.replayAll();

bean.deleteModel(MODEL_TWO.getName(), (String) null);
bean.deleteModel(MODEL_TWO.getName(), (String) null, false);
PowerMock.verifyAll();
PowerMock.resetAll();

Expand All @@ -257,7 +257,7 @@ public void testModelDelete() throws Exception {
connectionFactory.returnClient(client);
PowerMock.replayAll();
try {
bean.getModel(MODEL_TWO.getName(), (String) null);
bean.getModel(MODEL_TWO.getName(), (String) null, false);
Assert.fail("getModel should have failed");
} catch (DatawaveWebApplicationException e) {
if (e.getResponse().getStatus() == 404) {
Expand All @@ -276,7 +276,7 @@ public void testModelDelete() throws Exception {
EasyMock.expect(connectionFactory.getClient(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(client);
connectionFactory.returnClient(client);
PowerMock.replayAll();
datawave.webservice.model.Model model1 = bean.getModel(MODEL_ONE.getName(), (String) null);
datawave.webservice.model.Model model1 = bean.getModel(MODEL_ONE.getName(), (String) null, false);
PowerMock.verifyAll();
Assert.assertEquals(MODEL_ONE, model1);

Expand All @@ -291,7 +291,7 @@ public void testModelGetInvalidModelName() throws Exception {
connectionFactory.returnClient(client);
PowerMock.replayAll();

bean.getModel(MODEL_ONE.getName(), (String) null);
bean.getModel(MODEL_ONE.getName(), (String) null, false);
PowerMock.verifyAll();
}

Expand All @@ -318,7 +318,7 @@ public void testCloneModel() throws Exception {
EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP);
PowerMock.replayAll();

bean.cloneModel(MODEL_ONE.getName(), "MODEL2", (String) null);
bean.cloneModel(MODEL_ONE.getName(), "MODEL2", (String) null, false);
PowerMock.verifyAll();
PowerMock.resetAll();
EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal);
Expand All @@ -327,7 +327,7 @@ public void testCloneModel() throws Exception {
connectionFactory.returnClient(client);
PowerMock.replayAll();

datawave.webservice.model.Model model = bean.getModel("MODEL2", (String) null);
datawave.webservice.model.Model model = bean.getModel("MODEL2", (String) null, false);
PowerMock.verifyAll();

MODEL_ONE.setName("MODEL2");
Expand Down

0 comments on commit 38b0321

Please sign in to comment.