From 38b0321d46bff21bfd0744027f41e59ba7a73714 Mon Sep 17 00:00:00 2001 From: keith-ratcliffe Date: Tue, 30 Apr 2024 00:41:41 +0000 Subject: [PATCH] Allow table cache to be bypassed by ModelBean (#2359) --- pom.xml | 2 +- .../webservice/query/model/ModelBean.java | 56 ++++++++++++++----- .../webservice/query/model/ModelBeanTest.java | 22 ++++---- 3 files changed, 54 insertions(+), 26 deletions(-) diff --git a/pom.xml b/pom.xml index bb58df0fc6e..475f6ae5583 100644 --- a/pom.xml +++ b/pom.xml @@ -94,7 +94,7 @@ 2.5.2 1.6.0 3.0.0 - 3.0.2 + 3.0.3-SNAPSHOT 3.0.0 3.0.0 3.0.0 diff --git a/web-services/model/src/main/java/datawave/webservice/query/model/ModelBean.java b/web-services/model/src/main/java/datawave/webservice/query/model/ModelBean.java index c597d0e79fd..1ce4224c6e3 100644 --- a/web-services/model/src/main/java/datawave/webservice/query/model/ModelBean.java +++ b/web-services/model/src/main/java/datawave/webservice/query/model/ModelBean.java @@ -37,6 +37,7 @@ import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.client.MutationsRejectedException; import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; @@ -58,6 +59,8 @@ import datawave.security.util.ScannerHelper; import datawave.webservice.common.cache.AccumuloTableCache; import datawave.webservice.common.connection.AccumuloConnectionFactory; +import datawave.webservice.common.connection.WrappedAccumuloClient; +import datawave.webservice.common.connection.WrappedScannerHelper; import datawave.webservice.common.exception.DatawaveWebApplicationException; import datawave.webservice.common.exception.NotFoundException; import datawave.webservice.common.exception.PreConditionFailedException; @@ -126,7 +129,7 @@ public class ModelBean { @Path("/list") @GZIP @Interceptors(ResponseInterceptor.class) - public ModelList listModelNames(@QueryParam("modelTableName") String modelTableName) { + public ModelList listModelNames(@QueryParam("modelTableName") String modelTableName, @QueryParam("bypassTableCache") boolean bypassTableCache) { if (modelTableName == null) { modelTableName = defaultModelTableName; @@ -152,7 +155,7 @@ public ModelList listModelNames(@QueryParam("modelTableName") String modelTableN try { Map trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace()); client = connectionFactory.getClient(AccumuloConnectionFactory.Priority.LOW, trackingMap); - try (Scanner scanner = ScannerHelper.createScanner(client, this.checkModelTableName(modelTableName), cbAuths)) { + try (Scanner scanner = createScanner(modelTableName, cbAuths, client, bypassTableCache)) { for (Entry entry : scanner) { String colf = entry.getKey().getColumnFamily().toString(); if (!RESERVED_COLF_VALUES.contains(colf) && !modelNames.contains(colf)) { @@ -205,7 +208,8 @@ else if (parts.length == 2) @GZIP @RolesAllowed({"Administrator", "JBossAdministrator"}) @Interceptors(ResponseInterceptor.class) - public VoidResponse importModel(datawave.webservice.model.Model model, @QueryParam("modelTableName") String modelTableName) { + public VoidResponse importModel(datawave.webservice.model.Model model, @QueryParam("modelTableName") String modelTableName, + @QueryParam("bypassTableCache") boolean bypassTableCache) { if (modelTableName == null) { modelTableName = defaultModelTableName; @@ -216,7 +220,7 @@ public VoidResponse importModel(datawave.webservice.model.Model model, @QueryPar } VoidResponse response = new VoidResponse(); - ModelList models = listModelNames(modelTableName); + ModelList models = listModelNames(modelTableName, bypassTableCache); if (models.getNames().contains(model.getName())) throw new PreConditionFailedException(null, response); @@ -246,27 +250,28 @@ public VoidResponse importModel(datawave.webservice.model.Model model, @QueryPar @GZIP @RolesAllowed({"Administrator", "JBossAdministrator"}) @Interceptors({RequiredInterceptor.class, ResponseInterceptor.class}) - public VoidResponse deleteModel(@Required("name") @PathParam("name") String name, @QueryParam("modelTableName") String modelTableName) { + public VoidResponse deleteModel(@Required("name") @PathParam("name") String name, @QueryParam("modelTableName") String modelTableName, + @QueryParam("bypassTableCache") boolean bypassTableCache) { if (modelTableName == null) { modelTableName = defaultModelTableName; } - return deleteModel(name, modelTableName, true); + return deleteModel(name, modelTableName, true, bypassTableCache); } - private VoidResponse deleteModel(@Required("name") String name, String modelTableName, boolean reloadCache) { + private VoidResponse deleteModel(@Required("name") String name, String modelTableName, boolean reloadCache, boolean bypassTableCache) { if (log.isDebugEnabled()) { log.debug("model name: " + name); log.debug("modelTableName: " + (null == modelTableName ? "" : modelTableName)); } VoidResponse response = new VoidResponse(); - ModelList models = listModelNames(modelTableName); + ModelList models = listModelNames(modelTableName, bypassTableCache); if (!models.getNames().contains(name)) throw new NotFoundException(null, response); - datawave.webservice.model.Model model = getModel(name, modelTableName); + datawave.webservice.model.Model model = getModel(name, modelTableName, bypassTableCache); deleteMapping(model, modelTableName, reloadCache); return response; @@ -296,17 +301,17 @@ private VoidResponse deleteModel(@Required("name") String name, String modelTabl @RolesAllowed({"Administrator", "JBossAdministrator"}) @Interceptors({RequiredInterceptor.class, ResponseInterceptor.class}) public VoidResponse cloneModel(@Required("name") @FormParam("name") String name, @Required("newName") @FormParam("newName") String newName, - @FormParam("modelTableName") String modelTableName) { + @FormParam("modelTableName") String modelTableName, @FormParam("bypassTableCache") boolean bypassTableCache) { VoidResponse response = new VoidResponse(); if (modelTableName == null) { modelTableName = defaultModelTableName; } - datawave.webservice.model.Model model = getModel(name, modelTableName); + datawave.webservice.model.Model model = getModel(name, modelTableName, bypassTableCache); // Set the new name model.setName(newName); - importModel(model, modelTableName); + importModel(model, modelTableName, bypassTableCache); return response; } @@ -330,7 +335,8 @@ public VoidResponse cloneModel(@Required("name") @FormParam("name") String name, @Path("/{name}") @GZIP @Interceptors({RequiredInterceptor.class, ResponseInterceptor.class}) - public datawave.webservice.model.Model getModel(@Required("name") @PathParam("name") String name, @QueryParam("modelTableName") String modelTableName) { + public datawave.webservice.model.Model getModel(@Required("name") @PathParam("name") String name, @QueryParam("modelTableName") String modelTableName, + @QueryParam("bypassTableCache") boolean bypassTableCache) { if (modelTableName == null) { modelTableName = defaultModelTableName; @@ -355,7 +361,7 @@ public datawave.webservice.model.Model getModel(@Required("name") @PathParam("na try { Map trackingMap = connectionFactory.getTrackingMap(Thread.currentThread().getStackTrace()); client = connectionFactory.getClient(AccumuloConnectionFactory.Priority.LOW, trackingMap); - try (Scanner scanner = ScannerHelper.createScanner(client, this.checkModelTableName(modelTableName), cbAuths)) { + try (Scanner scanner = createScanner(modelTableName, cbAuths, client, bypassTableCache)) { IteratorSetting cfg = new IteratorSetting(21, "colfRegex", RegExFilter.class.getName()); cfg.addOption(RegExFilter.COLF_REGEX, "^" + name + "(\\x00.*)?"); scanner.addScanIterator(cfg); @@ -543,4 +549,26 @@ private String checkModelTableName(String tableName) { else return tableName; } + + /** + * Scanner instance factory method + * + * @param tableName + * the table name + * @param auths + * the scanner auths + * @param client + * the AccumuloClient instance + * @param bypassTableCache + * allows table cache to be bypassed when the client is {@link WrappedAccumuloClient}. Ignored otherwise + * @return + * @throws TableNotFoundException + */ + private Scanner createScanner(String tableName, Set auths, AccumuloClient client, boolean bypassTableCache) throws TableNotFoundException { + if (client instanceof WrappedAccumuloClient) { + return WrappedScannerHelper.createScanner((WrappedAccumuloClient) client, this.checkModelTableName(tableName), auths, bypassTableCache); + } else { + return ScannerHelper.createScanner(client, this.checkModelTableName(tableName), auths); + } + } } diff --git a/web-services/model/src/test/java/datawave/webservice/query/model/ModelBeanTest.java b/web-services/model/src/test/java/datawave/webservice/query/model/ModelBeanTest.java index 721c4ba0f62..9028a126dc9 100644 --- a/web-services/model/src/test/java/datawave/webservice/query/model/ModelBeanTest.java +++ b/web-services/model/src/test/java/datawave/webservice/query/model/ModelBeanTest.java @@ -139,7 +139,7 @@ public void testModelImportNoTable() throws Exception { EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); PowerMock.replayAll(); - bean.importModel(MODEL_ONE, (String) null); + bean.importModel(MODEL_ONE, (String) null, false); PowerMock.verifyAll(); } @@ -160,7 +160,7 @@ private void importModels() throws Exception { EasyMock.expect(cache.reloadCache(ModelBean.DEFAULT_MODEL_TABLE_NAME)).andReturn(null); PowerMock.replayAll(); - bean.importModel(MODEL_ONE, (String) null); + bean.importModel(MODEL_ONE, (String) null, false); PowerMock.verifyAll(); PowerMock.resetAll(); @@ -178,7 +178,7 @@ private void importModels() throws Exception { EasyMock.expect(cache.reloadCache(ModelBean.DEFAULT_MODEL_TABLE_NAME)).andReturn(null); PowerMock.replayAll(); - bean.importModel(MODEL_TWO, (String) null); + bean.importModel(MODEL_TWO, (String) null, false); PowerMock.verifyAll(); } @@ -195,7 +195,7 @@ public void testListModels() throws Exception { connectionFactory.returnClient(client); PowerMock.replayAll(); - ModelList list = bean.listModelNames((String) null); + ModelList list = bean.listModelNames((String) null, false); PowerMock.verifyAll(); Assert.assertEquals(2, list.getNames().size()); @@ -215,7 +215,7 @@ public void testModelGet() throws Exception { connectionFactory.returnClient(client); PowerMock.replayAll(); - datawave.webservice.model.Model model = bean.getModel(MODEL_ONE.getName(), (String) null); + datawave.webservice.model.Model model = bean.getModel(MODEL_ONE.getName(), (String) null, false); PowerMock.verifyAll(); Assert.assertEquals(MODEL_ONE, model); @@ -247,7 +247,7 @@ public void testModelDelete() throws Exception { EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); PowerMock.replayAll(); - bean.deleteModel(MODEL_TWO.getName(), (String) null); + bean.deleteModel(MODEL_TWO.getName(), (String) null, false); PowerMock.verifyAll(); PowerMock.resetAll(); @@ -257,7 +257,7 @@ public void testModelDelete() throws Exception { connectionFactory.returnClient(client); PowerMock.replayAll(); try { - bean.getModel(MODEL_TWO.getName(), (String) null); + bean.getModel(MODEL_TWO.getName(), (String) null, false); Assert.fail("getModel should have failed"); } catch (DatawaveWebApplicationException e) { if (e.getResponse().getStatus() == 404) { @@ -276,7 +276,7 @@ public void testModelDelete() throws Exception { EasyMock.expect(connectionFactory.getClient(EasyMock.eq(AccumuloConnectionFactory.Priority.LOW), EasyMock.eq(trackingMap))).andReturn(client); connectionFactory.returnClient(client); PowerMock.replayAll(); - datawave.webservice.model.Model model1 = bean.getModel(MODEL_ONE.getName(), (String) null); + datawave.webservice.model.Model model1 = bean.getModel(MODEL_ONE.getName(), (String) null, false); PowerMock.verifyAll(); Assert.assertEquals(MODEL_ONE, model1); @@ -291,7 +291,7 @@ public void testModelGetInvalidModelName() throws Exception { connectionFactory.returnClient(client); PowerMock.replayAll(); - bean.getModel(MODEL_ONE.getName(), (String) null); + bean.getModel(MODEL_ONE.getName(), (String) null, false); PowerMock.verifyAll(); } @@ -318,7 +318,7 @@ public void testCloneModel() throws Exception { EasyMock.expect(System.currentTimeMillis()).andReturn(TIMESTAMP); PowerMock.replayAll(); - bean.cloneModel(MODEL_ONE.getName(), "MODEL2", (String) null); + bean.cloneModel(MODEL_ONE.getName(), "MODEL2", (String) null, false); PowerMock.verifyAll(); PowerMock.resetAll(); EasyMock.expect(ctx.getCallerPrincipal()).andReturn(principal); @@ -327,7 +327,7 @@ public void testCloneModel() throws Exception { connectionFactory.returnClient(client); PowerMock.replayAll(); - datawave.webservice.model.Model model = bean.getModel("MODEL2", (String) null); + datawave.webservice.model.Model model = bean.getModel("MODEL2", (String) null, false); PowerMock.verifyAll(); MODEL_ONE.setName("MODEL2");