From 6f55333fcb32bac5456edd5e999bf7b7b5534fdf Mon Sep 17 00:00:00 2001 From: Reggie Hayes Date: Wed, 7 Feb 2024 06:58:06 -0500 Subject: [PATCH 01/50] Fields with a state of DROP on normalization failure are not written to the index but are still preserved in the event. --- .../ingest/data/config/ingest/BaseIngestHelper.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java index 8784be6ecd9..6d7d27cd24e 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java @@ -978,17 +978,12 @@ protected void applyNormalizationAndAddToResults(Multimap Date: Wed, 7 Feb 2024 12:31:49 -0500 Subject: [PATCH 02/50] Create docker images in CI (#2264) --- .github/workflows/build-images.yml | 46 ++++ pom.xml | 27 ++ properties/default.properties | 2 + properties/kubernetes.properties | 259 ++++++++++++++++++ warehouse/assemble/datawave/pom.xml | 89 +++++- .../datawave/src/main/docker/Dockerfile | 15 + web-services/deploy/application/pom.xml | 11 + .../application/src/main/docker/Dockerfile | 1 + .../src/main/docker/docker-entrypoint.sh | 3 + .../wildfly/add-datawave-configuration.cli | 2 + .../src/main/wildfly/runtime-config.cli | 30 ++ 11 files changed, 484 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build-images.yml create mode 100644 properties/kubernetes.properties create mode 100644 warehouse/assemble/datawave/src/main/docker/Dockerfile create mode 100644 web-services/deploy/application/src/main/wildfly/runtime-config.cli diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml new file mode 100644 index 00000000000..d36f222ad6b --- /dev/null +++ b/.github/workflows/build-images.yml @@ -0,0 +1,46 @@ +# +name: Create and publish a Docker image + +on: + push: + paths-ignore: ['*.md', 'CODEOWNERS', 'LICENSE'] + tags: + - "**" + branches: + - 'integration' + - 'release/version*' + - 'docker-images' + workflow_dispatch: + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository.lowercase }} + JAVA_VERSION: '11' + JAVA_DISTRIBUTION: 'zulu' #This is the default on v1 of the action for 1.8 + USER_NAME: ${{ secrets.GHCR_WRITE_USER_NAME }} + ACCESS_TOKEN: ${{ secrets.GHCR_WRITE_ACCESS_TOKEN }} + +jobs: + build-and-push-datawave-images: + runs-on: ubuntu-latest + steps: + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ env.USER_NAME }} + password: ${{ env.ACCESS_TOKEN }} + - name: Checkout Code + uses: actions/checkout@v3 + - name: Set up JDK ${{env.JAVA_VERSION}} + uses: actions/setup-java@v3 + with: + distribution: ${{env.JAVA_DISTRIBUTION}} + java-version: ${{env.JAVA_VERSION}} + cache: 'maven' + - name: Build Web and Ingest Docker Images (Maven) + run: | + mvn -s $GITHUB_WORKSPACE/.github/workflows/settings.xml clean install -Prpm,kubernetes,assemble,deploy-ws -Ddist -Pdocker -DpushImage -Ddocker-release -DskipTests + + diff --git a/pom.xml b/pom.xml index 243ba04b6ec..8410652244d 100644 --- a/pom.xml +++ b/pom.xml @@ -1534,6 +1534,7 @@ 2.7 UTF-8 + \ @@ -1912,6 +1913,32 @@ + + kubernetes + + false + + + kubernetes + kubernetes + + + + + + gov.nsa.datawave.plugins + read-properties + + + kubernetes.properties + kubernetes-passwords.properties + + + + + + + javadoc diff --git a/properties/default.properties b/properties/default.properties index 5be42c4cdcd..6a113e2561a 100644 --- a/properties/default.properties +++ b/properties/default.properties @@ -339,6 +339,8 @@ hornetq.port= webapp.transport.guarantee=CONFIDENTIAL # Tell the login module to expect client cert, and not DN stuffed in a header. trusted.header.login=false +trusted.header.issuer-header-name=X-SSL-ClientCert-Issuer +trusted.header.subject-header-name=X-SSL-ClientCert-Subject # web service response namespaces datawave.webservice.namespace=http://webservice.datawave.nsa/v1 # Name of the Cluster diff --git a/properties/kubernetes.properties b/properties/kubernetes.properties new file mode 100644 index 00000000000..ad0cc368626 --- /dev/null +++ b/properties/kubernetes.properties @@ -0,0 +1,259 @@ +CONFIGURATION=compose +RCPT_TO=hadoop@localhost + +docker.image.prefix=ghcr.io/nationalsecurityagency/ + +# ingest properties +DATAWAVE_INGEST_HOME=/opt/datawave-ingest/current + +trusted.header.login=true + +hdfs.site.config.urls=file:///etc/hadoop/conf/core-site.xml,file:///etc/hadoop/conf/hdfs-site.xml + +WAREHOUSE_ACCUMULO_HOME=/opt/accumulo +WAREHOUSE_HDFS_NAME_NODE=hdfs://hdfs-nn:9000 +WAREHOUSE_JOBTRACKER_NODE=yarn-rm:8032 +WAREHOUSE_ZOOKEEPERS=zookeeper:2181 +WAREHOUSE_INSTANCE_NAME=dev +#Sets variable sets the zookeeper location for the warehouse side +zookeeper.hosts=zookeeper:2181 + +INGEST_ACCUMULO_HOME=/opt/accumulo +INGEST_HDFS_NAME_NODE=hdfs://hdfs-nn:9000 +INGEST_JOBTRACKER_NODE=yarn-rm:8032 +INGEST_ZOOKEEPERS=zookeeper:2181 +INGEST_INSTANCE_NAME=dev + +JOB_CACHE_REPLICATION=1 + +STAGING_HOSTS=`nodeattr -c staging` +DUMPER_HOSTS=ingestmaster +INGEST_HOST=ingestmaster +ROLLUP_HOST=ingestmaster + +#extra mapreduce options (e.g. mapreduce.task.io.sort.mb and the like) +MAPRED_INGEST_OPTS=-useInlineCombiner + +#extra HADOOP_OPTS (java options) +HADOOP_INGEST_OPTS= + +#extra CHILD_OPTS (java options) +CHILD_INGEST_OPTS= + +BULK_CHILD_MAP_MAX_MEMORY_MB=2048 +LIVE_CHILD_MAP_MAX_MEMORY_MB=1024 +BULK_CHILD_REDUCE_MAX_MEMORY_MB=2048 +LIVE_CHILD_REDUCE_MAX_MEMORY_MB=1024 + +BULK_INGEST_DATA_TYPES=shardStats +LIVE_INGEST_DATA_TYPES=wikipedia,mycsv,myjson + +# Clear out these values if you do not want standard shard ingest. +DEFAULT_SHARD_HANDLER_CLASSES=datawave.ingest.mapreduce.handler.shard.AbstractColumnBasedHandler +ALL_HANDLER_CLASSES=datawave.ingest.mapreduce.handler.edge.ProtobufEdgeDataTypeHandler,datawave.ingest.mapreduce.handler.dateindex.DateIndexDataTypeHandler + +BULK_INGEST_REDUCERS=10 +LIVE_INGEST_REDUCERS=10 + +# Note the max blocks per job must be less than or equal to the number of mappers +INGEST_BULK_JOBS=1 +INGEST_BULK_MAPPERS=4 +INGEST_MAX_BULK_BLOCKS_PER_JOB=4 +INGEST_LIVE_JOBS=1 +INGEST_LIVE_MAPPERS=4 +INGEST_MAX_LIVE_BLOCKS_PER_JOB=4 + +INDEX_STATS_MAX_MAPPERS=1 + +NUM_MAP_LOADERS=1 + +USERNAME=root +PASSWORD=root + +ZOOKEEPER_HOME=/usr/lib/zookeeper +HADOOP_HOME=/usr/local/hadoop +MAPRED_HOME=/usr/local/hadoop-mapreduce + +WAREHOUSE_HADOOP_CONF=/usr/local/hadoop/etc/hadoop/ +INGEST_HADOOP_CONF=/usr/local/hadoop/etc/hadoop/ + +HDFS_BASE_DIR=/data + +MONITOR_SERVER_HOST=monitor + +LOG_DIR=/srv/logs/ingest +PDSH_LOG_DIR=${LOG_DIR}/pdsh_logs +FLAG_DIR=/srv/data/datawave/flags +FLAG_MAKER_CONFIG=/opt/datawave-ingest/current/config/flag-maker-live.xml,/opt/datawave-ingest/current/config/flag-maker-bulk.xml +BIN_DIR_FOR_FLAGS=/opt/datawave-ingest/current/bin + +PYTHON=/usr/bin/python + +# Setting discard interval to 0 in order to disable auto-ageoff @ ingest time +EVENT_DISCARD_INTERVAL=0 + +DATAWAVE_CACHE_PORT=20444 + +EDGE_DEFINITION_FILE=config/edge-definitions.xml + +ERROR_TABLE=errors +ANALYTIC_MTX=analytic_metrics +LOADER_MTX=loader_metrics +INGEST_MTX=ingest_metrics +BULK_INGEST_METRIC_THRESHOLD=1500000 +LIVE_INGEST_METRIC_THRESHOLD=1500000 + +KEYSTORE=/data/certs/keystore.p12 +KEYSTORE_TYPE=PKCS12 +KEYSTORE_PASSWORD=changeme +TRUSTSTORE=/data/certs/truststore.jks + +FLAG_METRICS_DIR=/srv/data/datawave/flagMetrics +TRUSTSTORE_PASSWORD=changeme +TRUSTSTORE_TYPE=JKS + +cluster.name=WAREHOUSE +accumulo.instance.name=dev +accumulo.user.name=root +accumulo.user.password=root +cached.results.hdfs.uri=hdfs://hdfs-nn:9000/ +cached.results.export.dir=/CachedResults + +lock.file.dir=/var/run/datawave +JAVA_HOME=/usr/lib/jvm/java/ + +# query properties +server.keystore.password=changeme +mysql.user.password=datawave +jboss.jmx.password=blah +hornetq.cluster.password=blah +hornetq.system.password=blah + +server.truststore.password=changeme + +#Sets up the Atom Service +atom.wildfly.hostname=localhost +atom.wildfly.port.number=8443 +atom.connection.pool.name=WAREHOUSE + +PASSWORD_INGEST_ENV=/opt/datawave-ingest/ingest-passwd.sh + +security.use.testauthservice=false +security.testauthservice.context.entry=classpath*:datawave/security/TestDatawaveUserServiceConfiguration.xml + +security.testauthservice.users= \ +\n \ +\n + +security.use.remoteauthservice=true +security.remoteuserservice.scheme=https +# The host to connect to (or do a SRV lookup on) for the remote user service +security.remoteuserservice.host=dwv-web-authorization +# The port to connect to (unless a SRV lookup was performed) for the remote user service +security.remoteuserservice.port=8443 + +webapp.transport.guarantee=NONE + +# other properties +rpm.file.owner=datawave +rpm.file.group=datawave +rpm.file.accumulo.owner=accumulo +rpm.file.accumulo.group=accumulo + +# Enable full table scans for the base event query? +#beq.fullTableScanEnabled=true + +event.query.data.decorators= \ + \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n + +lookup.uuid.uuidTypes= \ + \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n + +query.metrics.marking=(PUBLIC) +query.metrics.visibility=PUBLIC + +metrics.warehouse.namenode=localhost +metrics.warehouse.hadoop.path=/usr/local/hadoop +metrics.reporter.class=datawave.metrics.NoOpMetricsReporterFactory + +metadatahelper.default.auths=PUBLIC + +security.npe.ou.entries=EXAMPLE_SERVER_OU1,EXAMPLE_SERVER_OU2 +security.subject.dn.pattern=(?:^|,)\\s*OU\\s*=\\s*My Department\\s*(?:,|$) + +datawave.docs.menu.extras=
  • Accumulo
    • + diff --git a/warehouse/assemble/datawave/pom.xml b/warehouse/assemble/datawave/pom.xml index c364d4c622f..921d2d4c257 100644 --- a/warehouse/assemble/datawave/pom.xml +++ b/warehouse/assemble/datawave/pom.xml @@ -7,7 +7,7 @@ 6.6.0-SNAPSHOT assemble-datawave - pom + jar ${project.artifactId} @@ -267,6 +267,19 @@ + + maven-clean-plugin + 2.4.1 + + + auto-clean + + clean + + initialize + + + maven-dependency-plugin @@ -403,6 +416,38 @@ + + copy-rpm + + copy-resources + + verify + + ${project.build.directory}/docker + + + ${project.build.directory}/rpm/datawave-dw-${build.env}/RPMS/noarch/ + false + + + + + + copy-dockerfile + + copy-resources + + verify + + ${project.build.directory}/docker + + + ${project.basedir}/src/main/docker + true + + + + copy-additional-env @@ -423,6 +468,48 @@     + + docker + + + docker-release + + + + + + com.spotify + docker-maven-plugin + 1.1.1 + + ${docker.image.prefix}datawave/ingest-${build.env} + true + + latest + ${project.version} + + ${project.build.directory}/docker + + + / + ${project.build.directory} + rpm/RPMS/noarch/${project.build.finalName}-${build.env}.rpm + + + + + + tag-image + + build + + install + + + + + + + ${project.basedir}/src/main/docker + false + + runtime-config.clis + diff --git a/web-services/deploy/application/src/main/docker/Dockerfile b/web-services/deploy/application/src/main/docker/Dockerfile index 2078ce5a204..750db9d81c8 100644 --- a/web-services/deploy/application/src/main/docker/Dockerfile +++ b/web-services/deploy/application/src/main/docker/Dockerfile @@ -1,5 +1,6 @@ FROM azul/zulu-openjdk-centos:11.0.18-11.62.17 + RUN yum -y install deltarpm centos-release-sc && \ chmod -R 777 /usr/lib/jvm/zulu11/bin/* && \ groupadd -r jboss -g 1000 && \ diff --git a/web-services/deploy/application/src/main/docker/docker-entrypoint.sh b/web-services/deploy/application/src/main/docker/docker-entrypoint.sh index 005692bac18..4d0b309f293 100644 --- a/web-services/deploy/application/src/main/docker/docker-entrypoint.sh +++ b/web-services/deploy/application/src/main/docker/docker-entrypoint.sh @@ -63,6 +63,9 @@ else trap 'kill -WINCH $CMD_PID' WINCH trap 'kill -USR2 $CMD_PID' USR2 + printenv > env.properties + $WILDFLY_HOME/bin/jboss-cli.sh --file=./runtime-config.cli --properties=env.properties + eval "$@" "&" CMD_PID=${!} diff --git a/web-services/deploy/application/src/main/wildfly/add-datawave-configuration.cli b/web-services/deploy/application/src/main/wildfly/add-datawave-configuration.cli index 9a86c3cc405..069065e0cf5 100644 --- a/web-services/deploy/application/src/main/wildfly/add-datawave-configuration.cli +++ b/web-services/deploy/application/src/main/wildfly/add-datawave-configuration.cli @@ -56,6 +56,8 @@ module add --name=com.mysql.driver --dependencies=javax.api,javax.transaction.ap /system-property=dw.hornetq.system.password:add(value=${hornetq.system.password}) /system-property=dw.modification.cache.mdb.pool.size:add(value=${modification.cache.mdb.pool.size}) /system-property=dw.trusted.header.authentication:add(value=${trusted.header.login}) +/system-property=dw.trusted.header.issuerDn:add(value=${trusted.header.issuer-header-name}) +/system-property=dw.trusted.header.subjectDn:add(value=${trusted.header.subject-header-name}) /system-property=dw.transport.guarantee:add(value=${webapp.transport.guarantee}) /system-property=dw.connectionPool.default:add(value=WAREHOUSE) /system-property=dw.connectionPool.pools:add(value="WAREHOUSE,METRICS,UUID") diff --git a/web-services/deploy/application/src/main/wildfly/runtime-config.cli b/web-services/deploy/application/src/main/wildfly/runtime-config.cli new file mode 100644 index 00000000000..23cd2e4a8b1 --- /dev/null +++ b/web-services/deploy/application/src/main/wildfly/runtime-config.cli @@ -0,0 +1,30 @@ + +# Start processing batch commands (MUST BE FIRST) +batch +# Run an embedded server to perform the configuration +embed-server --server-config=standalone-full.xml + + +/system-property=dw.warehouse.accumulo.password:add(value=${env.ACCUMULO_PASSWORD}) +/system-property=dw.metrics.accumulo.password:add(value=${env.ACCUMULO_PASSWORD}) +/system-property=dw.uuid.accumulo.password:add(value=${env.ACCUMULO_PASSWORD}) +/system-property=dw.mapreduce.securitydomain.keyStoreURL:add(value=file://${env.KEYSTORE}) +/system-property=dw.mapreduce.securitydomain.keyStoreType:add(value="${env.KEYSTORE_TYPE}") +/system-property=dw.mapreduce.securitydomain.keyStorePassword:add(value="${env.KEYSTORE_PASSWORD}") +/system-property=dw.mapreduce.securitydomain.trustStoreURL:add(value=file://${env.TRUSTSTORE}) +/system-property=dw.mapreduce.securitydomain.trustStoreType:add(value="${env.TRUSTSTORE_TYPE}") +/system-property=dw.mapreduce.securitydomain.trustStorePassword:add(value="${env.TRUSTSTORE_PASSWORD}") + + +/system-property=dw.mapreduce.securitydomain.keyStoreURL:add(value=file://${env.KEYSTORE}) +/system-property=dw.mapreduce.securitydomain.keyStoreType:add(value="${env.KEYSTORE_TYPE}") +/system-property=dw.mapreduce.securitydomain.keyStorePassword:add(value="${env.KEYSTORE_PASSWORD}") +/system-property=dw.mapreduce.securitydomain.trustStoreURL:add(value=file://${env.TRUSTSTORE}) +/system-property=dw.mapreduce.securitydomain.trustStoreType:add(value="${env.TRUSTSTORE_TYPE}") +/system-property=dw.mapreduce.securitydomain.trustStorePassword:add(value="${env.TRUSTSTORE_PASSWORD}") +/core-service=management/security-realm=SSLRealm/server-identity=ssl/:add(enabled-protocols=["TLSv1.1","TLSv1.2"],keystore-path=${env.KEYSTORE},keystore-provider=${env.KEYSTORE_TYPE},keystore-password="${env.KEYSTORE_PASSWORD}") +/core-service=management/security-realm=SSLRealm/authentication=truststore/:add(keystore-path=${env.TRUSTSTORE},keystore-provider=${env.TRUSTSTORE_TYPE},keystore-password="${env.TRUSTSTORE_PASSWORD}") +/subsystem=security/security-domain=datawave/jsse=classic:add(keystore={type="${env.KEYSTORE_TYPE}",password="${env.KEYSTORE_PASSWORD}",url="file://${env.KEYSTORE}"},truststore={type="${env.TRUSTSTORE_TYPE}",password="${env.TRUSTSTORE_PASSWORD}",url="file://${env.TRUSTSTORE}"}) + +# Run the batch commands (MUST BE LAST) +run-batch \ No newline at end of file From 0fd8475c78d6d051f1dc4782f4c808a68b1065c0 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 8 Feb 2024 19:46:43 +0000 Subject: [PATCH 03/50] updated common-utils submodule and updated quickstart docker to not clean up old images --- contrib/datawave-quickstart/docker/pom.xml | 5 +++-- core/utils/common-utils | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/datawave-quickstart/docker/pom.xml b/contrib/datawave-quickstart/docker/pom.xml index 294b176deb5..747a69c60a0 100644 --- a/contrib/datawave-quickstart/docker/pom.xml +++ b/contrib/datawave-quickstart/docker/pom.xml @@ -12,13 +12,13 @@ ${project.artifactId} - accumulo-2.1.0-bin.tar.gz + accumulo-2.1.1-bin.tar.gz hadoop-3.3.4.tar.gz apache-maven-3.8.6-bin.tar.gz wildfly-17.0.1.Final.tar.gz apache-zookeeper-3.7.1-bin.tar.gz false - http://archive.apache.org/dist/accumulo/2.1.0/${dist.accumulo} + http://archive.apache.org/dist/accumulo/2.1.1/${dist.accumulo} http://archive.apache.org/dist/hadoop/common/hadoop-3.3.4/${dist.hadoop} https://archive.apache.org/dist/maven/maven-3/3.8.6/binaries/${dist.maven} https://download.jboss.org/wildfly/17.0.1.Final/${dist.wildfly} @@ -255,6 +255,7 @@ 8589934592 ${project.basedir}/../../../ + none ${project.basedir}/Dockerfile latest diff --git a/core/utils/common-utils b/core/utils/common-utils index 29b92b5c9fa..048fe1228b8 160000 --- a/core/utils/common-utils +++ b/core/utils/common-utils @@ -1 +1 @@ -Subproject commit 29b92b5c9fa82591d6446fbc0999b74f3f347388 +Subproject commit 048fe1228b82e3c43b336ee5458562e1f0d02775 From 93ddde2061cbe8ffae709748d1076fdc5c4eeddb Mon Sep 17 00:00:00 2001 From: Moriarty <22225248+apmoriarty@users.noreply.github.com> Date: Thu, 8 Feb 2024 17:37:33 +0000 Subject: [PATCH 04/50] Do not reduce fields when preloading iterator options --- .../java/datawave/query/planner/DefaultQueryPlanner.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java b/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java index 817508562da..e70b1f60cc9 100644 --- a/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java +++ b/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java @@ -2105,7 +2105,7 @@ protected Future loadQueryIterator(final MetadataHelper metadat addOption(cfg, QueryOptions.SORTED_UIDS, Boolean.toString(config.isSortedUIDs()), false); - configureTypeMappings(config, cfg, metadataHelper, getCompressOptionMappings()); + configureTypeMappings(config, cfg, metadataHelper, getCompressOptionMappings(), isPreload); configureAdditionalOptions(config, cfg); loadFields(cfg, config, isPreload); @@ -2273,6 +2273,11 @@ protected IteratorSetting getQueryIterator(MetadataHelper metadataHelper, ShardQ public static void configureTypeMappings(ShardQueryConfiguration config, IteratorSetting cfg, MetadataHelper metadataHelper, boolean compressMappings) throws DatawaveQueryException { + configureTypeMappings(config, cfg, metadataHelper, compressMappings, false); + } + + public static void configureTypeMappings(ShardQueryConfiguration config, IteratorSetting cfg, MetadataHelper metadataHelper, boolean compressMappings, + boolean isPreload) throws DatawaveQueryException { try { addOption(cfg, QueryOptions.QUERY_MAPPING_COMPRESS, Boolean.toString(compressMappings), false); @@ -2286,7 +2291,7 @@ public static void configureTypeMappings(ShardQueryConfiguration config, Iterato TypeMetadata typeMetadata = metadataHelper.getTypeMetadata(config.getDatatypeFilter()); - if (config.getReduceTypeMetadata()) { + if (config.getReduceTypeMetadata() && !isPreload) { Set fieldsToRetain = ReduceFields.getQueryFields(config.getQueryTree()); typeMetadata = typeMetadata.reduce(fieldsToRetain); } From 4c51cb65594211a9f604e27e5eb194037ad62074 Mon Sep 17 00:00:00 2001 From: jeff <1583214+jschmidt10@users.noreply.github.com> Date: Thu, 8 Feb 2024 16:58:58 -0500 Subject: [PATCH 05/50] Enforce term limits during EdgeQueries (#2274) * Enforce term limits during EdgeQueries * Update error message * Fix EdgeQuery maxQueryTerms setting --- .../query/config/EdgeQueryConfiguration.java | 1 + .../EdgeTableRangeBuildingVisitor.java | 26 ++++++------ .../EdgeTableRangeBuildingVisitorTest.java | 40 +++++++++++++++++++ 3 files changed, 53 insertions(+), 14 deletions(-) create mode 100644 warehouse/query-core/src/test/java/datawave/query/jexl/visitors/EdgeTableRangeBuildingVisitorTest.java diff --git a/warehouse/query-core/src/main/java/datawave/query/config/EdgeQueryConfiguration.java b/warehouse/query-core/src/main/java/datawave/query/config/EdgeQueryConfiguration.java index 76c07f7cb02..4516d3d4563 100644 --- a/warehouse/query-core/src/main/java/datawave/query/config/EdgeQueryConfiguration.java +++ b/warehouse/query-core/src/main/java/datawave/query/config/EdgeQueryConfiguration.java @@ -65,6 +65,7 @@ public EdgeQueryConfiguration(EdgeQueryLogic configuredLogic, Query query) { setModelName(configuredLogic.getModelName()); setModelTableName(configuredLogic.getModelTableName()); setEdgeQueryModel(configuredLogic.getEdgeQueryModel()); + setMaxQueryTerms(configuredLogic.getMaxQueryTerms()); } public List> getDataTypes() { diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/EdgeTableRangeBuildingVisitor.java b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/EdgeTableRangeBuildingVisitor.java index b0e1585295f..71efe38781c 100644 --- a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/EdgeTableRangeBuildingVisitor.java +++ b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/EdgeTableRangeBuildingVisitor.java @@ -168,11 +168,6 @@ public Object visit(ASTReferenceExpression node, Object data) { */ @Override public Object visit(ASTAndNode node, Object data) { - if (termCount > maxTerms) { - log.error("Query has too many terms"); - throw new IllegalArgumentException("Too many search terms " + termCount); - } - // run the visitor against all of the children List> childContexts = new ArrayList<>(node.jjtGetNumChildren()); for (int i = 0; i < node.jjtGetNumChildren(); i++) { @@ -274,11 +269,6 @@ public Object visit(ASTAndNode node, Object data) { */ @Override public Object visit(ASTOrNode node, Object data) { - if (termCount > maxTerms) { - log.error("Query has too many terms"); - throw new IllegalArgumentException("Too many search terms " + termCount); - } - // run the visitor against all of the children List> childContexts = new ArrayList<>(node.jjtGetNumChildren()); for (int i = 0; i < node.jjtGetNumChildren(); i++) { @@ -410,10 +400,18 @@ private void checkNotExclusion(IdentityContext context, String msg) { */ @Override public Object visit(ASTEQNode node, Object data) { - termCount++; + incrementTermCountAndCheck(); return visitExpresionNode(node, EQUALS); } + private void incrementTermCountAndCheck() { + if (++termCount > maxTerms) { + String message = "Exceeded max term limit of " + maxTerms; + log.error(message); + throw new IllegalArgumentException(message); + } + } + /** * Equals node (=~) should have exactly two children (reference nodes) one child will have the identifier eg: SOURCE ... the other child will have the * string literal eg: 'searchTerm' @@ -428,7 +426,7 @@ public Object visit(ASTEQNode node, Object data) { */ @Override public Object visit(ASTERNode node, Object data) { - termCount++; + incrementTermCountAndCheck(); List contexts = (List) visitExpresionNode(node, EQUALS_REGEX); if (contexts.get(0).getIdentity().equals(EDGE_SOURCE)) { sawEquivalenceRegexSource = true; @@ -440,13 +438,13 @@ public Object visit(ASTERNode node, Object data) { @Override public Object visit(ASTNRNode node, Object data) { - termCount++; + incrementTermCountAndCheck(); return visitExpresionNode(node, NOT_EQUALS_REGEX); } @Override public Object visit(ASTNENode node, Object data) { - termCount++; + incrementTermCountAndCheck(); return visitExpresionNode(node, NOT_EQUALS); } diff --git a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/EdgeTableRangeBuildingVisitorTest.java b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/EdgeTableRangeBuildingVisitorTest.java new file mode 100644 index 00000000000..476305e17f9 --- /dev/null +++ b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/EdgeTableRangeBuildingVisitorTest.java @@ -0,0 +1,40 @@ +package datawave.query.jexl.visitors; + +import static java.util.Collections.emptyList; + +import static org.junit.Assert.assertThrows; + +import org.apache.commons.jexl3.JexlFeatures; +import org.apache.commons.jexl3.parser.ASTJexlScript; +import org.apache.commons.jexl3.parser.Parser; +import org.apache.commons.jexl3.parser.StringProvider; +import org.junit.Before; +import org.junit.Test; + +import datawave.query.tables.edge.EdgeQueryLogic; + +public class EdgeTableRangeBuildingVisitorTest { + + private int termLimit = 3; + private Parser parser; + + private EdgeTableRangeBuildingVisitor visitor; + + @Before + public void setup() { + parser = new Parser(new StringProvider(";")); + + visitor = new EdgeTableRangeBuildingVisitor(false, emptyList(), termLimit, emptyList()); + } + + @Test + public void shouldEnforceTermLimit() { + ASTJexlScript parsedQuery = parseQuery("TYPE == 'like it' OR TYPE == 'love it' OR TYPE == 'gotta have it' OR TYPE == 'hand it over or else'"); + + assertThrows(IllegalArgumentException.class, () -> parsedQuery.jjtAccept(visitor, null)); + } + + private ASTJexlScript parseQuery(String query) { + return parser.parse(null, new JexlFeatures(), EdgeQueryLogic.fixQueryString(query), null); + } +} From 3e80d99cfea0127126a183eb31372555c533cd42 Mon Sep 17 00:00:00 2001 From: alerman Date: Fri, 9 Feb 2024 10:58:40 -0500 Subject: [PATCH 06/50] Change definedView to be a map definedViews (#2227) --- properties/default.properties | 2 +- properties/dev.properties | 20 ++- .../java/datawave/query/data/UUIDType.java | 43 +++++- .../data/config/ingest/CSVIngestHelper.java | 9 ++ .../error/ErrorShardedDataTypeHandler.java | 2 +- .../handler/shard/ShardedDataTypeHandler.java | 2 +- .../ingest/csv/mr/input/CSVRecordReader.java | 2 +- .../query/tables/ShardQueryLogic.java | 7 +- .../transformer/ContentQueryTransformer.java | 1 + .../query/result/event/SimpleEvent.java | 2 + .../testframework/AbstractDataTypeConfig.java | 2 +- .../testframework/cardata/CarsDataType.java | 17 ++- .../src/test/resources/input/ford-cars.csv | 6 +- .../src/test/resources/input/tesla-cars.csv | 6 +- .../datawave/query/QueryLogicFactory.xml | 12 +- .../IdTranslatorConfiguration.java | 2 +- .../LookupUUIDConfiguration.java | 11 ++ .../query/runner/QueryExecutorBean.java | 15 ++- .../util/AbstractUUIDLookupCriteria.java | 17 +++ .../webservice/query/util/LookupUUIDUtil.java | 122 +++++++++++++++--- .../runner/ExtendedQueryExecutorBeanTest.java | 5 +- .../query/util/LookupUUIDUtilTest.java | 1 + 22 files changed, 255 insertions(+), 51 deletions(-) diff --git a/properties/default.properties b/properties/default.properties index 6a113e2561a..771af191252 100644 --- a/properties/default.properties +++ b/properties/default.properties @@ -394,7 +394,7 @@ lookup.uuid.mappings= # Default uuidTypes lookup.uuid.uuidTypes= # Default lookup.uuid.beginDate -lookup.uuid.beginDate=20100101 +lookup.uuid.beginDate=19700101 ############################ # diff --git a/properties/dev.properties b/properties/dev.properties index 19cd93c67e0..b6c7d8bb355 100644 --- a/properties/dev.properties +++ b/properties/dev.properties @@ -211,28 +211,36 @@ event.query.data.decorators= \ lookup.uuid.uuidTypes= \ \ \n \ -\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ +\n \ \n \ \n \ \n \ -\n \ +\n \ \n \ \n \ \n \ \n \ -\n \ +\n \ \n \ \n \ \n \ -\n \ +\n \ \n \ \n \ \n \ -\n \ +\n \ \n \ \n \ \n \ -\n \ +\n \ \n query.metrics.marking=(PUBLIC) diff --git a/warehouse/core/src/main/java/datawave/query/data/UUIDType.java b/warehouse/core/src/main/java/datawave/query/data/UUIDType.java index 90ba395844b..e419da32d85 100644 --- a/warehouse/core/src/main/java/datawave/query/data/UUIDType.java +++ b/warehouse/core/src/main/java/datawave/query/data/UUIDType.java @@ -1,18 +1,34 @@ package datawave.query.data; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; + public class UUIDType { + public static final String DEFAULT_VIEW = "default"; + private String fieldName = null; - private String definedView = null; private Integer allowWildcardAfter = null; + private final Map definedViews = new HashMap<>(); + public UUIDType() {} public UUIDType(String field, String view, Integer allowWildcardAfter) { this.fieldName = field; - this.definedView = view; this.allowWildcardAfter = allowWildcardAfter; + + this.definedViews.put(DEFAULT_VIEW, view); + } + + public UUIDType(String field, Map views, Integer allowWildcardAfter) { + this.fieldName = field; + this.allowWildcardAfter = allowWildcardAfter; + + this.definedViews.putAll(views); } public Integer getAllowWildcardAfter() { @@ -31,11 +47,26 @@ public void setFieldName(String fieldName) { this.fieldName = fieldName; } - public String getDefinedView() { - return definedView; + public String getDefinedView(String context) { + if (StringUtils.isEmpty(context)) { + context = DEFAULT_VIEW; + } + return getDefinedViews().get(context); + } + + public Map getDefinedViews() { + return definedViews; + } + + public void setDefinedViews(Map views) { + this.definedViews.clear(); + this.definedViews.putAll(views); } - public void setDefinedView(String definedView) { - this.definedView = definedView; + public void setDefinedView(String context, String view) { + if (StringUtils.isEmpty(context)) { + context = DEFAULT_VIEW; + } + getDefinedViews().put(context, view); } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CSVIngestHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CSVIngestHelper.java index ecbe7b039a1..3bbf5bbbfae 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CSVIngestHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/CSVIngestHelper.java @@ -91,6 +91,15 @@ else if (helper.getSeparator().equals("\\t")) String[] dataFields = tokenizer.getTokenArray(); processFields(fields, dataFields); + // do the same for the aux data if it exists + if (event.getAuxData() instanceof String) { + String auxData = (String) event.getAuxData(); + tokenizer.reset(auxData); + for (String dataField : tokenizer.getTokenArray()) { + processExtraField(fields, dataField); + } + } + // and return the normalized fields return normalize(fields); } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java index 1b795a442fa..78df2c3230d 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java @@ -343,7 +343,7 @@ public static void getStackTrace(DataOutputBuffer buffer, Throwable e) { * @return the visibility */ @Override - protected byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) { + public byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) { byte[] visibility; if (value != null && value.getMarkings() != null && !value.getMarkings().isEmpty()) { try { diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java index 4f8e726acaf..ef1d6517216 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java @@ -783,7 +783,7 @@ protected Multimap createTermIndexColumn(RawRecordContainer * the entry value * @return the visibility */ - protected byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) { + public byte[] getVisibility(RawRecordContainer event, NormalizedContentInterface value) { ColumnVisibility visibility = event.getVisibility(); if (value.getMarkings() != null && !value.getMarkings().isEmpty()) { try { diff --git a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java index 35c8bbc1ae8..dfdb4e99417 100644 --- a/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java +++ b/warehouse/ingest-csv/src/main/java/datawave/ingest/csv/mr/input/CSVRecordReader.java @@ -37,7 +37,7 @@ public class CSVRecordReader extends CSVReaderBase implements EventFixer { private static final IngestConfiguration ingestConfig = IngestConfigurationFactory.getIngestConfiguration(); private static final MarkingFunctions markingFunctions = MarkingFunctionsFactory.createMarkingFunctions(); - private String csvEventId; + protected String csvEventId; private final Multimap metadataForValidation = ArrayListMultimap.create(100, 1); private String rawData = null; diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java index 81876eacadd..23f2165922c 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java @@ -599,7 +599,7 @@ public QueryLogicTransformer getTransformer(Query settings) { reducedInSettings = Boolean.parseBoolean(reducedResponseStr); } boolean reduced = (this.isReducedResponse() || reducedInSettings); - DocumentTransformer transformer = new DocumentTransformer(this, settings, markingFunctions, responseObjectFactory, reduced); + DocumentTransformer transformer = createDocumentTransformer(this, settings, markingFunctions, responseObjectFactory, reduced); transformer.setEventQueryDataDecoratorTransformer(eventQueryDataDecoratorTransformer); transformer.setContentFieldNames(getConfig().getContentFieldNames()); transformer.setLogTimingDetails(this.getLogTimingDetails()); @@ -612,6 +612,11 @@ public QueryLogicTransformer getTransformer(Query settings) { return this.transformerInstance; } + protected DocumentTransformer createDocumentTransformer(BaseQueryLogic> logic, Query settings, MarkingFunctions markingFunctions, + ResponseObjectFactory responseObjectFactory, Boolean reducedResponse) { + return new DocumentTransformer(logic, settings, markingFunctions, responseObjectFactory, reducedResponse); + } + public boolean isLongRunningQuery() { return getConfig().getGroupFields().hasGroupByFields(); } diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java index 61584cf26de..40a3f4aab3e 100644 --- a/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java +++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ContentQueryTransformer.java @@ -24,6 +24,7 @@ import datawave.webservice.result.BaseQueryResponse; import datawave.webservice.result.EventQueryResponseBase; +@SuppressWarnings("rawtypes") public class ContentQueryTransformer extends BaseQueryLogicTransformer,EventBase> { private static final Logger log = Logger.getLogger(ContentQueryTransformer.class); diff --git a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java index f7df34522ca..c52ee902b41 100644 --- a/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java +++ b/warehouse/query-core/src/main/java/datawave/webservice/query/result/event/SimpleEvent.java @@ -139,6 +139,8 @@ public String getFieldName(int number) { return "fields"; case 3: return "markings"; + case 4: + return "payload"; default: return null; } diff --git a/warehouse/query-core/src/test/java/datawave/query/testframework/AbstractDataTypeConfig.java b/warehouse/query-core/src/test/java/datawave/query/testframework/AbstractDataTypeConfig.java index 2a53f3a8b0f..1dc9eb1e51e 100644 --- a/warehouse/query-core/src/test/java/datawave/query/testframework/AbstractDataTypeConfig.java +++ b/warehouse/query-core/src/test/java/datawave/query/testframework/AbstractDataTypeConfig.java @@ -50,7 +50,7 @@ public abstract class AbstractDataTypeConfig implements DataTypeHadoopConfig { protected static final String DATE_FIELD_FORMAT = "yyyyMMdd"; public static final SimpleDateFormat YMD_DateFormat = new SimpleDateFormat(DATE_FIELD_FORMAT); private static final String TEST_VISIBILITY = "public"; - private static final String[] AUTH_VALUES = new String[] {"public", "Euro", "NA"}; + private static final String[] AUTH_VALUES = new String[] {"public", "private", "Euro", "NA"}; private static final Authorizations TEST_AUTHS = new Authorizations(AUTH_VALUES); /** diff --git a/warehouse/query-core/src/test/java/datawave/query/testframework/cardata/CarsDataType.java b/warehouse/query-core/src/test/java/datawave/query/testframework/cardata/CarsDataType.java index a01990dce3b..7db4df81d88 100644 --- a/warehouse/query-core/src/test/java/datawave/query/testframework/cardata/CarsDataType.java +++ b/warehouse/query-core/src/test/java/datawave/query/testframework/cardata/CarsDataType.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.net.URISyntaxException; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -15,9 +16,11 @@ import datawave.data.normalizer.Normalizer; import datawave.data.type.NumberType; +import datawave.ingest.csv.config.helper.ExtendedCSVHelper; import datawave.ingest.data.config.CSVHelper; import datawave.ingest.data.config.ingest.BaseIngestHelper; import datawave.ingest.input.reader.EventRecordReader; +import datawave.marking.MarkingFunctions; import datawave.query.testframework.AbstractDataTypeConfig; import datawave.query.testframework.FieldConfig; import datawave.query.testframework.RawDataManager; @@ -74,7 +77,8 @@ public enum CarField { COLOR(Normalizer.LC_NO_DIACRITICS_NORMALIZER, true), DOORS(Normalizer.NUMBER_NORMALIZER), WHEELS(Normalizer.NUMBER_NORMALIZER), - DESC(Normalizer.LC_NO_DIACRITICS_NORMALIZER, true); + DESC(Normalizer.LC_NO_DIACRITICS_NORMALIZER, true), + VISIBILITY(Normalizer.NOOP_NORMALIZER, false); private static final List Headers; @@ -122,6 +126,14 @@ public static List headers() { return Headers; } + public static List eventSecurityFieldNames() { + return Collections.singletonList(VISIBILITY.name()); + } + + public static List eventSecurityFieldDomains() { + return Collections.singletonList(MarkingFunctions.Default.COLUMN_VISIBILITY); + } + /** * Returns a random set of fields, with o without {@link #EVENT_ID}. * @@ -221,6 +233,9 @@ public CarsDataType(final String car, final String ingestFile, final FieldConfig // fields this.hConf.set(this.dataType + CSVHelper.DATA_HEADER, String.join(",", CarField.headers())); + this.hConf.set(this.dataType + ExtendedCSVHelper.Properties.EVENT_SECURITY_MARKING_FIELD_NAMES, String.join(",", CarField.eventSecurityFieldNames())); + this.hConf.set(this.dataType + ExtendedCSVHelper.Properties.EVENT_SECURITY_MARKING_FIELD_DOMAINS, + String.join(",", CarField.eventSecurityFieldDomains())); log.debug(this.toString()); } diff --git a/warehouse/query-core/src/test/resources/input/ford-cars.csv b/warehouse/query-core/src/test/resources/input/ford-cars.csv index 273a1e89873..11c1407e239 100644 --- a/warehouse/query-core/src/test/resources/input/ford-cars.csv +++ b/warehouse/query-core/src/test/resources/input/ford-cars.csv @@ -1,3 +1,3 @@ -20150501,ford-eventid-001,ford,explorer,blue,4,4,common description -20150601,ford-eventid-002,ford,mustang,silver,2,4,common description -20150701,ford-eventid-003,ford,fusion,magenta,4,4,common description \ No newline at end of file +20150501,ford-eventid-001,ford,explorer,blue,4,4,common description,public +20150601,ford-eventid-002,ford,mustang,silver,2,4,common description,public +20150701,ford-eventid-003,ford,fusion,magenta,4,4,common description,private \ No newline at end of file diff --git a/warehouse/query-core/src/test/resources/input/tesla-cars.csv b/warehouse/query-core/src/test/resources/input/tesla-cars.csv index c235192a622..cf26db18523 100644 --- a/warehouse/query-core/src/test/resources/input/tesla-cars.csv +++ b/warehouse/query-core/src/test/resources/input/tesla-cars.csv @@ -1,3 +1,3 @@ -20150501,tesla-eventid-001,tesla,models,red,4,4,common description -20150601,tesla-eventid-002,tesla,modelx,blue,4,4,common description -20150701,tesla-eventid-003,tesla,model3,yellow,4,4,common description \ No newline at end of file +20150501,tesla-eventid-001,tesla,models,red,4,4,common description,public +20150601,tesla-eventid-002,tesla,modelx,blue,4,4,common description,public +20150701,tesla-eventid-003,tesla,model3,yellow,4,4,common description,private \ No newline at end of file diff --git a/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml b/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml index c3d48a3b201..b6d04a60dd8 100644 --- a/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml +++ b/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml @@ -344,12 +344,17 @@ + + + + ${lookup.uuid.uuidTypes} + @@ -398,7 +403,12 @@ - + + + + + + diff --git a/web-services/query/src/main/java/datawave/webservice/query/configuration/IdTranslatorConfiguration.java b/web-services/query/src/main/java/datawave/webservice/query/configuration/IdTranslatorConfiguration.java index 2e3706ed92c..9c055795abe 100644 --- a/web-services/query/src/main/java/datawave/webservice/query/configuration/IdTranslatorConfiguration.java +++ b/web-services/query/src/main/java/datawave/webservice/query/configuration/IdTranslatorConfiguration.java @@ -42,7 +42,7 @@ public void setUuidTypes(List uuidTypes) { List goodTypes = new ArrayList<>(); if (uuidTypes != null) { for (UUIDType uuidType : uuidTypes) { - if (uuidType.getDefinedView().equalsIgnoreCase("LuceneUUIDEventQuery")) { + if ("LuceneUUIDEventQuery".equalsIgnoreCase(uuidType.getDefinedView("default"))) { goodTypes.add(uuidType); } } diff --git a/web-services/query/src/main/java/datawave/webservice/query/configuration/LookupUUIDConfiguration.java b/web-services/query/src/main/java/datawave/webservice/query/configuration/LookupUUIDConfiguration.java index 7c87e099994..504928006e7 100644 --- a/web-services/query/src/main/java/datawave/webservice/query/configuration/LookupUUIDConfiguration.java +++ b/web-services/query/src/main/java/datawave/webservice/query/configuration/LookupUUIDConfiguration.java @@ -1,6 +1,7 @@ package datawave.webservice.query.configuration; import java.util.List; +import java.util.Map; import javax.ws.rs.core.MultivaluedMap; @@ -16,6 +17,8 @@ public class LookupUUIDConfiguration { protected List uuidTypes = null; + protected Map contentLookupTypes = null; + protected int batchLookupUpperLimit = LookupUUIDConstants.DEFAULT_BATCH_LOOKUP_UPPER_LIMIT; protected String beginDate = null; protected String columnVisibility; @@ -37,6 +40,10 @@ public String getColumnVisibility() { return this.columnVisibility; } + public Map getContentLookupTypes() { + return this.contentLookupTypes; + } + public List getUuidTypes() { return this.uuidTypes; } @@ -63,6 +70,10 @@ public void setUuidTypes(List uuidTypes) { this.uuidTypes = uuidTypes; } + public void setContentLookupTypes(Map contentLookupTypes) { + this.contentLookupTypes = contentLookupTypes; + } + public MultivaluedMap optionalParamsToMap() { MultivaluedMap p = new MultivaluedMapImpl<>(); if (this.columnVisibility != null) { diff --git a/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java b/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java index ec87801848d..07d21f100cf 100644 --- a/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java +++ b/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java @@ -187,6 +187,7 @@ public class QueryExecutorBean implements QueryExecutor { */ public static final String EXPAND_VALUES = "expand.values"; public static final String EXPAND_FIELDS = "expand.fields"; + public static final String CONTEXT_PARAMETER = "context"; private final Logger log = Logger.getLogger(QueryExecutorBean.class); @@ -1454,15 +1455,20 @@ private T lookupContentByUUID(String uuidType, String uuid, MultivaluedMap T lookupUUID(String uuidType, String uuid, MultivaluedMap que if (!StringUtils.isEmpty(streaming)) { streamingOutput = Boolean.parseBoolean(streaming); } + String uuidTypeContext = queryParameters.getFirst(CONTEXT_PARAMETER); final UUIDType matchingType = this.lookupUUIDUtil.getUUIDType(uuidType); String queryId = null; T response; try { // Construct the criteria used to perform the query final GetUUIDCriteria criteria; - final String view = (null != matchingType) ? matchingType.getDefinedView() : null; + final String view = (null != matchingType) ? matchingType.getDefinedView(uuidTypeContext) : null; if ((LookupUUIDUtil.UID_QUERY.equals(view) || LookupUUIDUtil.LOOKUP_UID_QUERY.equals(view))) { criteria = new UIDQueryCriteria(uuid, uuidType, queryParameters); } else { @@ -1605,6 +1612,10 @@ T lookupUUID(String uuidType, String uuid, MultivaluedMap que criteria.setStreamingOutputHeaders(httpHeaders); } + if (!StringUtils.isEmpty(uuidTypeContext)) { + criteria.setUUIDTypeContext(uuidTypeContext); + } + // Perform the query and get the first set of results response = this.lookupUUIDUtil.createUUIDQueryAndNext(criteria); if (response instanceof BaseQueryResponse) { diff --git a/web-services/query/src/main/java/datawave/webservice/query/util/AbstractUUIDLookupCriteria.java b/web-services/query/src/main/java/datawave/webservice/query/util/AbstractUUIDLookupCriteria.java index 837da62827a..6026bcdab5d 100644 --- a/web-services/query/src/main/java/datawave/webservice/query/util/AbstractUUIDLookupCriteria.java +++ b/web-services/query/src/main/java/datawave/webservice/query/util/AbstractUUIDLookupCriteria.java @@ -16,6 +16,8 @@ public abstract class AbstractUUIDLookupCriteria { private HttpHeaders headersForStreamedResponse; private MultivaluedMap queryParameters; + private String uuidTypeContext; + /** * Constructor * @@ -73,4 +75,19 @@ public MultivaluedMap getQueryParameters() { } public abstract String getRawQueryString(); + + /** + * returns a context for the lookup request if any was specfied in the request. The lookup context is used to obtain alternate query logics for the lookup + * requests to use. This can be used to modify the types of responses the query operations provide (e.g., plaintext responses.) + * + * @return + */ + public String getUUIDTypeContext() { + return uuidTypeContext; + } + + public void setUUIDTypeContext(String uuidTypeContext) { + this.uuidTypeContext = uuidTypeContext; + } + } diff --git a/web-services/query/src/main/java/datawave/webservice/query/util/LookupUUIDUtil.java b/web-services/query/src/main/java/datawave/webservice/query/util/LookupUUIDUtil.java index e19a889177c..00a3149315e 100644 --- a/web-services/query/src/main/java/datawave/webservice/query/util/LookupUUIDUtil.java +++ b/web-services/query/src/main/java/datawave/webservice/query/util/LookupUUIDUtil.java @@ -20,6 +20,7 @@ import javax.ws.rs.core.StreamingOutput; import org.apache.commons.lang.time.DateUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.jboss.resteasy.specimpl.MultivaluedMapImpl; @@ -89,6 +90,7 @@ public class LookupUUIDUtil { public static final String LOOKUP_UID_QUERY = "LookupUIDQuery"; protected static final String UUID_TERM_DELIMITER = ":"; + public static final String DEFAULT_CONTENT_LOOKUP_TYPE = "default"; private Date beginAsDate = null; private EJBContext ctx; @@ -109,6 +111,7 @@ public class LookupUUIDUtil { private final UserOperations userOperations; private Map uuidTypes = Collections.synchronizedMap(new HashMap<>()); + private Map contentLookupTypes = Collections.synchronizedMap(new HashMap<>()); MultivaluedMap defaultOptionalParams; @@ -153,17 +156,21 @@ public LookupUUIDUtil(final LookupUUIDConfiguration configuration, final QueryEx // set the query logic factory this.queryLogicFactory = queryLogicFactory; - // Populate the UUIDType map - final List types = this.lookupUUIDConfiguration.getUuidTypes(); this.uuidTypes.clear(); + + // load uuidTypes from the flat list + final List types = this.lookupUUIDConfiguration.getUuidTypes(); + if (null != types) { for (final UUIDType type : types) { - if (null != type) { - this.uuidTypes.put(type.getFieldName().toUpperCase(), type); - } + addUUIDType(type.getFieldName(), type); } } + // Populate the content lookup types map + this.contentLookupTypes.clear(); + this.contentLookupTypes = this.lookupUUIDConfiguration.getContentLookupTypes(); + // Assign the begin date try { this.beginAsDate = DateHelper.parseWithGMT(this.lookupUUIDConfiguration.getBeginDate()); @@ -486,21 +493,92 @@ public String getAuths(String logicName, MultivaluedMap queryPara } /** - * Returns a UUIDType implementation, if any, matching the specified field name + * Add the specified uuid type to the internal uuidtypes map if the type is not null. + * + * @param uuidType + * the uuid type name / field + * @param type + * the uuid type. + */ + private void addUUIDType(final String uuidType, final UUIDType type) { + if (type != null) { + final String key = buildUUIDTypeKey(uuidType); + this.uuidTypes.put(key.toUpperCase(), type); + } + } + + /** + * Build the string key used to store uuid types in the internal uuidtype map. There can be multiple lists of uuid types, referred to as contexts - this + * allows multiple logicNames to be associated with the same field. + * + * @param uuidType + * the name/field of the specified UUIDType. + * @return the key used to store/retrieve the UUID type from the uuidType map. + */ + private String buildUUIDTypeKey(String uuidType) { + if (uuidType == null) + return null; + return uuidType.toUpperCase(); + } + + /** + * Returns a UUIDType implementation, if any, matching the specified field name. * * @param uuidType * the field name of the desired UUIDType - * @return a UUIDType implementation, if any, matching the specified field name + * @return a UUIDType implementation, if any, matching the specified field name, null if one does not exist. */ public UUIDType getUUIDType(final String uuidType) { - final UUIDType type; - if (null != uuidType) { - type = this.uuidTypes.get(uuidType.toUpperCase()); + String uuidTypeKey = buildUUIDTypeKey(uuidType); + return (uuidTypeKey == null) ? null : uuidTypes.get(uuidTypeKey); + } + + /** + * check if the query logic name is one of the configured content lookup type logics. + * + * @param queryLogicName + * @return + */ + public boolean isContentLookup(String queryLogicName) { + if (queryLogicName == null) + return false; + + for (Map.Entry typeEntry : contentLookupTypes.entrySet()) { + if (queryLogicName.equals(typeEntry.getValue())) { + return true; + } + } + return false; + } + + /** + * find the appropriate content lookup type logic from the specified lookup context. + * + * @param lookupContext + * the lookupContext for the current request. + * @return the name of the content lookup type logic configured for the specified context, or the default content lookup type logic name, if none is + * configured for the context specified. + */ + public String getLookupType(String lookupContext) { + String lookupType = contentLookupTypes.get(lookupContext); + if (StringUtils.isEmpty(lookupType)) { + log.info("Request contained lookupContext '" + lookupContext + "', which was not configured, retrieving default lookup type."); } else { - type = null; + + if (log.isDebugEnabled()) { + log.debug("Using lookupContext " + lookupContext); + } + return lookupType; + } + + lookupType = contentLookupTypes.get(DEFAULT_CONTENT_LOOKUP_TYPE); + if (StringUtils.isEmpty(lookupType)) { + log.warn("Request contained lookupContext '" + lookupContext + + "', which was not configured, yet no default is configured either, returning hard-coded default: " + CONTENT_QUERY); + lookupType = CONTENT_QUERY; } - return type; + return lookupType; } /** @@ -538,7 +616,7 @@ public T lookupContentByNextResponse(final AbstractUUIDLookupCriteria valida // be evaluated since the getLogicName() expression would always evaluate as false for an instance // of AllEventMockResponse. // - if (CONTENT_QUERY.equals(nextQueryResponse.getLogicName()) && !(nextQueryResponse instanceof AllEventMockResponse)) { + if (isContentLookup(nextQueryResponse.getLogicName()) && !(nextQueryResponse instanceof AllEventMockResponse)) { contentQueryResponse = (T) nextQueryResponse; } // Handle the case where /next has returned results of a UUID lookup, and a secondary content @@ -695,7 +773,8 @@ private EventQueryResponseBase lookupPagedContent(final String queryName, final } } - final GenericResponse createResponse = this.queryExecutor.createQuery(CONTENT_QUERY, queryParameters); + final String contentLookupType = getLookupType(validatedCriteria.getUUIDTypeContext()); + final GenericResponse createResponse = this.queryExecutor.createQuery(contentLookupType, queryParameters); final String contentQueryId = createResponse.getResult(); boolean preventCloseOfMergedQueryId = ((null == mergedContentQueryResponse) && allEventMockResponse); try { @@ -811,7 +890,8 @@ private StreamingOutput lookupStreamedContent(final String queryName, final Abst // Call the ContentQuery for one or more events final HttpHeaders headers = validatedCriteria.getStreamingOutputHeaders(); - return this.queryExecutor.execute(CONTENT_QUERY, queryParameters, headers); + final String contentLookupType = getLookupType(validatedCriteria.getUUIDTypeContext()); + return this.queryExecutor.execute(contentLookupType, queryParameters, headers); } private void mergeNextUUIDLookups(final EventQueryResponseBase mergedResponse) { @@ -899,11 +979,11 @@ private AbstractUUIDLookupCriteria validateLookupCriteria(final AbstractUUIDLook // Validate the "potential" UUID term. It's potential because it could be an OR operator // or some other query syntax that would be validated with more scrutiny once the query // executor is invoked. - final UUIDType uuidType = this.validateUUIDTerm(potentialUUIDTerm.trim(), logicName); + final UUIDType uuidType = this.validateUUIDTerm(criteria.getUUIDTypeContext(), potentialUUIDTerm.trim(), logicName); if (null != uuidType) { // Assign the query logic name if undefined if (null == logicName) { - logicName = uuidType.getDefinedView(); + logicName = uuidType.getDefinedView(criteria.getUUIDTypeContext()); } // Increment the UUID type/value count @@ -987,13 +1067,15 @@ private EventQueryResponseBase validatePagedResponse(final BaseQueryResponse res * Validate the specified token as a UUID lookup term, either as a LUCENE-formatted field/value or a UIDQuery field/value. Tokens missing the appropriate * delimiter are ignored and return with a null UUIDType. * + * @param lookupContext additional information about the lookup purpose or type. + * * @param uuidTypeValueTerm A token to evaluate as a possible UUID field/value term * * @param logicName The existing assigned query logic name, if any * * @return A valid UUIDType, or null if the specified token is obviously not a UUID field/value term */ - private UUIDType validateUUIDTerm(final String possibleUUIDTerm, final String logicName) { + private UUIDType validateUUIDTerm(final String lookupContext, final String possibleUUIDTerm, final String logicName) { // Declare the return value final UUIDType matchingUuidType; @@ -1026,8 +1108,8 @@ else if ((null == uuid) || uuid.isEmpty()) { throw new DatawaveWebApplicationException(new IllegalArgumentException(message), errorReponse); } // Reject conflicting logic name - else if ((null != logicName) && !logicName.equals(matchingUuidType.getDefinedView())) { - final String message = "Multiple UUID types '" + logicName + "' and '" + matchingUuidType.getDefinedView() + "' not " + else if ((null != logicName) && !logicName.equals(matchingUuidType.getDefinedView(lookupContext))) { + final String message = "Multiple UUID types '" + logicName + "' and '" + matchingUuidType.getDefinedView(lookupContext) + "' not " + " supported within the same lookup request"; final GenericResponse errorReponse = new GenericResponse<>(); errorReponse.addMessage(message); diff --git a/web-services/query/src/test/java/datawave/webservice/query/runner/ExtendedQueryExecutorBeanTest.java b/web-services/query/src/test/java/datawave/webservice/query/runner/ExtendedQueryExecutorBeanTest.java index 8641025e46c..234e97ddbb0 100644 --- a/web-services/query/src/test/java/datawave/webservice/query/runner/ExtendedQueryExecutorBeanTest.java +++ b/web-services/query/src/test/java/datawave/webservice/query/runner/ExtendedQueryExecutorBeanTest.java @@ -2745,6 +2745,7 @@ public void testInit() throws Exception { expect(this.lookupUUIDConfiguration.getUuidTypes()).andReturn(null); expect(this.lookupUUIDConfiguration.getBeginDate()).andReturn("not a date"); expect(this.lookupUUIDConfiguration.getBatchLookupUpperLimit()).andReturn(0); + expect(this.lookupUUIDConfiguration.getContentLookupTypes()).andReturn(Collections.emptyMap()); expect(this.context.getCallerPrincipal()).andReturn(this.principal).anyTimes(); LookupUUIDConfiguration tmpCfg = new LookupUUIDConfiguration(); tmpCfg.setColumnVisibility("PUBLIC"); @@ -3762,7 +3763,7 @@ public void testLookupUUID_happyPath() { expect(uriInfo.getQueryParameters()).andReturn(new MultivaluedHashMap<>()); expect(lookupUUIDUtil.getUUIDType("uuidType")).andReturn(uuidType); - expect(uuidType.getDefinedView()).andReturn("abc"); + expect(uuidType.getDefinedView(null)).andReturn("abc"); expect(lookupUUIDUtil.createUUIDQueryAndNext(isA(GetUUIDCriteria.class))).andReturn(response); expect(response.getQueryId()).andReturn("11111"); expect(context.getCallerPrincipal()).andReturn(principal); @@ -3796,7 +3797,7 @@ public void testLookupUUID_closeFail() { expect(uriInfo.getQueryParameters()).andReturn(new MultivaluedHashMap<>()); expect(lookupUUIDUtil.getUUIDType("uuidType")).andReturn(uuidType); - expect(uuidType.getDefinedView()).andReturn("abc"); + expect(uuidType.getDefinedView(null)).andReturn("abc"); expect(lookupUUIDUtil.createUUIDQueryAndNext(isA(GetUUIDCriteria.class))).andReturn(response); expect(response.getQueryId()).andReturn("11111"); expect(context.getCallerPrincipal()).andReturn(principal); diff --git a/web-services/query/src/test/java/datawave/webservice/query/util/LookupUUIDUtilTest.java b/web-services/query/src/test/java/datawave/webservice/query/util/LookupUUIDUtilTest.java index a1509238b8a..b61eec6caf8 100644 --- a/web-services/query/src/test/java/datawave/webservice/query/util/LookupUUIDUtilTest.java +++ b/web-services/query/src/test/java/datawave/webservice/query/util/LookupUUIDUtilTest.java @@ -45,6 +45,7 @@ public class LookupUUIDUtilTest { @Test public void testCreateSettings() { + expect(configuration.getContentLookupTypes()).andReturn(Collections.emptyMap()); expect(configuration.getUuidTypes()).andReturn(Collections.singletonList(new UUIDType("ID", "LuceneUUIDEventQuery", 28))); expect(configuration.getBeginDate()).andReturn("20230101"); expect(configuration.getBatchLookupUpperLimit()).andReturn(10); From 262cddee243097351541fcc4942a2af3215409d6 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Mon, 12 Feb 2024 10:12:09 -0500 Subject: [PATCH 07/50] Added remote edge query logic and tests (#2250) * Added remote edge query logic and tests * factored the BaseRemoteQueryLogic out of the remote edge and event instances --- .../config/RemoteQueryConfiguration.java | 27 -- .../query/tables/BaseRemoteQueryLogic.java | 188 +++++++++++++ .../query/tables/RemoteEdgeQueryLogic.java | 131 +++++++++ .../query/tables/RemoteEventQueryLogic.java | 151 +---------- .../tables/RemoteEdgeQueryLogicHttpTest.java | 252 ++++++++++++++++++ .../tables/RemoteEdgeQueryLogicTest.java | 172 ++++++++++++ .../common/json/DefaultMapperDecorator.java | 5 + .../common/remote/RemoteHttpService.java | 2 +- .../query/remote/RemoteQueryServiceImpl.java | 19 +- 9 files changed, 771 insertions(+), 176 deletions(-) create mode 100644 warehouse/query-core/src/main/java/datawave/query/tables/BaseRemoteQueryLogic.java create mode 100644 warehouse/query-core/src/main/java/datawave/query/tables/RemoteEdgeQueryLogic.java create mode 100644 warehouse/query-core/src/test/java/datawave/query/tables/RemoteEdgeQueryLogicHttpTest.java create mode 100644 warehouse/query-core/src/test/java/datawave/query/tables/RemoteEdgeQueryLogicTest.java diff --git a/warehouse/query-core/src/main/java/datawave/query/config/RemoteQueryConfiguration.java b/warehouse/query-core/src/main/java/datawave/query/config/RemoteQueryConfiguration.java index ea857921601..ead2d40f4c7 100644 --- a/warehouse/query-core/src/main/java/datawave/query/config/RemoteQueryConfiguration.java +++ b/warehouse/query-core/src/main/java/datawave/query/config/RemoteQueryConfiguration.java @@ -1,6 +1,5 @@ package datawave.query.config; -import java.io.ObjectStreamException; import java.io.Serializable; import java.util.Objects; @@ -78,32 +77,6 @@ public static RemoteQueryConfiguration create(RemoteQueryConfiguration other) { return new RemoteQueryConfiguration(other); } - /** - * Factory method that creates a RemoteQueryConfiguration deep copy from a RemoteQueryLogic - * - * @param remoteQueryLogic - * - a configured RemoteQueryLogic - * @return - a RemoteQueryConfiguration - */ - public static RemoteQueryConfiguration create(RemoteEventQueryLogic remoteQueryLogic) { - return create(remoteQueryLogic.getConfig()); - } - - /** - * Factory method that creates a RemoteQueryConfiguration from a RemoteQueryLogic and a Query - * - * @param remoteQueryLogic - * - a configured RemoteQueryLogic - * @param query - * - a configured Query object - * @return - a RemoteQueryConfiguration - */ - public static RemoteQueryConfiguration create(RemoteEventQueryLogic remoteQueryLogic, Query query) { - RemoteQueryConfiguration config = create(remoteQueryLogic); - config.setQuery(query); - return config; - } - public String getRemoteId() { return remoteId; } diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/BaseRemoteQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/BaseRemoteQueryLogic.java new file mode 100644 index 00000000000..37b0ab0002c --- /dev/null +++ b/warehouse/query-core/src/main/java/datawave/query/tables/BaseRemoteQueryLogic.java @@ -0,0 +1,188 @@ +package datawave.query.tables; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.log4j.Logger; + +import datawave.marking.MarkingFunctions; +import datawave.query.config.RemoteQueryConfiguration; +import datawave.query.tables.remote.RemoteQueryLogic; +import datawave.security.authorization.UserOperations; +import datawave.webservice.common.connection.AccumuloConnectionFactory; +import datawave.webservice.common.logging.ThreadConfigurableLogger; +import datawave.webservice.common.remote.RemoteQueryService; +import datawave.webservice.query.Query; +import datawave.webservice.query.configuration.GenericQueryConfiguration; +import datawave.webservice.query.exception.QueryException; +import datawave.webservice.query.logic.BaseQueryLogic; +import datawave.webservice.query.logic.QueryLogicTransformer; +import datawave.webservice.query.result.event.ResponseObjectFactory; +import datawave.webservice.result.GenericResponse; + +/** + *

    Overview

    This is a base query logic implementation that can handle delegating to a remote query logic + */ +public abstract class BaseRemoteQueryLogic extends BaseQueryLogic implements RemoteQueryLogic { + + protected static final Logger log = ThreadConfigurableLogger.getLogger(BaseRemoteQueryLogic.class); + + public static final String QUERY_ID = "queryId"; + + protected RemoteQueryConfiguration config; + + protected RemoteQueryService remoteQueryService; + + protected UserOperations userOperations; + + protected QueryLogicTransformer transformerInstance = null; + + /** + * Basic constructor + */ + public BaseRemoteQueryLogic() { + super(); + if (log.isTraceEnabled()) + log.trace("Creating " + this.getClass().getSimpleName() + ": " + System.identityHashCode(this)); + } + + /** + * Copy constructor + * + * @param other + * - another ShardQueryLogic object + */ + public BaseRemoteQueryLogic(BaseRemoteQueryLogic other) { + super(other); + + if (log.isTraceEnabled()) + log.trace("Creating Cloned " + this.getClass().getSimpleName() + ": " + System.identityHashCode(this) + " from " + System.identityHashCode(other)); + + setRemoteQueryService(other.getRemoteQueryService()); + setUserOperations(other.getUserOperations()); + + // Set ShardQueryConfiguration variables + setConfig(RemoteQueryConfiguration.create(other.getConfig())); + + // transformer instance is created dynamically + } + + public void setupConfig(GenericQueryConfiguration genericConfig) throws Exception { + if (!RemoteQueryConfiguration.class.isAssignableFrom(genericConfig.getClass())) { + throw new QueryException("Did not receive a RemoteQueryConfiguration instance!!"); + } + + config = (RemoteQueryConfiguration) genericConfig; + } + + public String getRemoteId() { + return getConfig().getRemoteId(); + } + + public void setRemoteId(String id) { + getConfig().setRemoteId(id); + getConfig().setQueryString("( metrics = '" + remoteQueryService.getQueryMetricsURI(id).toString() + "' )"); + } + + public String getRemoteQueryLogic() { + return getConfig().getRemoteQueryLogic(); + } + + public void setRemoteQueryLogic(String remoteQueryLogic) { + getConfig().setRemoteQueryLogic(remoteQueryLogic); + } + + public Object getCallerObject() { + return getPrincipal(); + } + + @Override + public GenericQueryConfiguration initialize(AccumuloClient connection, Query settings, Set auths) throws Exception { + Map> parms = settings.toMap(); + // we need to ensure that the remote query request includes the local query id for tracking purposes via query metrics. + if (!parms.containsKey(QUERY_ID) && settings.getId() != null) { + parms.put(QUERY_ID, Collections.singletonList(settings.getId().toString())); + } + GenericResponse createResponse = remoteQueryService.createQuery(getRemoteQueryLogic(), parms, getCallerObject()); + setRemoteId(createResponse.getResult()); + log.info("Local query " + settings.getId() + " maps to remote query " + getRemoteId()); + RemoteQueryConfiguration config = getConfig(); + config.setQuery(settings); + return config; + } + + @Override + public String getPlan(AccumuloClient connection, Query settings, Set auths, boolean expandFields, boolean expandValues) throws Exception { + GenericResponse planResponse = remoteQueryService.planQuery(getRemoteQueryLogic(), settings.toMap(), getCallerObject()); + return planResponse.getResult(); + } + + @Override + public QueryLogicTransformer getTransformer(Query settings) { + // a transformer that turns EventBase objects into a response + if (transformerInstance == null) { + transformerInstance = createTransformer(settings, getMarkingFunctions(), getResponseObjectFactory()); + } + + return transformerInstance; + } + + public abstract QueryLogicTransformer createTransformer(Query settings, MarkingFunctions markingFunctions, + ResponseObjectFactory responseObjectFactory); + + @Override + public void close() { + + super.close(); + + log.debug("Closing " + this.getClass().getSimpleName() + ": " + System.identityHashCode(this)); + + if (getRemoteId() != null) { + try { + remoteQueryService.close(getRemoteId(), getCallerObject()); + } catch (Exception e) { + log.error("Failed to close remote query", e); + } + } + } + + @Override + public RemoteQueryConfiguration getConfig() { + if (config == null) { + config = RemoteQueryConfiguration.create(); + } + + return config; + } + + public void setConfig(RemoteQueryConfiguration config) { + this.config = config; + } + + public RemoteQueryService getRemoteQueryService() { + return remoteQueryService; + } + + public void setRemoteQueryService(RemoteQueryService remoteQueryService) { + this.remoteQueryService = remoteQueryService; + } + + @Override + public AccumuloConnectionFactory.Priority getConnectionPriority() { + return AccumuloConnectionFactory.Priority.NORMAL; + } + + @Override + public void setUserOperations(UserOperations userOperations) { + this.userOperations = userOperations; + } + + @Override + public UserOperations getUserOperations() { + return userOperations; + } +} diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEdgeQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEdgeQueryLogic.java new file mode 100644 index 00000000000..9152ef54348 --- /dev/null +++ b/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEdgeQueryLogic.java @@ -0,0 +1,131 @@ +package datawave.query.tables; + +import java.util.Iterator; +import java.util.LinkedList; +import java.util.Queue; +import java.util.Set; + +import org.apache.log4j.Logger; + +import datawave.marking.MarkingFunctions; +import datawave.query.tables.edge.EdgeQueryLogic; +import datawave.query.tables.remote.RemoteQueryLogic; +import datawave.query.transformer.EdgeQueryTransformerSupport; +import datawave.webservice.common.logging.ThreadConfigurableLogger; +import datawave.webservice.query.Query; +import datawave.webservice.query.configuration.GenericQueryConfiguration; +import datawave.webservice.query.exception.EmptyObjectException; +import datawave.webservice.query.logic.QueryLogicTransformer; +import datawave.webservice.query.result.EdgeQueryResponseBase; +import datawave.webservice.query.result.edge.EdgeBase; +import datawave.webservice.query.result.event.ResponseObjectFactory; + +/** + *

    Overview

    This is a query logic implementation that can handle delegating to a remote edge query logic (i.e. one that returns an extension of + * EdgeQueryResponseBase). + */ +public class RemoteEdgeQueryLogic extends BaseRemoteQueryLogic implements RemoteQueryLogic { + + protected static final Logger log = ThreadConfigurableLogger.getLogger(RemoteEdgeQueryLogic.class); + + /** + * Basic constructor + */ + public RemoteEdgeQueryLogic() { + super(); + } + + /** + * Copy constructor + * + * @param other + * - another RemoteEdgeQueryLogic object + */ + public RemoteEdgeQueryLogic(RemoteEdgeQueryLogic other) { + super(other); + } + + @Override + public void setupQuery(GenericQueryConfiguration genericConfig) throws Exception { + setupConfig(genericConfig); + + // Create an iterator that returns a stream of EdgeBase objects + iterator = new RemoteQueryLogicIterator(); + } + + @Override + public QueryLogicTransformer createTransformer(Query settings, MarkingFunctions markingFunctions, + ResponseObjectFactory responseObjectFactory) { + return new EdgeBaseTransformer(settings, markingFunctions, responseObjectFactory); + } + + @Override + public RemoteEdgeQueryLogic clone() { + return new RemoteEdgeQueryLogic(this); + } + + @Override + public Set getOptionalQueryParameters() { + return new EdgeQueryLogic().getOptionalQueryParameters(); + } + + @Override + public Set getRequiredQueryParameters() { + return new EdgeQueryLogic().getRequiredQueryParameters(); + } + + @Override + public Set getExampleQueries() { + return new EdgeQueryLogic().getExampleQueries(); + } + + private class RemoteQueryLogicIterator implements Iterator { + private Queue data = new LinkedList<>(); + private boolean complete = false; + + @Override + public boolean hasNext() { + if (data.isEmpty() && !complete) { + try { + EdgeQueryResponseBase response = (EdgeQueryResponseBase) remoteQueryService.next(getRemoteId(), getCallerObject()); + if (response != null) { + if (response.getTotalResults() == 0) { + if (!response.isPartialResults()) { + complete = true; + } + } else { + for (EdgeBase edge : response.getEdges()) { + data.add(edge); + } + } + } else { + // in this case we must have gotten a 204, so we are done + complete = true; + } + } catch (Exception e) { + complete = true; + throw new RuntimeException(e.getMessage(), e); + } + } + return !data.isEmpty(); + } + + @Override + public EdgeBase next() { + return data.poll(); + } + } + + private class EdgeBaseTransformer extends EdgeQueryTransformerSupport { + + public EdgeBaseTransformer(Query settings, MarkingFunctions markingFunctions, ResponseObjectFactory responseObjectFactory) { + super(settings, markingFunctions, responseObjectFactory); + } + + @Override + public EdgeBase transform(EdgeBase input) throws EmptyObjectException { + return input; + } + + } +} diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEventQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEventQueryLogic.java index f98f08008bd..5612244c59f 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEventQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/RemoteEventQueryLogic.java @@ -1,144 +1,65 @@ package datawave.query.tables; -import java.util.Collections; import java.util.Iterator; import java.util.LinkedList; -import java.util.List; import java.util.Map; import java.util.Queue; import java.util.Set; -import org.apache.accumulo.core.client.AccumuloClient; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.security.Authorizations; import org.apache.log4j.Logger; import datawave.marking.MarkingFunctions; -import datawave.query.config.RemoteQueryConfiguration; import datawave.query.tables.remote.RemoteQueryLogic; import datawave.query.transformer.EventQueryTransformerSupport; -import datawave.security.authorization.UserOperations; -import datawave.webservice.common.connection.AccumuloConnectionFactory; import datawave.webservice.common.logging.ThreadConfigurableLogger; -import datawave.webservice.common.remote.RemoteQueryService; import datawave.webservice.query.Query; import datawave.webservice.query.configuration.GenericQueryConfiguration; import datawave.webservice.query.exception.EmptyObjectException; -import datawave.webservice.query.exception.QueryException; import datawave.webservice.query.logic.BaseQueryLogic; import datawave.webservice.query.logic.QueryLogicTransformer; import datawave.webservice.query.result.event.EventBase; import datawave.webservice.query.result.event.ResponseObjectFactory; import datawave.webservice.result.EventQueryResponseBase; -import datawave.webservice.result.GenericResponse; /** *

    Overview

    This is a query logic implementation that can handle delegating to a remote event query logic (i.e. one that returns an extension of * EventQueryResponseBase). */ -public class RemoteEventQueryLogic extends BaseQueryLogic implements RemoteQueryLogic { +public class RemoteEventQueryLogic extends BaseRemoteQueryLogic implements RemoteQueryLogic { protected static final Logger log = ThreadConfigurableLogger.getLogger(RemoteEventQueryLogic.class); - public static final String QUERY_ID = "queryId"; - - private RemoteQueryConfiguration config; - - private RemoteQueryService remoteQueryService; - - private UserOperations userOperations; - - private QueryLogicTransformer transformerInstance = null; - /** * Basic constructor */ public RemoteEventQueryLogic() { super(); - if (log.isTraceEnabled()) - log.trace("Creating RemoteQueryLogic: " + System.identityHashCode(this)); } /** * Copy constructor * * @param other - * - another ShardQueryLogic object + * - another RemoteEventQueryLogic object */ public RemoteEventQueryLogic(RemoteEventQueryLogic other) { super(other); - - if (log.isTraceEnabled()) - log.trace("Creating Cloned RemoteQueryLogic: " + System.identityHashCode(this) + " from " + System.identityHashCode(other)); - - setRemoteQueryService(other.getRemoteQueryService()); - setUserOperations(other.getUserOperations()); - - // Set ShardQueryConfiguration variables - setConfig(RemoteQueryConfiguration.create(other)); - } - - public String getRemoteId() { - return getConfig().getRemoteId(); - } - - public void setRemoteId(String id) { - getConfig().setRemoteId(id); - getConfig().setQueryString("( metrics = '" + remoteQueryService.getQueryMetricsURI(id).toString() + "' )"); - } - - public String getRemoteQueryLogic() { - return getConfig().getRemoteQueryLogic(); - } - - public void setRemoteQueryLogic(String remoteQueryLogic) { - getConfig().setRemoteQueryLogic(remoteQueryLogic); - } - - public Object getCallerObject() { - return getPrincipal(); - } - - @Override - public GenericQueryConfiguration initialize(AccumuloClient connection, Query settings, Set auths) throws Exception { - Map> parms = settings.toMap(); - // we need to ensure that the remote query request includes the local query id for tracking purposes via query metrics. - if (!parms.containsKey(QUERY_ID) && settings.getId() != null) { - parms.put(QUERY_ID, Collections.singletonList(settings.getId().toString())); - } - GenericResponse createResponse = remoteQueryService.createQuery(getRemoteQueryLogic(), parms, getCallerObject()); - setRemoteId(createResponse.getResult()); - log.info("Local query " + settings.getId() + " maps to remote query " + getRemoteId()); - return getConfig(); - } - - @Override - public String getPlan(AccumuloClient connection, Query settings, Set auths, boolean expandFields, boolean expandValues) throws Exception { - GenericResponse planResponse = remoteQueryService.planQuery(getRemoteQueryLogic(), settings.toMap(), getCallerObject()); - return planResponse.getResult(); } @Override public void setupQuery(GenericQueryConfiguration genericConfig) throws Exception { - if (!RemoteQueryConfiguration.class.isAssignableFrom(genericConfig.getClass())) { - throw new QueryException("Did not receive a RemoteQueryConfiguration instance!!"); - } - - config = (RemoteQueryConfiguration) genericConfig; + setupConfig(genericConfig); // Create an iterator that returns a stream of EventBase objects iterator = new RemoteQueryLogicIterator(); } @Override - public QueryLogicTransformer getTransformer(Query settings) { - // a transformer that turns EventBase objects into a response - if (transformerInstance == null) { - transformerInstance = new EventBaseTransformer(settings, getMarkingFunctions(), getResponseObjectFactory()); - } - - return transformerInstance; + public QueryLogicTransformer createTransformer(Query settings, MarkingFunctions markingFunctions, + ResponseObjectFactory responseObjectFactory) { + return new EventBaseTransformer(settings, markingFunctions, responseObjectFactory); } @Override @@ -146,48 +67,6 @@ public RemoteEventQueryLogic clone() { return new RemoteEventQueryLogic(this); } - @Override - public void close() { - - super.close(); - - log.debug("Closing RemoteQueryLogic: " + System.identityHashCode(this)); - - if (getRemoteId() != null) { - try { - remoteQueryService.close(getRemoteId(), getCallerObject()); - } catch (Exception e) { - log.error("Failed to close remote query", e); - } - } - } - - @Override - public RemoteQueryConfiguration getConfig() { - if (config == null) { - config = RemoteQueryConfiguration.create(); - } - - return config; - } - - public void setConfig(RemoteQueryConfiguration config) { - this.config = config; - } - - public RemoteQueryService getRemoteQueryService() { - return remoteQueryService; - } - - public void setRemoteQueryService(RemoteQueryService remoteQueryService) { - this.remoteQueryService = remoteQueryService; - } - - @Override - public AccumuloConnectionFactory.Priority getConnectionPriority() { - return AccumuloConnectionFactory.Priority.NORMAL; - } - @Override public Set getOptionalQueryParameters() { return new ShardQueryLogic().getOptionalQueryParameters(); @@ -203,14 +82,6 @@ public Set getExampleQueries() { return new ShardQueryLogic().getExampleQueries(); } - public Query getSettings() { - return getConfig().getQuery(); - } - - public void setSettings(Query settings) { - getConfig().setQuery(settings); - } - private class RemoteQueryLogicIterator implements Iterator { private Queue data = new LinkedList<>(); private boolean complete = false; @@ -269,14 +140,4 @@ public EventBase transform(EventBase input) throws EmptyObjectException { } } - - @Override - public void setUserOperations(UserOperations userOperations) { - this.userOperations = userOperations; - } - - @Override - public UserOperations getUserOperations() { - return userOperations; - } } diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEdgeQueryLogicHttpTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEdgeQueryLogicHttpTest.java new file mode 100644 index 00000000000..825b5ab1360 --- /dev/null +++ b/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEdgeQueryLogicHttpTest.java @@ -0,0 +1,252 @@ +package datawave.query.tables; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.math.BigInteger; +import java.net.HttpURLConnection; +import java.net.InetSocketAddress; +import java.nio.charset.Charset; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.PrivateKey; +import java.security.SecureRandom; +import java.security.cert.X509Certificate; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.UUID; + +import javax.ws.rs.core.MediaType; + +import org.apache.commons.io.IOUtils; +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URLEncodedUtils; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.X509v3CertificateBuilder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.operator.ContentSigner; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; + +import datawave.security.authorization.DatawavePrincipal; +import datawave.security.util.DnUtils; +import datawave.webservice.common.json.DefaultMapperDecorator; +import datawave.webservice.common.remote.TestJSSESecurityDomain; +import datawave.webservice.query.QueryImpl; +import datawave.webservice.query.QueryParameters; +import datawave.webservice.query.configuration.GenericQueryConfiguration; +import datawave.webservice.query.remote.RemoteQueryServiceImpl; +import datawave.webservice.query.result.edge.DefaultEdge; +import datawave.webservice.query.result.edge.EdgeBase; +import datawave.webservice.query.result.event.DefaultResponseObjectFactory; +import datawave.webservice.result.DefaultEdgeQueryResponse; +import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.VoidResponse; + +public class RemoteEdgeQueryLogicHttpTest { + + private static final int keysize = 2048; + + private static final String commonName = "cn=www.test.us"; + private static final String alias = "tomcat"; + private static final char[] keyPass = "changeit".toCharArray(); + private static final String query = "Grinning\uD83D\uDE00Face"; + + private X500Name x500Name; + RemoteEdgeQueryLogic logic = new RemoteEdgeQueryLogic(); + + private static final int PORT = 0; + + private HttpServer server; + + volatile int nextCalls = 0; + + private volatile String content = null; + + private void setContent(InputStream content) throws IOException { + StringBuilder builder = new StringBuilder(); + InputStreamReader reader = new InputStreamReader(content, "UTF8"); + char[] buffer = new char[1024]; + int chars = reader.read(buffer); + while (chars >= 0) { + builder.append(buffer, 0, chars); + chars = reader.read(buffer); + } + List data = URLEncodedUtils.parse(builder.toString(), Charset.forName("UTF-8")); + for (NameValuePair pair : data) { + if (pair.getName().equals(QueryParameters.QUERY_STRING)) { + this.content = pair.getValue(); + break; + } + } + } + + @Before + public void setup() throws Exception { + final ObjectMapper objectMapper = new DefaultMapperDecorator().decorate(new ObjectMapper()); + System.setProperty(DnUtils.SUBJECT_DN_PATTERN_PROPERTY, ".*ou=server.*"); + KeyPairGenerator generater = KeyPairGenerator.getInstance("RSA"); + generater.initialize(keysize); + KeyPair keypair = generater.generateKeyPair(); + PrivateKey privKey = keypair.getPrivate(); + final X509Certificate[] chain = new X509Certificate[1]; + x500Name = new X500Name(commonName); + SubjectPublicKeyInfo subPubKeyInfo = SubjectPublicKeyInfo.getInstance(keypair.getPublic().getEncoded()); + final Date start = new Date(); + final Date until = Date.from(LocalDate.now().plus(365, ChronoUnit.DAYS).atStartOfDay().toInstant(ZoneOffset.UTC)); + X509v3CertificateBuilder builder = new X509v3CertificateBuilder(x500Name, new BigInteger(10, new SecureRandom()), // Choose something better for real + // use + start, until, x500Name, subPubKeyInfo); + ContentSigner signer = new JcaContentSignerBuilder("SHA256WithRSA").setProvider(new BouncyCastleProvider()).build(keypair.getPrivate()); + final X509CertificateHolder holder = builder.build(signer); + + chain[0] = new JcaX509CertificateConverter().setProvider(new BouncyCastleProvider()).getCertificate(holder); + + server = HttpServer.create(new InetSocketAddress(PORT), 0); + server.setExecutor(null); + server.start(); + + UUID uuid = UUID.randomUUID(); + GenericResponse createResponse = new GenericResponse(); + createResponse.setResult(uuid.toString()); + + HttpHandler createHandler = new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + setContent(exchange.getRequestBody()); + String responseBody = objectMapper.writeValueAsString(createResponse); + exchange.getResponseHeaders().add("Content-Type", MediaType.APPLICATION_JSON); + exchange.sendResponseHeaders(HttpURLConnection.HTTP_OK, responseBody.length()); + IOUtils.write(responseBody, exchange.getResponseBody(), Charset.forName("UTF-8")); + exchange.close(); + } + }; + + DefaultEdgeQueryResponse response1 = new DefaultEdgeQueryResponse(); + DefaultEdge edge1 = new DefaultEdge(); + edge1.setSource("source1"); + edge1.setSink("sink1"); + edge1.setEdgeAttribute1Source("edgeAttr1Source1"); + edge1.setEdgeRelationship("edgeRel1"); + edge1.setEdgeAttribute2("edgeAttr21"); + edge1.setEdgeAttribute3("edgeAttr31"); + edge1.setDate("20230101"); + edge1.setLoadDate("20230101"); + edge1.setActivityDate("20230101"); + edge1.setEdgeType("type1"); + edge1.setColumnVisibility("FOO|BAR"); + response1.setEdges(Collections.singletonList(edge1)); + response1.setTotalResults(1L); + + DefaultEdgeQueryResponse response2 = new DefaultEdgeQueryResponse(); + DefaultEdge edge2 = new DefaultEdge(); + edge2.setSource("source2"); + edge2.setSink("sink2"); + edge2.setEdgeAttribute1Source("edgeAttr1Source2"); + edge2.setEdgeRelationship("edgeRel2"); + edge2.setEdgeAttribute2("edgeAttr22"); + edge2.setEdgeAttribute3("edgeAttr32"); + edge2.setDate("20230101"); + edge2.setLoadDate("20230101"); + edge2.setActivityDate("20230101"); + edge2.setEdgeType("type2"); + edge2.setColumnVisibility("FOO|BAR"); + response2.setEdges(Collections.singletonList(edge1)); + response2.setTotalResults(1L); + + DefaultEdgeQueryResponse response3 = new DefaultEdgeQueryResponse(); + response3.setTotalResults(0L); + + HttpHandler nextHandler = new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + nextCalls++; + DefaultEdgeQueryResponse response = (nextCalls == 1 ? response1 : (nextCalls == 2 ? response2 : response3)); + String responseBody = objectMapper.writeValueAsString(response); + exchange.getResponseHeaders().add("Content-Type", MediaType.APPLICATION_JSON); + int responseCode = nextCalls > 2 ? HttpURLConnection.HTTP_NO_CONTENT : HttpURLConnection.HTTP_OK; + exchange.sendResponseHeaders(responseCode, responseBody.length()); + IOUtils.write(responseBody, exchange.getResponseBody(), Charset.forName("UTF-8")); + exchange.close(); + } + }; + + VoidResponse closeResponse = new VoidResponse(); + closeResponse.addMessage(uuid.toString() + " closed."); + + HttpHandler closeHandler = new HttpHandler() { + @Override + public void handle(HttpExchange exchange) throws IOException { + String responseBody = objectMapper.writeValueAsString(closeResponse); + exchange.getResponseHeaders().add("Content-Type", MediaType.APPLICATION_JSON); + exchange.sendResponseHeaders(HttpURLConnection.HTTP_OK, responseBody.length()); + IOUtils.write(responseBody, exchange.getResponseBody(), Charset.forName("UTF-8")); + exchange.close(); + } + }; + + server.createContext("/DataWave/Query/TestQuery/create", createHandler); + server.createContext("/DataWave/Query/" + uuid.toString() + "/next", nextHandler); + server.createContext("/DataWave/Query/" + uuid.toString() + "/close", closeHandler); + + // create a remote edge query logic that has our own server behind it + RemoteQueryServiceImpl remote = new RemoteQueryServiceImpl(); + remote.setQueryServiceURI("/DataWave/Query/"); + remote.setQueryServiceScheme("http"); + remote.setQueryServiceHost("localhost"); + remote.setQueryServicePort(server.getAddress().getPort()); + remote.setExecutorService(null); + remote.setObjectMapperDecorator(new DefaultMapperDecorator()); + remote.setResponseObjectFactory(new DefaultResponseObjectFactory()); + remote.setJsseSecurityDomain(new TestJSSESecurityDomain(alias, privKey, keyPass, chain)); + remote.setNextQueryResponseClass(remote.getResponseObjectFactory().getEdgeQueryResponse().getClass()); + + logic.setRemoteQueryService(remote); + logic.setRemoteQueryLogic("TestQuery"); + } + + @After + public void after() { + if (server != null) { + server.stop(0); + } + } + + @Test + public void testRemoteQuery() throws Exception { + logic.setPrincipal(new DatawavePrincipal(commonName)); + QueryImpl settings = new QueryImpl(); + settings.setQuery(query); + GenericQueryConfiguration config = logic.initialize(null, settings, null); + logic.setupQuery(config); + + Iterator t = logic.iterator(); + List edges = new ArrayList(); + while (t.hasNext()) { + edges.add(t.next()); + } + assertEquals(2, edges.size()); + assertNotNull(content); + assertEquals(query, content); + } + +} diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEdgeQueryLogicTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEdgeQueryLogicTest.java new file mode 100644 index 00000000000..7326badcf01 --- /dev/null +++ b/warehouse/query-core/src/test/java/datawave/query/tables/RemoteEdgeQueryLogicTest.java @@ -0,0 +1,172 @@ +package datawave.query.tables; + +import static org.junit.Assert.assertEquals; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +import datawave.security.authorization.DatawavePrincipal; +import datawave.security.authorization.DatawaveUser; +import datawave.security.authorization.SubjectIssuerDNPair; +import datawave.webservice.common.remote.RemoteHttpService; +import datawave.webservice.common.remote.RemoteQueryService; +import datawave.webservice.query.QueryImpl; +import datawave.webservice.query.configuration.GenericQueryConfiguration; +import datawave.webservice.query.result.edge.DefaultEdge; +import datawave.webservice.query.result.edge.EdgeBase; +import datawave.webservice.result.BaseQueryResponse; +import datawave.webservice.result.DefaultEdgeQueryResponse; +import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.VoidResponse; + +public class RemoteEdgeQueryLogicTest { + + RemoteEdgeQueryLogic logic = new RemoteEdgeQueryLogic(); + + @Before + public void setup() { + UUID uuid = UUID.randomUUID(); + GenericResponse createResponse = new GenericResponse(); + createResponse.setResult(uuid.toString()); + + DefaultEdgeQueryResponse response1 = new DefaultEdgeQueryResponse(); + DefaultEdge edge1 = new DefaultEdge(); + edge1.setSource("source1"); + edge1.setSink("sink1"); + edge1.setCount(1L); + edge1.setEdgeAttribute1Source("edgeAttr1Source1"); + edge1.setEdgeRelationship("edgeRel1"); + edge1.setEdgeAttribute2("edgeAttr21"); + edge1.setEdgeAttribute3("edgeAttr31"); + edge1.setDate("20230101"); + edge1.setLoadDate("20230101"); + edge1.setActivityDate("20230101"); + edge1.setEdgeType("type1"); + edge1.setColumnVisibility("PUBLIC"); + response1.setEdges(Collections.singletonList(edge1)); + response1.setTotalResults(1L); + + DefaultEdgeQueryResponse response2 = new DefaultEdgeQueryResponse(); + DefaultEdge edge2 = new DefaultEdge(); + edge2.setSource("source2"); + edge2.setSink("sink2"); + edge2.setCount(1L); + edge2.setEdgeAttribute1Source("edgeAttr1Source2"); + edge2.setEdgeRelationship("edgeRel2"); + edge2.setEdgeAttribute2("edgeAttr22"); + edge2.setEdgeAttribute3("edgeAttr32"); + edge2.setDate("20230101"); + edge2.setLoadDate("20230101"); + edge2.setActivityDate("20230101"); + edge2.setEdgeType("type2"); + edge2.setColumnVisibility("PUBLIC"); + response2.setEdges(Collections.singletonList(edge1)); + response2.setTotalResults(1L); + + DefaultEdgeQueryResponse response3 = new DefaultEdgeQueryResponse(); + response3.setTotalResults(0L); + + // create a remote Edge query logic that has our own remote query service behind it + logic.setRemoteQueryService(new TestRemoteQueryService(createResponse, response1, response2, response3)); + logic.setRemoteQueryLogic("TestQuery"); + } + + @Test + public void testRemoteQuery() throws Exception { + GenericQueryConfiguration config = logic.initialize(null, new QueryImpl(), null); + logic.setupQuery(config); + Iterator t = logic.iterator(); + List Edges = new ArrayList(); + while (t.hasNext()) { + Edges.add(t.next()); + } + assertEquals(2, Edges.size()); + } + + @Test + public void testProxiedHeaders() throws Exception { + SubjectIssuerDNPair userDN = SubjectIssuerDNPair.of("userDN", "issuerDN"); + SubjectIssuerDNPair p1dn = SubjectIssuerDNPair.of("entity1UserDN", "entity1IssuerDN"); + SubjectIssuerDNPair p2dn = SubjectIssuerDNPair.of("entity2UserDN", "entity2IssuerDN"); + SubjectIssuerDNPair p3dn = SubjectIssuerDNPair.of("entity3UserDN", "entity3IssuerDN"); + + DatawaveUser user = new DatawaveUser(userDN, DatawaveUser.UserType.USER, Sets.newHashSet("A", "C", "D"), null, null, System.currentTimeMillis()); + DatawaveUser p1 = new DatawaveUser(p1dn, DatawaveUser.UserType.SERVER, Sets.newHashSet("A", "B", "E"), null, null, System.currentTimeMillis()); + DatawaveUser p2 = new DatawaveUser(p2dn, DatawaveUser.UserType.SERVER, Sets.newHashSet("A", "F", "G"), null, null, System.currentTimeMillis()); + DatawaveUser p3 = new DatawaveUser(p3dn, DatawaveUser.UserType.SERVER, Sets.newHashSet("A", "B", "G"), null, null, System.currentTimeMillis()); + + DatawavePrincipal proxiedUserPrincipal = new DatawavePrincipal(Lists.newArrayList(user, p1, p2)); + DatawavePrincipal proxiedServerPrincipal1 = new DatawavePrincipal(Lists.newArrayList(p3, p1)); + DatawavePrincipal proxiedServerPrincipal2 = new DatawavePrincipal(Lists.newArrayList(p2, p3, p1)); + + Assert.assertEquals("", RemoteHttpService.getProxiedEntities(proxiedUserPrincipal)); + Assert.assertEquals("", RemoteHttpService.getProxiedEntities(proxiedServerPrincipal1)); + Assert.assertEquals("", RemoteHttpService.getProxiedEntities(proxiedServerPrincipal2)); + + Assert.assertEquals("", RemoteHttpService.getProxiedIssuers(proxiedUserPrincipal)); + Assert.assertEquals("", RemoteHttpService.getProxiedIssuers(proxiedServerPrincipal1)); + Assert.assertEquals("", RemoteHttpService.getProxiedIssuers(proxiedServerPrincipal2)); + } + + public static class TestRemoteQueryService implements RemoteQueryService { + GenericResponse createResponse; + LinkedList nextResponses; + + public TestRemoteQueryService(GenericResponse createResponse, BaseQueryResponse response1, BaseQueryResponse response2, + BaseQueryResponse response3) { + this.createResponse = createResponse; + this.nextResponses = new LinkedList<>(); + nextResponses.add(response1); + nextResponses.add(response2); + nextResponses.add(response3); + } + + @Override + public GenericResponse createQuery(String queryLogicName, Map> queryParameters, Object callerObject) { + return createResponse; + } + + @Override + public BaseQueryResponse next(String id, Object callerObject) { + return nextResponses.poll(); + } + + @Override + public VoidResponse close(String id, Object callerObject) { + return new VoidResponse(); + } + + @Override + public GenericResponse planQuery(String queryLogicName, Map> queryParameters, Object callerObject) { + throw new UnsupportedOperationException(); + } + + @Override + public GenericResponse planQuery(String id, Object callerObject) { + throw new UnsupportedOperationException(); + } + + @Override + public URI getQueryMetricsURI(String id) { + try { + return new URI("https://localhost:8443/DataWave/Query/Metrics/id/" + id); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } +} diff --git a/web-services/common/src/main/java/datawave/webservice/common/json/DefaultMapperDecorator.java b/web-services/common/src/main/java/datawave/webservice/common/json/DefaultMapperDecorator.java index bc9de595280..759b105a19d 100644 --- a/web-services/common/src/main/java/datawave/webservice/common/json/DefaultMapperDecorator.java +++ b/web-services/common/src/main/java/datawave/webservice/common/json/DefaultMapperDecorator.java @@ -1,10 +1,13 @@ package datawave.webservice.common.json; +import com.fasterxml.jackson.databind.AnnotationIntrospector; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector; import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.datatype.guava.GuavaModule; +import com.fasterxml.jackson.module.jaxb.JaxbAnnotationIntrospector; import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule; import datawave.microservice.querymetric.BaseQueryMetricListResponse; @@ -24,6 +27,8 @@ public ObjectMapper decorate(ObjectMapper mapper) { mapper.registerModule(new GuavaModule()); mapper.registerModule(new JaxbAnnotationModule()); mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + mapper.setAnnotationIntrospector( + AnnotationIntrospector.pair(new JacksonAnnotationIntrospector(), new JaxbAnnotationIntrospector(mapper.getTypeFactory()))); registerAbstractTypes(mapper); diff --git a/web-services/common/src/main/java/datawave/webservice/common/remote/RemoteHttpService.java b/web-services/common/src/main/java/datawave/webservice/common/remote/RemoteHttpService.java index 248285118e8..24814d6ab8d 100644 --- a/web-services/common/src/main/java/datawave/webservice/common/remote/RemoteHttpService.java +++ b/web-services/common/src/main/java/datawave/webservice/common/remote/RemoteHttpService.java @@ -347,7 +347,7 @@ public T readResponse(HttpEntity entity, ObjectReader reader1, ObjectReader log.error("Failed to read entity content. Trying as a VoidResponse.", ioe); log.error(content); VoidResponse response = voidResponseReader.readValue(content); - throw new RuntimeException(response.getMessages().toString()); + throw new RuntimeException(String.valueOf(response.getMessages()), ioe1); } } } diff --git a/web-services/query/src/main/java/datawave/webservice/query/remote/RemoteQueryServiceImpl.java b/web-services/query/src/main/java/datawave/webservice/query/remote/RemoteQueryServiceImpl.java index e0c5741f41d..63433ea70e2 100644 --- a/web-services/query/src/main/java/datawave/webservice/query/remote/RemoteQueryServiceImpl.java +++ b/web-services/query/src/main/java/datawave/webservice/query/remote/RemoteQueryServiceImpl.java @@ -50,10 +50,12 @@ public class RemoteQueryServiceImpl extends RemoteHttpService implements RemoteQ private ObjectReader baseQueryResponseReader; - private ObjectReader eventQueryResponseReader; + private ObjectReader nextQueryResponseReader; private boolean initialized = false; + private Class nextQueryResponseClass; + @Override @PostConstruct public void init() { @@ -61,7 +63,11 @@ public void init() { super.init(); genericResponseReader = objectMapper.readerFor(GenericResponse.class); baseQueryResponseReader = objectMapper.readerFor(BaseQueryResponse.class); - eventQueryResponseReader = objectMapper.readerFor(responseObjectFactory.getEventQueryResponse().getClass()); + if (nextQueryResponseClass == null) { + nextQueryResponseReader = objectMapper.readerFor(responseObjectFactory.getEventQueryResponse().getClass()); + } else { + nextQueryResponseReader = objectMapper.readerFor(nextQueryResponseClass); + } initialized = true; } } @@ -122,7 +128,7 @@ public BaseQueryResponse next(String id, Object callerObject) { httpGet.setHeader(PROXIED_ENTITIES_HEADER, getProxiedEntities(principal)); httpGet.setHeader(PROXIED_ISSUERS_HEADER, getProxiedIssuers(principal)); }, entity -> { - return readResponse(entity, eventQueryResponseReader, baseQueryResponseReader); + return readResponse(entity, nextQueryResponseReader, baseQueryResponseReader); }, () -> suffix); } @@ -175,4 +181,11 @@ private DatawavePrincipal getDatawavePrincipal(Object callerObject) { throw new RuntimeException("Cannot handle a " + callerObject.getClass() + ". Only DatawavePrincipal is accepted"); } + public Class getNextQueryResponseClass() { + return nextQueryResponseClass; + } + + public void setNextQueryResponseClass(Class nextQueryResponseClass) { + this.nextQueryResponseClass = nextQueryResponseClass; + } } From 35561c1bb05854d0eaa4636eeac16cd8755d9602 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Mon, 12 Feb 2024 11:10:18 -0500 Subject: [PATCH 08/50] Updates to try and avoid hanging ivarators (#2093) * Updates to try and avoid hanging ivarators * fixed configuration issues of the thread pools * Updates to allow value to be set via properties. --- properties/default.properties | 2 ++ .../DatawaveFieldIndexCachingIteratorJexl.java | 10 ++++++++-- .../core/iterators/IteratorThreadPoolManager.java | 11 +++++++++-- .../query/config/ShardQueryConfiguration.java | 10 ++++++++++ .../datawave/query/iterator/QueryIterator.java | 9 +++++---- .../datawave/query/iterator/QueryOptions.java | 15 +++++++++++++++ .../query/planner/DefaultQueryPlanner.java | 1 + .../datawave/query/tables/ShardQueryLogic.java | 8 ++++++++ .../query/config/ShardQueryConfigurationTest.java | 2 ++ .../datawave/query/QueryLogicFactory.xml | 2 ++ 10 files changed, 62 insertions(+), 8 deletions(-) diff --git a/properties/default.properties b/properties/default.properties index 771af191252..edc970f95d3 100644 --- a/properties/default.properties +++ b/properties/default.properties @@ -448,6 +448,8 @@ beq.maxTermExpansionThreshold=2000 beq.fieldIndexRangeSplit=16 # The max number of sources that can be created across ivarators for one scan beq.maxIvaratorSources=20 +# The max wait time in ms for an ivarator source (default 30 minutes) +beq.maxIvaratorSourceWait=1800000 # The max number of files that one ivarator can open at one time beq.maxIvaratorOpenFiles=100 # The max number of evaluation pipelines. They are run in a pool of threads controlled by the tserver.datawave.evaluation.threads accumulo configuration property which defaults to 100 (IteratorThreadPoolManager). diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/DatawaveFieldIndexCachingIteratorJexl.java b/warehouse/query-core/src/main/java/datawave/core/iterators/DatawaveFieldIndexCachingIteratorJexl.java index 53244434311..902529b63e7 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/DatawaveFieldIndexCachingIteratorJexl.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/DatawaveFieldIndexCachingIteratorJexl.java @@ -1224,8 +1224,14 @@ protected Future fillSet(final Range boundingFiRange, final TotalResults tota } }; - return IteratorThreadPoolManager.executeIvarator(runnable, DatawaveFieldIndexCachingIteratorJexl.this + " in " + boundingFiRange, this.initEnv); - + try { + return IteratorThreadPoolManager.executeIvarator(runnable, DatawaveFieldIndexCachingIteratorJexl.this + " in " + boundingFiRange, this.initEnv); + } catch (Exception e) { + log.error("Failed to execute a fill Set", e); + // if the execute somehow failed, we need to return the pool source. + returnPoolSource(source); + throw new RuntimeException(e); + } } /** diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/IteratorThreadPoolManager.java b/warehouse/query-core/src/main/java/datawave/core/iterators/IteratorThreadPoolManager.java index b0324f86ace..4afa1e815b3 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/IteratorThreadPoolManager.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/IteratorThreadPoolManager.java @@ -53,8 +53,15 @@ private ThreadPoolExecutor createExecutorService(final String prop, final String int max = getMaxThreads(prop, accumuloConfiguration); if (service.getMaximumPoolSize() != max) { log.info("Changing " + prop + " to " + max); - service.setMaximumPoolSize(max); - service.setCorePoolSize(max); + // if raising the max size, then we need to set the max first before the core + // otherwise we get an exception. Same in the reverse. + if (service.getMaximumPoolSize() < max) { + service.setMaximumPoolSize(max); + service.setCorePoolSize(max); + } else { + service.setCorePoolSize(max); + service.setMaximumPoolSize(max); + } } } catch (Throwable t) { log.error(t, t); diff --git a/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java b/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java index c90ea74865d..23bdb8e1f92 100644 --- a/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java +++ b/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java @@ -346,6 +346,7 @@ public class ShardQueryConfiguration extends GenericQueryConfiguration implement private boolean ivaratorPersistVerify = true; private int ivaratorPersistVerifyCount = 100; private int maxIvaratorSources = 33; + private long maxIvaratorSourceWait = 1000L * 60 * 30; private long maxIvaratorResults = -1; private int maxEvaluationPipelines = 25; private int maxPipelineCachedResults = 25; @@ -614,6 +615,7 @@ public ShardQueryConfiguration(ShardQueryConfiguration other) { this.setIvaratorPersistVerify(other.isIvaratorPersistVerify()); this.setIvaratorPersistVerifyCount(other.getIvaratorPersistVerifyCount()); this.setMaxIvaratorSources(other.getMaxIvaratorSources()); + this.setMaxIvaratorSourceWait(other.getMaxIvaratorSourceWait()); this.setMaxIvaratorResults(other.getMaxIvaratorResults()); this.setMaxEvaluationPipelines(other.getMaxEvaluationPipelines()); this.setMaxPipelineCachedResults(other.getMaxPipelineCachedResults()); @@ -1430,6 +1432,14 @@ public void setMaxIvaratorSources(int maxIvaratorSources) { this.maxIvaratorSources = maxIvaratorSources; } + public long getMaxIvaratorSourceWait() { + return maxIvaratorSourceWait; + } + + public void setMaxIvaratorSourceWait(long maxIvaratorSourceWait) { + this.maxIvaratorSourceWait = maxIvaratorSourceWait; + } + public long getMaxIvaratorResults() { return maxIvaratorResults; } diff --git a/warehouse/query-core/src/main/java/datawave/query/iterator/QueryIterator.java b/warehouse/query-core/src/main/java/datawave/query/iterator/QueryIterator.java index 42f01b1df57..55cac224149 100644 --- a/warehouse/query-core/src/main/java/datawave/query/iterator/QueryIterator.java +++ b/warehouse/query-core/src/main/java/datawave/query/iterator/QueryIterator.java @@ -1422,7 +1422,7 @@ protected IteratorBuildingVisitor createIteratorBuildingVisitor(Class> createIvaratorSourcePool(int maxIvaratorSources) { - return new GenericObjectPool<>(createIvaratorSourceFactory(this), createIvaratorSourcePoolConfig(maxIvaratorSources)); + protected GenericObjectPool> createIvaratorSourcePool(int maxIvaratorSources, long maxWait) { + return new GenericObjectPool<>(createIvaratorSourceFactory(this), createIvaratorSourcePoolConfig(maxIvaratorSources, maxWait)); } private BasePoolableObjectFactory> createIvaratorSourceFactory(SourceFactory sourceFactory) { @@ -1458,12 +1458,13 @@ public SortedKeyValueIterator makeObject() throws Exception { }; } - private GenericObjectPool.Config createIvaratorSourcePoolConfig(int maxIvaratorSources) { + private GenericObjectPool.Config createIvaratorSourcePoolConfig(int maxIvaratorSources, long maxWait) { GenericObjectPool.Config poolConfig = new GenericObjectPool.Config(); poolConfig.maxActive = maxIvaratorSources; poolConfig.maxIdle = maxIvaratorSources; poolConfig.minIdle = 0; poolConfig.whenExhaustedAction = WHEN_EXHAUSTED_BLOCK; + poolConfig.maxWait = maxWait; return poolConfig; } diff --git a/warehouse/query-core/src/main/java/datawave/query/iterator/QueryOptions.java b/warehouse/query-core/src/main/java/datawave/query/iterator/QueryOptions.java index e30602fad03..e556697cbed 100644 --- a/warehouse/query-core/src/main/java/datawave/query/iterator/QueryOptions.java +++ b/warehouse/query-core/src/main/java/datawave/query/iterator/QueryOptions.java @@ -227,6 +227,7 @@ public class QueryOptions implements OptionDescriber { public static final String IVARATOR_PERSIST_VERIFY_COUNT = "ivarator.persist.verify.count"; public static final String MAX_IVARATOR_SOURCES = "max.ivarator.sources"; + public static final String MAX_IVARATOR_SOURCE_WAIT = "max.ivarator.source.wait"; public static final String MAX_IVARATOR_RESULTS = "max.ivarator.results"; @@ -361,6 +362,7 @@ public class QueryOptions implements OptionDescriber { protected FileSortedSet.PersistOptions ivaratorPersistOptions = new FileSortedSet.PersistOptions(); protected int maxIvaratorSources = 33; + protected long maxIvaratorSourceWait = 1000L * 60 * 30; protected long maxIvaratorResults = -1; @@ -495,6 +497,7 @@ public void deepCopy(QueryOptions other) { this.maxIndexRangeSplit = other.maxIndexRangeSplit; this.ivaratorMaxOpenFiles = other.ivaratorMaxOpenFiles; this.maxIvaratorSources = other.maxIvaratorSources; + this.maxIvaratorSourceWait = other.maxIvaratorSourceWait; this.maxIvaratorResults = other.maxIvaratorResults; this.yieldThresholdMs = other.yieldThresholdMs; @@ -1001,6 +1004,14 @@ public void setMaxIvaratorSources(int maxIvaratorSources) { this.maxIvaratorSources = maxIvaratorSources; } + public long getMaxIvaratorSourceWait() { + return maxIvaratorSourceWait; + } + + public void setMaxIvaratorSourceWait(long maxIvaratorSourceWait) { + this.maxIvaratorSourceWait = maxIvaratorSourceWait; + } + public long getMaxIvaratorResults() { return maxIvaratorResults; } @@ -1656,6 +1667,10 @@ public boolean validateOptions(Map options) { this.setMaxIvaratorSources(Integer.parseInt(options.get(MAX_IVARATOR_SOURCES))); } + if (options.containsKey(MAX_IVARATOR_SOURCE_WAIT)) { + this.setMaxIvaratorSourceWait(Long.parseLong(options.get(MAX_IVARATOR_SOURCE_WAIT))); + } + if (options.containsKey(MAX_IVARATOR_RESULTS)) { this.setMaxIvaratorResults(Long.parseLong(options.get(MAX_IVARATOR_RESULTS))); } diff --git a/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java b/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java index e70b1f60cc9..ff7aa805081 100644 --- a/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java +++ b/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java @@ -2098,6 +2098,7 @@ protected Future loadQueryIterator(final MetadataHelper metadat addOption(cfg, QueryOptions.MAX_EVALUATION_PIPELINES, Integer.toString(config.getMaxEvaluationPipelines()), false); addOption(cfg, QueryOptions.MAX_PIPELINE_CACHED_RESULTS, Integer.toString(config.getMaxPipelineCachedResults()), false); addOption(cfg, QueryOptions.MAX_IVARATOR_SOURCES, Integer.toString(config.getMaxIvaratorSources()), false); + addOption(cfg, QueryOptions.MAX_IVARATOR_SOURCE_WAIT, Long.toString(config.getMaxIvaratorSourceWait()), false); if (config.getYieldThresholdMs() != Long.MAX_VALUE && config.getYieldThresholdMs() > 0) { addOption(cfg, QueryOptions.YIELD_THRESHOLD_MS, Long.toString(config.getYieldThresholdMs()), false); diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java index 23f2165922c..dbeed1383a9 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java +++ b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java @@ -1950,6 +1950,14 @@ public void setMaxIvaratorSources(int maxIvaratorSources) { getConfig().setMaxIvaratorSources(maxIvaratorSources); } + public long getMaxIvaratorSourceWait() { + return getConfig().getMaxIvaratorSourceWait(); + } + + public void setMaxIvaratorSourceWait(long maxIvaratorSourceWait) { + getConfig().setMaxIvaratorSourceWait(maxIvaratorSourceWait); + } + public long getMaxIvaratorResults() { return getConfig().getMaxIvaratorResults(); } diff --git a/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java b/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java index 065752aad48..65d3131230f 100644 --- a/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java @@ -377,6 +377,8 @@ public void setUp() throws Exception { updatedValues.put("maxIvaratorSources", 16); defaultValues.put("maxIvaratorResults", -1L); updatedValues.put("maxIvaratorResults", 10000L); + defaultValues.put("maxIvaratorSourceWait", 1000L * 60 * 30); + updatedValues.put("maxIvaratorSourceWait", 1000L * 60 * 10); defaultValues.put("maxEvaluationPipelines", 25); updatedValues.put("maxEvaluationPipelines", 24); defaultValues.put("maxPipelineCachedResults", 25); diff --git a/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml b/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml index b6d04a60dd8..9f441455ea7 100644 --- a/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml +++ b/web-services/deploy/configuration/src/main/resources/datawave/query/QueryLogicFactory.xml @@ -268,6 +268,8 @@ + + From 7afbc59db1e5159b2717641e54c4130ac76252c2 Mon Sep 17 00:00:00 2001 From: "P.A" Date: Mon, 12 Feb 2024 12:09:12 -0500 Subject: [PATCH 09/50] Clearing noisy logs in start-loader (#2258) * Cleaning noisy logs in start-loader * Removing tracer log calls * Adjusting logs for additional loader * Small adjustment to echo --------- Co-authored-by: pcagbu --- .../src/main/resources/bin/ingest/start-loader.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/warehouse/ingest-scripts/src/main/resources/bin/ingest/start-loader.sh b/warehouse/ingest-scripts/src/main/resources/bin/ingest/start-loader.sh index 8e8d8eab3d6..b73ec0b860c 100755 --- a/warehouse/ingest-scripts/src/main/resources/bin/ingest/start-loader.sh +++ b/warehouse/ingest-scripts/src/main/resources/bin/ingest/start-loader.sh @@ -78,11 +78,9 @@ if [[ ${TOTAL} -gt 0 ]]; then for ((LOADER = 0; LOADER < ${#MAP_LOADER_HDFS_NAME_NODES[@]}; LOADER = $((LOADER + 1)))); do # Make sure that there are no active loaders running on the namenode in question before "resetting" perceived stuck files in HDFS currentLoaders=$(ps -eaf | grep [b]ulkIngestMap | grep ${MAP_LOADER_HDFS_NAME_NODES[$LOADER]} | cut -d" " -f1-7) - echo "currentLoaders is: " $currentLoaders FILES_STUCK_LOADING=$(${INGEST_HADOOP_HOME}/bin/hadoop fs -ls "${MAP_LOADER_HDFS_NAME_NODES[$LOADER]}$BASE_WORK_DIR/*/job.loading" | awk '{print $NF}') if [[ ! -z $FILES_STUCK_LOADING && -z $currentLoaders ]]; then echo "About to reset stuck files, no active loaders detected on ${MAP_LOADER_HDFS_NAME_NODES[$LOADER]}" - echo "FILES_STUCK_LOADING: " $FILES_STUCK_LOADING for stuckFile in $FILES_STUCK_LOADING; do echo "Resetting ${stuckFile} to ${stuckFile%.loading}.complete" moving=$(${INGEST_HADOOP_HOME}/bin/hadoop fs -mv $stuckFile ${stuckFile%.loading}.complete 2>&1) @@ -94,7 +92,7 @@ if [[ ${TOTAL} -gt 0 ]]; then COUNT=${NUM_MAP_LOADERS_COPY[$LOADER]} MAP_LOADER_HDFS_NAME_NODE=${MAP_LOADER_HDFS_NAME_NODES[$LOADER]} export MAP_LOADER_WORKDIR=${MAP_LOADER_HDFS_DIRS[$LOADER]} - echo "starting $COUNT map file loaders for ${MAP_LOADER_WORKDIR} on ${MAP_LOADER_HDFS_NAME_NODE} ..." + echo "starting $COUNT map file loaders for ${MAP_LOADER_HDFS_NAME_NODE} ..." portInUse=$(lsof -i:${SHUTDOWN_PORT} | grep $SHUTDOWN_PORT) portUsed=$(ps -eaf | grep [b]ulkIngestMap | grep $SHUTDOWN_PORT) for (( x=0; x < $COUNT; x=$((x+1)) )) ; do @@ -105,7 +103,7 @@ if [[ ${TOTAL} -gt 0 ]]; then portInUse=$(lsof -i:${SHUTDOWN_PORT} | grep $SHUTDOWN_PORT) portUsed=$(ps -eaf | grep [b]ulkIngestMap | grep $SHUTDOWN_PORT) done - echo starting $MAPFILE_LOADER_CMD -srcHdfs ${MAP_LOADER_HDFS_NAME_NODE} -destHdfs ${MAP_LOADER_HDFS_NAME_NODE} -shutdownPort ${SHUTDOWN_PORT} with log file $LOG_DIR/map-file-loader.$LOADER$x.log + echo starting map file loader with log file map-file-loader.$LOADER$x.log $MAPFILE_LOADER_CMD -srcHdfs ${MAP_LOADER_HDFS_NAME_NODE} -destHdfs ${MAP_LOADER_HDFS_NAME_NODE} -shutdownPort ${SHUTDOWN_PORT} >> $LOG_DIR/map-file-loader.$LOADER$x.log 2>&1 & SHUTDOWN_PORT=$((SHUTDOWN_PORT + 1)) portInUse=$(lsof -i:${SHUTDOWN_PORT} | grep $SHUTDOWN_PORT) @@ -125,7 +123,7 @@ if [[ ${TOTAL} -gt 0 ]]; then LOADER=${#MAP_LOADER_HDFS_NAME_NODES[@]} COUNT=0 export MAP_LOADER_WORKDIR=${BASE_WORK_DIR} - echo "starting 1 extra map file loader for ${MAP_LOADER_WORKDIR} on ${EXTRA_MAP_LOADER} ..." + echo "starting 1 extra map file loader for ${EXTRA_MAP_LOADER} ..." SHUTDOWN_PORT=$((SHUTDOWN_PORT + 1)) portInUse=$(lsof -i:${SHUTDOWN_PORT} | grep $SHUTDOWN_PORT) portUsed=$(ps -eaf | grep [b]ulkIngestMap | grep $SHUTDOWN_PORT) @@ -135,7 +133,7 @@ if [[ ${TOTAL} -gt 0 ]]; then portInUse=$(lsof -i:${SHUTDOWN_PORT} | grep $SHUTDOWN_PORT) portUsed=$(ps -eaf | grep [b]ulkIngestMap | grep $SHUTDOWN_PORT) done - echo starting $MAPFILE_LOADER_CMD -srcHdfs ${EXTRA_MAP_LOADER} -destHdfs ${EXTRA_MAP_LOADER} -shutdownPort ${SHUTDOWN_PORT} with log file $LOG_DIR/map-file-loader.$LOADER$COUNT.log + echo starting map file loader with log file map-file-loader.$LOADER$COUNT.log $MAPFILE_LOADER_CMD -srcHdfs ${EXTRA_MAP_LOADER} -destHdfs ${EXTRA_MAP_LOADER} -shutdownPort ${SHUTDOWN_PORT} >>$LOG_DIR/map-file-loader.$LOADER$COUNT.log 2>&1 & fi @@ -151,7 +149,7 @@ if [[ ${TOTAL} -gt 0 ]]; then portInUse=$(lsof -i:${SHUTDOWN_PORT} | grep $SHUTDOWN_PORT) portUsed=$(ps -eaf | grep [b]ulkIngestMap | grep $SHUTDOWN_PORT) done - echo starting ${MAP_LOADER_CUSTOM[$CUSTOM_LOADER]} -shutdownPort ${SHUTDOWN_PORT} with log file $LOG_DIR/map-file-loader-custom.$CUSTOM_LOADER.log + echo starting map file loader with log file $LOG_DIR/map-file-loader-custom.$CUSTOM_LOADER.log ${MAP_LOADER_CUSTOM[$CUSTOM_LOADER]} -shutdownPort ${SHUTDOWN_PORT} >>$LOG_DIR/map-file-loader-custom.$CUSTOM_LOADER.log 2>&1 & done fi From 8a05595cc6e9252840ff721e748c30d4e22cae98 Mon Sep 17 00:00:00 2001 From: Christopher Gross Date: Mon, 12 Feb 2024 13:02:37 -0500 Subject: [PATCH 10/50] 1799 add codestyle (#1811) - added git pre-push hook to ensure code is formatted properly Co-authored-by: Christopher Gross cogross --- githooks/README.md | 6 ++++++ githooks/pre-push | 4 ++++ .../main/java/datawave/query/tracking/ActiveQueryLog.java | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 githooks/README.md create mode 100755 githooks/pre-push diff --git a/githooks/README.md b/githooks/README.md new file mode 100644 index 00000000000..60640f86454 --- /dev/null +++ b/githooks/README.md @@ -0,0 +1,6 @@ +githooks enable a user to have things happen automatically based on the git action taken. DataWave has the following hooks: + +pre-push +Before pushing the code up to github, this script will +RUN: + ln -fns githooks/pre-push .git/hooks/pre-push diff --git a/githooks/pre-push b/githooks/pre-push new file mode 100755 index 00000000000..03831136deb --- /dev/null +++ b/githooks/pre-push @@ -0,0 +1,4 @@ +#!/bin/sh + +# Format files +mvn -V -B -e -ntp "-Dstyle.color=always" clean formatter:format sortpom:sort impsort:sort -Pautoformat diff --git a/warehouse/query-core/src/main/java/datawave/query/tracking/ActiveQueryLog.java b/warehouse/query-core/src/main/java/datawave/query/tracking/ActiveQueryLog.java index a44631570dc..0c4fc79ed6b 100644 --- a/warehouse/query-core/src/main/java/datawave/query/tracking/ActiveQueryLog.java +++ b/warehouse/query-core/src/main/java/datawave/query/tracking/ActiveQueryLog.java @@ -77,7 +77,7 @@ public static ActiveQueryLog getInstance() { /** * Return the {@link ActiveQueryLog} instance associated with the specified name. If one does not exist, it will be created. - * + *

    * If the specified name is null or blank, the default instance with the name '{@value #DEFAULT_NAME}' will be returned. Additionally, the time the log was * last accessed will be updated to the current time in milliseconds, and if the log's timer was cancelled, it will be restarted. * From 13f196e4712516cdb4a94a38a1cf2e504c4cd200 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Mon, 12 Feb 2024 15:59:03 -0500 Subject: [PATCH 11/50] Use the pluginEnvironment instead of AccumuloConfiguration as the (#2200) * Use the pluginEnvironment instead of AccumuloConfiguration as the later class will break when tables are dropped * Updated tests to handle use of PluginEnvironment * Reapply TablePropKey error fix to ConfigurableAgeOffFilter --- .../iterators/IteratorThreadPoolManager.java | 23 ++++++----- .../query/iterator/QueryIteratorIT.java | 6 +++ .../query/iterator/SourceManagerTest.java | 38 +++++++++++++++++++ 3 files changed, 58 insertions(+), 9 deletions(-) diff --git a/warehouse/query-core/src/main/java/datawave/core/iterators/IteratorThreadPoolManager.java b/warehouse/query-core/src/main/java/datawave/core/iterators/IteratorThreadPoolManager.java index 4afa1e815b3..b0d47f7207d 100644 --- a/warehouse/query-core/src/main/java/datawave/core/iterators/IteratorThreadPoolManager.java +++ b/warehouse/query-core/src/main/java/datawave/core/iterators/IteratorThreadPoolManager.java @@ -10,6 +10,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import org.apache.accumulo.core.client.PluginEnvironment; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.iterators.IteratorEnvironment; @@ -39,18 +40,23 @@ private IteratorThreadPoolManager(IteratorEnvironment env) { createExecutorService(EVALUATOR_THREAD_PROP, EVALUATOR_THREAD_NAME, env); } - private ThreadPoolExecutor createExecutorService(final String prop, final String name, IteratorEnvironment env) { + private ThreadPoolExecutor createExecutorService(final String prop, final String name, final IteratorEnvironment env) { final AccumuloConfiguration accumuloConfiguration; + final PluginEnvironment pluginEnv; if (env != null) { + pluginEnv = env.getPluginEnv(); accumuloConfiguration = env.getConfig(); } else { + pluginEnv = null; accumuloConfiguration = DefaultConfiguration.getInstance(); } - final ThreadPoolExecutor service = createExecutorService(getMaxThreads(prop, accumuloConfiguration), name + " (" + instanceId + ')'); + final ThreadPoolExecutor service = createExecutorService(getMaxThreads(prop, pluginEnv), name + " (" + instanceId + ')'); threadPools.put(name, service); ThreadPools.getServerThreadPools().createGeneralScheduledExecutorService(accumuloConfiguration).scheduleWithFixedDelay(() -> { try { - int max = getMaxThreads(prop, accumuloConfiguration); + // Very important to not use the accumuloConfiguration in this thread and instead use the pluginEnv + // The accumuloConfiguration caches table ids which may no longer exist down the road. + int max = getMaxThreads(prop, pluginEnv); if (service.getMaximumPoolSize() != max) { log.info("Changing " + prop + " to " + max); // if raising the max size, then we need to set the max first before the core @@ -77,12 +83,11 @@ private ThreadPoolExecutor createExecutorService(int maxThreads, String name) { return pool; } - private int getMaxThreads(final String prop, AccumuloConfiguration conf) { - if (conf != null) { - Map properties = new TreeMap<>(); - conf.getProperties(properties, k -> Objects.equals(k, prop)); - if (properties.containsKey(prop)) { - return Integer.parseInt(properties.get(prop)); + private int getMaxThreads(final String prop, PluginEnvironment pluginEnv) { + if (pluginEnv != null && pluginEnv.getConfiguration() != null) { + String value = pluginEnv.getConfiguration().get(prop); + if (value != null) { + return Integer.parseInt(value); } } return DEFAULT_THREAD_POOL_SIZE; diff --git a/warehouse/query-core/src/test/java/datawave/query/iterator/QueryIteratorIT.java b/warehouse/query-core/src/test/java/datawave/query/iterator/QueryIteratorIT.java index 7ed1ee8d3bc..244f076503b 100644 --- a/warehouse/query-core/src/test/java/datawave/query/iterator/QueryIteratorIT.java +++ b/warehouse/query-core/src/test/java/datawave/query/iterator/QueryIteratorIT.java @@ -36,12 +36,14 @@ import java.util.Set; import java.util.stream.Collectors; +import org.apache.accumulo.core.client.PluginEnvironment; import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.PartialKey; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.IteratorEnvironment; +import org.apache.accumulo.core.util.ConfigurationImpl; import org.easymock.EasyMock; import org.easymock.EasyMockSupport; import org.junit.After; @@ -80,6 +82,7 @@ public class QueryIteratorIT extends EasyMockSupport { protected SortedListKeyValueIterator baseIterator; protected Map options; protected IteratorEnvironment environment; + protected PluginEnvironment penvironment; protected EventDataQueryFilter filter; protected TypeMetadata typeMetadata; @@ -145,7 +148,10 @@ public void setup() throws IOException { typeMetadata.put("INDEX_ONLY_FIELD3", DEFAULT_DATATYPE, "datawave.data.type.LcNoDiacriticsType"); environment = createMock(IteratorEnvironment.class); + penvironment = createMock(PluginEnvironment.class); EasyMock.expect(environment.getConfig()).andReturn(DefaultConfiguration.getInstance()).anyTimes(); + EasyMock.expect(environment.getPluginEnv()).andReturn(penvironment).anyTimes(); + EasyMock.expect(penvironment.getConfiguration()).andReturn(new ConfigurationImpl(DefaultConfiguration.getInstance())).anyTimes(); filter = createMock(EventDataQueryFilter.class); } diff --git a/warehouse/query-core/src/test/java/datawave/query/iterator/SourceManagerTest.java b/warehouse/query-core/src/test/java/datawave/query/iterator/SourceManagerTest.java index 717e1d4a606..b7d7f488be7 100644 --- a/warehouse/query-core/src/test/java/datawave/query/iterator/SourceManagerTest.java +++ b/warehouse/query-core/src/test/java/datawave/query/iterator/SourceManagerTest.java @@ -16,13 +16,16 @@ import java.util.TimeZone; import java.util.TreeMap; +import org.apache.accumulo.core.client.PluginEnvironment; import org.apache.accumulo.core.client.SampleNotPresentException; +import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.sample.SamplerConfiguration; import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.conf.DefaultConfiguration; import org.apache.accumulo.core.crypto.CryptoFactoryLoader; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.file.rfile.RFileOperations; import org.apache.accumulo.core.iterators.IteratorEnvironment; @@ -32,6 +35,7 @@ import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.spi.crypto.CryptoEnvironment; import org.apache.accumulo.core.spi.crypto.CryptoService; +import org.apache.accumulo.core.util.ConfigurationImpl; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.junit.Before; @@ -40,6 +44,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import net.bytebuddy.build.Plugin; + public class SourceManagerTest { private static final SimpleDateFormat shardFormatter = new SimpleDateFormat("yyyyMMdd HHmmss"); private static long ts = -1; @@ -367,6 +373,11 @@ public SortedKeyValueIterator reserveMapFileReader(String mapFileName .withTableConfiguration(DefaultConfiguration.getInstance()).seekToBeginning().build(); } + @Override + public PluginEnvironment getPluginEnv() { + return new MockPluginEnvironment(); + } + @Override public AccumuloConfiguration getConfig() { return conf; @@ -407,5 +418,32 @@ public void registerSideChannel(SortedKeyValueIterator iter) { throw new UnsupportedOperationException(); } + public class MockPluginEnvironment implements PluginEnvironment { + @Override + public Configuration getConfiguration() { + return new ConfigurationImpl(conf); + } + + @Override + public Configuration getConfiguration(TableId tableId) { + return new ConfigurationImpl(conf); + } + + @Override + public String getTableName(TableId tableId) throws TableNotFoundException { + return null; + } + + @Override + public T instantiate(String className, Class base) throws Exception { + return null; + } + + @Override + public T instantiate(TableId tableId, String className, Class base) throws Exception { + return null; + } + } + } } From cf9d8cf76326ffbfed8208fe4190d946f58fee8f Mon Sep 17 00:00:00 2001 From: Laura Schanno Date: Tue, 13 Feb 2024 14:25:17 -0500 Subject: [PATCH 12/50] Enrich 'i' and 'ri' rows in metadata table with event date (#2094) * Enrich 'i' and 'ri' rows in metadata table with event date Modify the generation of 'i' (indexed rows) and 'ri' (reverse indexed rows) in the metadata table such that the column qualifier contains the event date. This is required as a first step to support efforts for issue #825 so that we can identify dates when an event was ingested and included in a frequency count for an associated 'f' row, but was not indexed. * Add counts to 'i' and 'ri' rows --- .../ingest/metadata/EventMetadata.java | 86 ++- .../ingest/metadata/MetadataCounterGroup.java | 24 +- .../metadata/MetadataWithEventDate.java | 74 ++ .../metadata/MetadataWithMostRecentDate.java | 16 +- .../config/MetadataTableConfigHelper.java | 28 +- .../java/datawave/TestBaseIngestHelper.java | 1 + .../ingest/metadata/EventMetadataTest.java | 711 ++++++++++++------ .../metadata/MetadataCounterGroupTest.java | 10 +- .../MetadataWithMostRecentDateTest.java | 6 +- .../csv/TableConfigurationUtilTest.java | 8 +- 10 files changed, 656 insertions(+), 308 deletions(-) create mode 100644 warehouse/ingest-core/src/main/java/datawave/ingest/metadata/MetadataWithEventDate.java diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/metadata/EventMetadata.java b/warehouse/ingest-core/src/main/java/datawave/ingest/metadata/EventMetadata.java index 947ceb521a9..39bb38ea42a 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/metadata/EventMetadata.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/metadata/EventMetadata.java @@ -89,24 +89,27 @@ */ public class EventMetadata implements RawRecordMetadata { - private MetadataWithMostRecentDate compositeFieldsInfo = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_CI); - private MetadataWithMostRecentDate compositeSeparators = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_CISEP); - private MetadataWithMostRecentDate dataTypeFieldsInfo = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_T); - private MetadataWithMostRecentDate normalizedFieldsInfo = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_N); - private static final Logger log = getLogger(EventMetadata.class); private final Text metadataTableName; private final Text loadDatesTableName; + + private final MetadataWithMostRecentDate compositeFieldsInfo = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_CI); + private final MetadataWithMostRecentDate compositeSeparators = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_CISEP); + private final MetadataWithMostRecentDate dataTypeFieldsInfo = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_T); + private final MetadataWithMostRecentDate normalizedFieldsInfo = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_N); + // stores field name, data type, and most recent event date private final MetadataWithMostRecentDate eventFieldsInfo = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_E); - private final MetadataWithMostRecentDate indexedFieldsInfo = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_I); - private final MetadataWithMostRecentDate reverseIndexedFieldsInfo = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_RI); private final MetadataWithMostRecentDate termFrequencyFieldsInfo = new MetadataWithMostRecentDate(ColumnFamilyConstants.COLF_TF); + // stores counts private final MetadataCounterGroup frequencyCounts = new MetadataCounterGroup(ColumnFamilyConstants.COLF_F); // by event date private final MetadataCounterGroup indexedFieldsLoadDateCounts; private final MetadataCounterGroup reverseIndexedFieldsLoadDateCounts; - private boolean frequency = false; + private final MetadataCounterGroup indexedCounts = new MetadataCounterGroup(ColumnFamilyConstants.COLF_I); + private final MetadataCounterGroup reverseIndexedCounts = new MetadataCounterGroup(ColumnFamilyConstants.COLF_RI); + + private boolean writeFrequencyCounts = false; /** * @param shardTableName @@ -119,14 +122,14 @@ public class EventMetadata implements RawRecordMetadata { * used as part of column family within LOAD_DATES_TABLE_NAME * @param loadDatesTableName * the table name for the load dates - * @param frequency + * @param writeFrequencyCounts * whether to add to the metadata table's counts for field name by event date */ public EventMetadata(@SuppressWarnings("UnusedParameters") Text shardTableName, Text metadataTableName, Text loadDatesTableName, Text shardIndexTableName, - Text shardReverseIndexTableName, boolean frequency) { + Text shardReverseIndexTableName, boolean writeFrequencyCounts) { this.metadataTableName = metadataTableName; this.loadDatesTableName = loadDatesTableName; - this.frequency = frequency; + this.writeFrequencyCounts = writeFrequencyCounts; this.indexedFieldsLoadDateCounts = new MetadataCounterGroup("FIELD_NAME", shardIndexTableName); this.reverseIndexedFieldsLoadDateCounts = new MetadataCounterGroup("FIELD_NAME", shardReverseIndexTableName); @@ -134,17 +137,17 @@ public EventMetadata(@SuppressWarnings("UnusedParameters") Text shardTableName, @Override public void addEvent(IngestHelperInterface helper, RawRecordContainer event, Multimap fields, long loadTimeInMillis) { - addEvent(helper, event, fields, this.frequency, INCLUDE_LOAD_DATES, loadTimeInMillis); + addEvent(helper, event, fields, this.writeFrequencyCounts, INCLUDE_LOAD_DATES, loadTimeInMillis); } @Override public void addEvent(IngestHelperInterface helper, RawRecordContainer event, Multimap fields) { - addEvent(helper, event, fields, this.frequency, INCLUDE_LOAD_DATES, System.currentTimeMillis()); + addEvent(helper, event, fields, this.writeFrequencyCounts, INCLUDE_LOAD_DATES, System.currentTimeMillis()); } @Override public void addEventWithoutLoadDates(IngestHelperInterface helper, RawRecordContainer event, Multimap fields) { - addEvent(helper, event, fields, this.frequency, EXCLUDE_LOAD_DATES, System.currentTimeMillis()); + addEvent(helper, event, fields, this.writeFrequencyCounts, EXCLUDE_LOAD_DATES, System.currentTimeMillis()); } @Override @@ -241,17 +244,17 @@ protected void addToFrequencyCounts(RawRecordContainer event, String fieldName, protected void updateForIndexedField(@SuppressWarnings("UnusedParameters") IngestHelperInterface helper, RawRecordContainer event, Multimap fields, long countDelta, String loadDate, String tokenDesignator, String fieldName) { - update(event, fields.get(fieldName), tokenDesignator, countDelta, loadDate, indexedFieldsInfo, indexedFieldsLoadDateCounts); + update(event, fields.get(fieldName), tokenDesignator, countDelta, loadDate, indexedCounts, indexedFieldsLoadDateCounts); } private void updateForCompositeField(@SuppressWarnings("UnusedParameters") IngestHelperInterface helper, RawRecordContainer event, Multimap fields, long countDelta, String loadDate, String tokenDesignator, String fieldName) { - update(event, fields.get(fieldName), tokenDesignator, countDelta, loadDate, indexedFieldsInfo, indexedFieldsLoadDateCounts); + update(event, fields.get(fieldName), tokenDesignator, countDelta, loadDate, indexedCounts, indexedFieldsLoadDateCounts); } protected void updateForReverseIndexedField(@SuppressWarnings("UnusedParameters") IngestHelperInterface helper, RawRecordContainer event, Multimap fields, long countDelta, String loadDate, String tokenDesignator, String fieldName) { - update(event, fields.get(fieldName), tokenDesignator, countDelta, loadDate, reverseIndexedFieldsInfo, reverseIndexedFieldsLoadDateCounts); + update(event, fields.get(fieldName), tokenDesignator, countDelta, loadDate, reverseIndexedCounts, reverseIndexedFieldsLoadDateCounts); } protected void update(List> types, RawRecordContainer event, Collection norms, @@ -300,6 +303,28 @@ protected void update(RawRecordContainer event, Collection norms, String tokenDesignator, long countDelta, String loadDate, + MetadataWithEventDate metadata, MetadataCounterGroup counts) { + for (NormalizedContentInterface norm : norms) { + String field = norm.getIndexedFieldName() + tokenDesignator; + metadata.put(field, event.getDataType().outputName(), event.getDate()); + if (null != loadDate) { + updateLoadDateCounters(counts, event, field, countDelta, loadDate); + } + } + } + + protected void update(RawRecordContainer event, Collection norms, String tokenDesignator, long countDelta, String loadDate, + MetadataCounterGroup metadata, MetadataCounterGroup counts) { + for (NormalizedContentInterface norm : norms) { + String field = norm.getIndexedFieldName() + tokenDesignator; + metadata.addToCount(countDelta, event.getDataType().outputName(), field, DateHelper.format(event.getDate())); + if (null != loadDate) { + updateLoadDateCounters(counts, event, field, countDelta, loadDate); + } + } + } + protected void updateMetadata(MetadataWithMostRecentDate metadataInfo, @SuppressWarnings("UnusedParameters") IngestHelperInterface helper, RawRecordContainer event, Multimap fields, String fieldName) { // will ignore the update load counters because it's null @@ -381,10 +406,9 @@ public Multimap getBulkMetadata() { addIndexedFieldToMetadata(bulkData, eventFieldsInfo); addIndexedFieldToMetadata(bulkData, termFrequencyFieldsInfo); - addIndexedFieldToMetadata(bulkData, indexedFieldsInfo); - addIndexedFieldToMetadata(bulkData, reverseIndexedFieldsInfo); - - addFrequenciesToMetadata(bulkData); + addCountsToMetadata(bulkData, indexedCounts); + addCountsToMetadata(bulkData, reverseIndexedCounts); + addCountsToMetadata(bulkData, frequencyCounts); addIndexedFieldToMetadata(bulkData, dataTypeFieldsInfo); addIndexedFieldToMetadata(bulkData, normalizedFieldsInfo); @@ -400,7 +424,7 @@ public Multimap getBulkMetadata() { protected void addToLoadDates(Multimap results, MetadataCounterGroup countsGroup) { if (loadDatesTableName != null) { - for (MetadataCounterGroup.CountAndKeyComponents entry : countsGroup.getEntries()) { + for (MetadataCounterGroup.Components entry : countsGroup.getEntries()) { Long count = entry.getCount(); Key k = new Key(new Text(entry.getRowId()), countsGroup.getColumnFamily(), new Text(entry.getDate() + DELIMITER + entry.getDataType())); @@ -409,13 +433,14 @@ protected void addToLoadDates(Multimap results, MetadataCou } } - protected void addFrequenciesToMetadata(Multimap results) { - if (frequency) { - for (MetadataCounterGroup.CountAndKeyComponents entry : frequencyCounts.getEntries()) { + protected void addCountsToMetadata(Multimap results, MetadataCounterGroup frequencies) { + // Do not write the counts if these are for "f" rows and writeFrequencyCounts is false. + if (!frequencies.getColumnFamily().equals(ColumnFamilyConstants.COLF_F) || writeFrequencyCounts) { + for (MetadataCounterGroup.Components entry : frequencies.getEntries()) { Long count = entry.getCount(); - Key k = new Key(new Text(entry.getRowId()), frequencyCounts.getColumnFamily(), new Text(entry.getDataType() + DELIMITER + entry.getDate()), + Key key = new Key(new Text(entry.getRowId()), frequencies.getColumnFamily(), new Text(entry.getDataType() + DELIMITER + entry.getDate()), DateHelper.parse(entry.getDate()).getTime()); - addToResults(results, count, k, this.metadataTableName); + addToResults(results, count, key, this.metadataTableName); } } } @@ -426,7 +451,7 @@ protected void addToResults(Multimap results, Long value, K } protected void addIndexedFieldToMetadata(Multimap results, MetadataWithMostRecentDate mostRecentDates) { - for (MetadataWithMostRecentDate.MostRecentEventDateAndKeyComponents entry : mostRecentDates.entries()) { + for (MetadataWithMostRecentDate.Components entry : mostRecentDates.entries()) { long mostRecentDate = entry.getMostRecentDate(); Text fieldName = new Text(entry.getFieldName()); Text colq = new Text(entry.getDataType()); @@ -445,10 +470,9 @@ public void clear() { this.termFrequencyFieldsInfo.clear(); this.frequencyCounts.clear(); - this.indexedFieldsInfo.clear(); - this.indexedFieldsLoadDateCounts.clear(); + this.indexedCounts.clear(); + this.reverseIndexedCounts.clear(); - this.reverseIndexedFieldsInfo.clear(); this.reverseIndexedFieldsLoadDateCounts.clear(); } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/metadata/MetadataCounterGroup.java b/warehouse/ingest-core/src/main/java/datawave/ingest/metadata/MetadataCounterGroup.java index 00fcd3db2ec..c81da79a765 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/metadata/MetadataCounterGroup.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/metadata/MetadataCounterGroup.java @@ -11,7 +11,7 @@ public class MetadataCounterGroup { private static final Logger log = Logger.getLogger(MetadataCounterGroup.class); private final Text columnFamily; - private HashMap counts = new HashMap<>(); + private HashMap counts = new HashMap<>(); public MetadataCounterGroup(String groupName, Text tableName) { this.columnFamily = new Text(groupName + RawRecordMetadata.DELIMITER + tableName); @@ -28,9 +28,9 @@ private static String createKey(String dataType, String rowid, String date) { /* rowId is either the fieldName or the Lac */ public void addToCount(long countDelta, String dataType, String rowId, String date) { String hashMapKey = createKey(dataType, rowId, date); - CountAndKeyComponents value = counts.get(hashMapKey); + Components value = counts.get(hashMapKey); if (null == value) { - counts.put(hashMapKey, new CountAndKeyComponents(dataType, rowId, date, countDelta)); + counts.put(hashMapKey, new Components(dataType, rowId, date, countDelta)); } else { value.incrementCount(countDelta); } @@ -44,17 +44,17 @@ public Text getColumnFamily() { return columnFamily; } - public Collection getEntries() { + public Collection getEntries() { return counts.values(); } - public class CountAndKeyComponents { + public static class Components { private final String dataType; private final String rowId; private final String date; private long count; - public CountAndKeyComponents(String dataType, String rowId, String date, long countDelta) { + public Components(String dataType, String rowId, String date, long countDelta) { this.dataType = dataType; this.rowId = rowId; this.date = date; @@ -84,18 +84,18 @@ public int hashCode() { return hashCode; } + public long getCount() { + return count; + } + @Override public boolean equals(Object o) { - if (!(o instanceof CountAndKeyComponents)) { + if (!(o instanceof Components)) { return false; } - CountAndKeyComponents other = (CountAndKeyComponents) o; + Components other = (Components) o; return Objects.equal(this.dataType, other.dataType) && Objects.equal(this.rowId, other.rowId) && Objects.equal(this.date, other.date) && count == other.count; } - - public long getCount() { - return count; - } } } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/metadata/MetadataWithEventDate.java b/warehouse/ingest-core/src/main/java/datawave/ingest/metadata/MetadataWithEventDate.java new file mode 100644 index 00000000000..71d63c703ca --- /dev/null +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/metadata/MetadataWithEventDate.java @@ -0,0 +1,74 @@ +package datawave.ingest.metadata; + +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +import org.apache.hadoop.io.Text; + +public class MetadataWithEventDate { + + private final Text columnFamily; + private final Set entries = new HashSet<>(); + + public MetadataWithEventDate(Text columnFamily) { + this.columnFamily = columnFamily; + } + + public Text getColumnFamily() { + return columnFamily; + } + + public void put(String fieldName, String dataTypeName, long date) { + entries.add(new Components(fieldName, dataTypeName, date)); + } + + public void clear() { + entries.clear(); + } + + public Set getEntries() { + return entries; + } + + public static class Components { + private final String fieldName; + private final String dataType; + private final long date; + + public Components(String fieldName, String dataType, long date) { + this.fieldName = fieldName; + this.dataType = dataType; + this.date = date; + } + + public String getFieldName() { + return fieldName; + } + + public String getDataType() { + return dataType; + } + + public long getDate() { + return date; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Components that = (Components) o; + return Objects.equals(fieldName, that.fieldName) && Objects.equals(dataType, that.dataType) && Objects.equals(date, that.date); + } + + @Override + public int hashCode() { + return Objects.hash(fieldName, dataType, date); + } + } +} diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/metadata/MetadataWithMostRecentDate.java b/warehouse/ingest-core/src/main/java/datawave/ingest/metadata/MetadataWithMostRecentDate.java index 7ca146a1e92..5aaa428a8dd 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/metadata/MetadataWithMostRecentDate.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/metadata/MetadataWithMostRecentDate.java @@ -11,7 +11,7 @@ public class MetadataWithMostRecentDate { public static final String IGNORED_NORMALIZER_CLASS = null; private final Text columnFamily; - private Map mostRecentDates = new HashMap<>(); + private Map mostRecentDates = new HashMap<>(); public MetadataWithMostRecentDate(Text columnFamily) { this.columnFamily = columnFamily; @@ -27,15 +27,15 @@ private static String createKey(String fieldName, String dataTypeOutputName, Str public void createOrUpdate(String fieldName, String dataTypeOutputName, String normalizerClassName, long eventDate) { String identifier = createKey(fieldName, dataTypeOutputName, normalizerClassName); - MostRecentEventDateAndKeyComponents value = mostRecentDates.get(identifier); + Components value = mostRecentDates.get(identifier); if (null == value) { - mostRecentDates.put(identifier, new MostRecentEventDateAndKeyComponents(fieldName, dataTypeOutputName, normalizerClassName, eventDate)); + mostRecentDates.put(identifier, new Components(fieldName, dataTypeOutputName, normalizerClassName, eventDate)); } else if (eventDate > value.getMostRecentDate()) { value.setMostRecentDate(eventDate); } } - public Collection entries() { + public Collection entries() { return mostRecentDates.values(); } @@ -47,13 +47,13 @@ public Text getColumnFamily() { return columnFamily; } - class MostRecentEventDateAndKeyComponents { + static class Components { private final String fieldName; private final String dataTypeOutputName; private final String normalizerClassName; private long mostRecentDate; - public MostRecentEventDateAndKeyComponents(String fieldName, String dataTypeOutputName, String normalizerClassName, long eventDate) { + public Components(String fieldName, String dataTypeOutputName, String normalizerClassName, long eventDate) { this.fieldName = fieldName; this.dataTypeOutputName = dataTypeOutputName; this.normalizerClassName = normalizerClassName; @@ -89,10 +89,10 @@ public int hashCode() { @Override public boolean equals(Object o) { - if (!(o instanceof MostRecentEventDateAndKeyComponents)) { + if (!(o instanceof Components)) { return false; } - MostRecentEventDateAndKeyComponents other = (MostRecentEventDateAndKeyComponents) o; + Components other = (Components) o; return Objects.equal(dataTypeOutputName, other.dataTypeOutputName) && Objects.equal(fieldName, other.fieldName) && Objects.equal(normalizerClassName, other.normalizerClassName) && this.mostRecentDate == other.mostRecentDate; } diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/MetadataTableConfigHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/MetadataTableConfigHelper.java index ca41fdd7592..25e409421cc 100644 --- a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/MetadataTableConfigHelper.java +++ b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/MetadataTableConfigHelper.java @@ -24,6 +24,8 @@ public void configure(TableOperations tops) throws AccumuloException, AccumuloSe if (tableName != null) { for (IteratorScope scope : IteratorScope.values()) { setFrequencyCombiner(tops, scope.name()); + setIndexCombiner(tops, scope.name()); + setReverseIndexCombiner(tops, scope.name()); setCombinerForCountMetadata(tops, scope.name()); setCombinerForEdgeMetadata(tops, scope.name()); } @@ -31,7 +33,7 @@ public void configure(TableOperations tops) throws AccumuloException, AccumuloSe } - // add the EdgeMetadataCombiner to the edge column + // Add the EdgeMetadataCombiner to the edge column. private String setCombinerForEdgeMetadata(TableOperations tops, String scopeName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { String stem = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scopeName, "EdgeMetadataCombiner"); @@ -40,7 +42,7 @@ private String setCombinerForEdgeMetadata(TableOperations tops, String scopeName return stem; } - // add the CountMetadataCombiner to the count column + // Add the CountMetadataCombiner to the count column. private String setCombinerForCountMetadata(TableOperations tops, String scopeName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { String stem = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scopeName, "CountMetadataCombiner"); @@ -49,7 +51,7 @@ private String setCombinerForCountMetadata(TableOperations tops, String scopeNam return stem; } - // add the EdgeMetadataCombiner to the edge column + // Add the SummingCombiner to the frequency column. private String setFrequencyCombiner(TableOperations tops, String scopeName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { String stem = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scopeName, "FrequencyCombiner"); setPropertyIfNecessary(tableName, stem, "10," + SummingCombiner.class.getName(), tops, log); @@ -58,6 +60,26 @@ private String setFrequencyCombiner(TableOperations tops, String scopeName) thro return stem; } + // Add the SummingCombiner to the indexed column. + private String setIndexCombiner(TableOperations tops, String scopeName) throws AccumuloException, TableNotFoundException, AccumuloSecurityException { + String stem = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scopeName, "IndexCombiner"); + setPropertyIfNecessary(tableName, stem, "11," + SummingCombiner.class.getName(), tops, log); + setPropertyIfNecessary(tableName, stem + ".opt.columns", ColumnFamilyConstants.COLF_I.toString(), tops, log); + setPropertyIfNecessary(tableName, stem + ".opt.lossy", "true", tops, log); + setPropertyIfNecessary(tableName, stem + ".opt.type", "VARLEN", tops, log); + return stem; + } + + // Add the SummingCombiner to the reverse indexed column. + private String setReverseIndexCombiner(TableOperations tops, String scopeName) throws AccumuloException, TableNotFoundException, AccumuloSecurityException { + String stem = String.format("%s%s.%s", Property.TABLE_ITERATOR_PREFIX, scopeName, "ReverseIndexCombiner"); + setPropertyIfNecessary(tableName, stem, "12," + SummingCombiner.class.getName(), tops, log); + setPropertyIfNecessary(tableName, stem + ".opt.columns", ColumnFamilyConstants.COLF_RI.toString(), tops, log); + setPropertyIfNecessary(tableName, stem + ".opt.lossy", "true", tops, log); + setPropertyIfNecessary(tableName, stem + ".opt.type", "VARLEN", tops, log); + return stem; + } + @Override public void setup(String tableName, Configuration config, Logger log) throws IllegalArgumentException { diff --git a/warehouse/ingest-core/src/test/java/datawave/TestBaseIngestHelper.java b/warehouse/ingest-core/src/test/java/datawave/TestBaseIngestHelper.java index 0fcf52ebe0e..b42288402ab 100644 --- a/warehouse/ingest-core/src/test/java/datawave/TestBaseIngestHelper.java +++ b/warehouse/ingest-core/src/test/java/datawave/TestBaseIngestHelper.java @@ -7,6 +7,7 @@ import datawave.ingest.data.config.ingest.BaseIngestHelper; public class TestBaseIngestHelper extends BaseIngestHelper { + private final Multimap eventFields; /** diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/metadata/EventMetadataTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/metadata/EventMetadataTest.java index cd5dea61b5d..3cdebccb3fd 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/metadata/EventMetadataTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/metadata/EventMetadataTest.java @@ -1,331 +1,556 @@ package datawave.ingest.metadata; -import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Collection; -import java.util.Date; -import java.util.Iterator; +import static datawave.ingest.mapreduce.handler.DataTypeHandler.NULL_VALUE; +import static datawave.ingest.metadata.RawRecordMetadata.DELIMITER; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.niceMock; +import static org.easymock.EasyMock.replay; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.HashSet; import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.PartialKey; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.user.SummingCombiner; import org.apache.hadoop.io.Text; -import org.easymock.EasyMock; +import org.junit.After; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; +import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.HashMultimap; import com.google.common.collect.Multimap; import datawave.IdentityDataType; import datawave.TestAbstractContentIngestHelper; import datawave.TestBaseIngestHelper; +import datawave.data.type.LcNoDiacriticsType; +import datawave.data.type.NumberType; import datawave.ingest.data.RawRecordContainer; import datawave.ingest.data.Type; import datawave.ingest.data.config.NormalizedContentInterface; import datawave.ingest.data.config.NormalizedFieldAndValue; -import datawave.ingest.data.config.ingest.BaseIngestHelper; +import datawave.ingest.data.config.ingest.IngestHelperInterface; import datawave.ingest.mapreduce.job.BulkIngestKey; -// currently this only tests LoadDate counts public class EventMetadataTest { - private static final String FIELD_NAME = "FIELD_NAME"; - private static final String FIELD_TO_COUNT = "FIELD_1"; - private static final String FIELD_WITH_TOKEN_TO_COUNT = "FIELD_1_TOKEN"; - private static final String DATA_TYPE = "xyzabc"; - private static final String FIELD_NAME_FOR_LOAD_DATE = "LOAD_DATE"; - private static final String TF = "tf"; - private static final String T = "t"; - private static final String E = "e"; - private static final String I = "i"; - private static final String VALUE_FOR_LOAD_DATE = "20140404"; - private static final String NUMBER_TYPE = "datawave.data.type.NumberType"; - private static final Text METADATA_TABLE_NAME = new Text("table123456"); - private static final Text LOADDATES_TABLE_NAME = new Text("loaddates"); - private static final Text INDEX_TABLE_NAME = new Text("index"); - private static final Text RINDEX_TABLE_NAME = new Text("reverseIndex"); - private BaseIngestHelper helper; - private RawRecordContainer event = EasyMock.createMock(RawRecordContainer.class); - private SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyyMMdd"); - - @Before - public void setupIngestHelper() { - helper = new TestBaseIngestHelper(createEventFields()) { - @Override - public List> getDataTypes(String fieldName) { - return Arrays.asList(new IdentityDataType()); - } - }; - } - - public void setupAbstractContentIngestHelper(boolean tokenIndex, boolean listIndex) { - helper = new TestAbstractContentIngestHelper(createEventFields()) { - - private boolean isTokenIndex = tokenIndex; - private boolean isListIndex = listIndex; - - @Override - public boolean isDataTypeField(String fieldname) { - return true; - } - - @Override - public List> getDataTypes(String fieldName) { - datawave.data.type.Type type[] = {datawave.data.type.Type.Factory.createType(NUMBER_TYPE)}; - return Arrays.asList(type); - } - @Override - public boolean isContentIndexField(String field) { - return isTokenIndex; - } - - @Override - public boolean isIndexListField(String field) { - return isListIndex; - }; - - }; + private static final Text SHARD_TABLE_NAME = new Text("shard"); + private static final Text SHARD_INDEX_TABLE_NAME = new Text("shardIndex"); + private static final Text SHARD_REVERSE_INDEX_TABLE_NAME = new Text("shardReverseIndex"); + private static final Text METADATA_TABLE_NAME = new Text("metadata"); + private static final Text LOAD_DATE_TABLE_NAME = new Text("loadDate"); + private static final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyyMMdd", Locale.US); + + private final Multimap fieldValues = HashMultimap.create(); + private EventMetadata eventMetadata; + private Multimap bulkMetadata; + + @After + public void tearDown() throws Exception { + this.fieldValues.clear(); + this.eventMetadata = null; + this.bulkMetadata = null; } + /** + * Test ingesting a single event. + */ @Test - public void testCreatesIndexedField() throws IOException { - setupMocks(); - - helper.addIndexedField(FIELD_TO_COUNT); - RawRecordMetadata eventMetadata = new EventMetadata(null, METADATA_TABLE_NAME, LOADDATES_TABLE_NAME, INDEX_TABLE_NAME, RINDEX_TABLE_NAME, true); - eventMetadata.addEvent(helper, event, createEventFields(), getLoadDateAsMillis()); - - assertFieldNameCountEquals(1L, INDEX_TABLE_NAME, FIELD_TO_COUNT, eventMetadata); - Assert.assertFalse(assertContainsKey(eventMetadata, RINDEX_TABLE_NAME, FIELD_TO_COUNT)); - assertNonIndexedFieldNameIsMissing(eventMetadata); - - Assert.assertTrue(assertEExists(FIELD_TO_COUNT, eventMetadata)); - Assert.assertTrue(assertIExists(FIELD_TO_COUNT, eventMetadata)); - - EasyMock.verify(event); + public void testSingleEvent() { + // Configure the field values. + givenFieldValue("FIELD_1", "HEY HO HEY HO"); + long loadDate = getMillis("20140404"); + givenFieldValue("LOAD_DATE", String.valueOf(loadDate)); + + // Configure the helper interface. + IngestHelper helper = createIngestHelper(); + helper.addDataType("FIELD_1", new IdentityDataType()); + + // Init the event metadata and add the event. + initEventMetadata(); + long eventDate = getMillis("20140402"); + eventMetadata.addEvent(helper, createMockEvent("xyzabc", eventDate, helper), fieldValues, loadDate); + + // Validate the resulting bulk entries. + collectBulkEntries(); + assertTotalBulkEntries(4); + assertContainsMetadataTableEntry("FIELD_1", "e", "xyzabc", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("FIELD_1", "f", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(1L)); + assertContainsMetadataTableEntry("LOAD_DATE", "e", "xyzabc", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("LOAD_DATE", "f", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(1L)); } + /** + * Test ingesting an event for an excluded field. + */ @Test - public void testCreatesTermFrequencyForTokenization() throws IOException { - setupAbstractContentIngestHelper(true, false); - setupMocks(); - - helper.addIndexedField(FIELD_TO_COUNT); - RawRecordMetadata eventMetadata = new EventMetadata(null, METADATA_TABLE_NAME, LOADDATES_TABLE_NAME, INDEX_TABLE_NAME, RINDEX_TABLE_NAME, true); - eventMetadata.addEvent(helper, event, createEventFields(), getLoadDateAsMillis()); - - assertFieldNameCountEquals(1L, INDEX_TABLE_NAME, FIELD_TO_COUNT, eventMetadata); - assertFieldNameCountEquals(1L, INDEX_TABLE_NAME, FIELD_WITH_TOKEN_TO_COUNT, eventMetadata); - assertTfCountEquals(0L, FIELD_WITH_TOKEN_TO_COUNT, eventMetadata); - Assert.assertTrue(assertTExists(FIELD_WITH_TOKEN_TO_COUNT, eventMetadata)); - Assert.assertFalse(assertContainsKey(eventMetadata, RINDEX_TABLE_NAME, FIELD_TO_COUNT)); - assertNonIndexedFieldNameIsMissing(eventMetadata); - - EasyMock.verify(event); + public void testExcludedField() { + // Configure the field values. + givenFieldValue("FIELD_1", "HEY HO HEY HO"); + long loadDate = getMillis("20140404"); + givenFieldValue("LOAD_DATE", String.valueOf(loadDate)); + + // Configure the helper interface. + IngestHelper helper = createIngestHelper(); + helper.addDataType("FIELD_1", new IdentityDataType()); + + // Mark the field as excluded. + helper.addShardExclusionField("FIELD_1"); + + // Init the event metadata and add the event. + initEventMetadata(); + long eventDate = getMillis("20140402"); + eventMetadata.addEvent(helper, createMockEvent("xyzabc", eventDate, helper), fieldValues, loadDate); + + // Validate the resulting bulk entries. + collectBulkEntries(); + assertTotalBulkEntries(2); + assertContainsMetadataTableEntry("LOAD_DATE", "e", "xyzabc", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("LOAD_DATE", "f", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(1L)); } + /** + * Test ingesting an event for an indexed field. + */ @Test - public void testCreatesTermFrequencyForLists() throws IOException { - setupAbstractContentIngestHelper(false, true); - setupMocks(); - - RawRecordMetadata eventMetadata = new EventMetadata(null, METADATA_TABLE_NAME, LOADDATES_TABLE_NAME, INDEX_TABLE_NAME, RINDEX_TABLE_NAME, true); - eventMetadata.addEvent(helper, event, createEventFields(), getLoadDateAsMillis()); - - assertFieldNameCountEquals(1L, INDEX_TABLE_NAME, FIELD_TO_COUNT, eventMetadata); - Assert.assertFalse(assertContainsKey(eventMetadata, INDEX_TABLE_NAME, FIELD_WITH_TOKEN_TO_COUNT)); - assertTfCountEquals(0L, FIELD_TO_COUNT, eventMetadata); - assertTCountEquals(0L, FIELD_TO_COUNT, eventMetadata); - Assert.assertFalse(assertContainsKey(eventMetadata, RINDEX_TABLE_NAME, FIELD_TO_COUNT)); - assertNonIndexedFieldNameIsMissing(eventMetadata); - - EasyMock.verify(event); + public void testIndexedField() { + // Configure the field values. + givenFieldValue("FIELD_1", "HEY HO HEY HO"); + long loadDate = getMillis("20140404"); + givenFieldValue("LOAD_DATE", String.valueOf(loadDate)); + + // Configure the helper interface. + IngestHelper helper = createIngestHelper(); + helper.addDataType("FIELD_1", new IdentityDataType()); + + // Mark the field as indexed. + helper.addIndexedField("FIELD_1"); + + // Init the event metadata and add the event. + initEventMetadata(); + long eventDate = getMillis("20140402"); + eventMetadata.addEvent(helper, createMockEvent("xyzabc", eventDate, helper), fieldValues, loadDate); + eventMetadata.addEvent(helper, createMockEvent("xyzabc", eventDate, helper), fieldValues, loadDate); + + // Validate the resulting bulk entries. + collectBulkEntries(); + assertTotalBulkEntries(7); + assertContainsMetadataTableEntry("FIELD_1", "e", "xyzabc", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("FIELD_1", "i", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(2L)); + assertContainsMetadataTableEntry("FIELD_1", "f", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(2L)); + assertContainsMetadataTableEntry("FIELD_1", "t", "xyzabc" + DELIMITER + "datawave.IdentityDataType", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("LOAD_DATE", "e", "xyzabc", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("LOAD_DATE", "f", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(2L)); + assertContainsLoadDateTableEntry("FIELD_1", "FIELD_NAME" + DELIMITER + "shardIndex", "20140404" + DELIMITER + "xyzabc", encodeCount(2L)); } + /** + * Test ingesting an event for a reverse indexed field. + */ @Test - public void testCreatesReverseIndexedField() throws IOException { - setupMocks(); - - helper.addReverseIndexedField(FIELD_TO_COUNT); - RawRecordMetadata eventMetadata = new EventMetadata(null, METADATA_TABLE_NAME, LOADDATES_TABLE_NAME, INDEX_TABLE_NAME, RINDEX_TABLE_NAME, true); - eventMetadata.addEvent(helper, event, createEventFields(), getLoadDateAsMillis()); - - assertFieldNameCountEquals(1L, RINDEX_TABLE_NAME, FIELD_TO_COUNT, eventMetadata); - Assert.assertFalse(assertContainsKey(eventMetadata, INDEX_TABLE_NAME, FIELD_TO_COUNT)); - assertNonIndexedFieldNameIsMissing(eventMetadata); - - EasyMock.verify(event); + public void testReverseIndexedField() { + // Configure the field values. + givenFieldValue("FIELD_1", "HEY HO HEY HO"); + long loadDate = getMillis("20140404"); + givenFieldValue("LOAD_DATE", String.valueOf(loadDate)); + + // Configure the helper interface. + IngestHelper helper = createIngestHelper(); + helper.addDataType("FIELD_1", new IdentityDataType()); + + // Mark the field as reverse indexed. + helper.addReverseIndexedField("FIELD_1"); + + // Init the event metadata and add the event. + initEventMetadata(); + long eventDate = getMillis("20140402"); + eventMetadata.addEvent(helper, createMockEvent("xyzabc", eventDate, helper), fieldValues, loadDate); + eventMetadata.addEvent(helper, createMockEvent("xyzabc", eventDate, helper), fieldValues, loadDate); + + // Validate the resulting bulk entries. + collectBulkEntries(); + assertTotalBulkEntries(7); + assertContainsMetadataTableEntry("FIELD_1", "e", "xyzabc", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("FIELD_1", "ri", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(2L)); + assertContainsMetadataTableEntry("FIELD_1", "f", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(2L)); + assertContainsMetadataTableEntry("FIELD_1", "t", "xyzabc" + DELIMITER + "datawave.IdentityDataType", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("LOAD_DATE", "e", "xyzabc", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("LOAD_DATE", "f", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(2L)); + assertContainsLoadDateTableEntry("FIELD_1", "FIELD_NAME" + DELIMITER + "shardReverseIndex", "20140404" + DELIMITER + "xyzabc", encodeCount(2L)); } + /** + * Test ingesting an event for a field that is both indexed and reverse indexed. + */ @Test - public void testCreatesBoth() throws IOException { - setupMocks(); - - helper.addReverseIndexedField(FIELD_TO_COUNT); - helper.addIndexedField(FIELD_TO_COUNT); - - RawRecordMetadata eventMetadata = new EventMetadata(null, METADATA_TABLE_NAME, LOADDATES_TABLE_NAME, INDEX_TABLE_NAME, RINDEX_TABLE_NAME, true); - eventMetadata.addEvent(helper, event, createEventFields(), getLoadDateAsMillis()); - - assertFieldNameCountEquals(1L, INDEX_TABLE_NAME, FIELD_TO_COUNT, eventMetadata); - assertFieldNameCountEquals(1L, RINDEX_TABLE_NAME, FIELD_TO_COUNT, eventMetadata); - - assertNonIndexedFieldNameIsMissing(eventMetadata); - - EasyMock.verify(event); + public void testIndexedAndReverseIndexedField() { + // Configure the field values. + givenFieldValue("FIELD_1", "HEY HO HEY HO"); + long loadDate = getMillis("20140404"); + givenFieldValue("LOAD_DATE", String.valueOf(loadDate)); + + // Configure the helper interface. + IngestHelper helper = createIngestHelper(); + helper.addDataType("FIELD_1", new IdentityDataType()); + + // Mark the field as indexed and reverse indexed. + helper.addIndexedField("FIELD_1"); + helper.addReverseIndexedField("FIELD_1"); + + // Init the event metadata and add the event. + initEventMetadata(); + long eventDate = getMillis("20140402"); + eventMetadata.addEvent(helper, createMockEvent("xyzabc", eventDate, helper), fieldValues, loadDate); + eventMetadata.addEvent(helper, createMockEvent("xyzabc", eventDate, helper), fieldValues, loadDate); + + // Validate the resulting bulk entries. + collectBulkEntries(); + assertTotalBulkEntries(9); + assertContainsMetadataTableEntry("FIELD_1", "e", "xyzabc", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("FIELD_1", "ri", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(2L)); + assertContainsMetadataTableEntry("FIELD_1", "i", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(2L)); + assertContainsMetadataTableEntry("FIELD_1", "f", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(2L)); + assertContainsMetadataTableEntry("FIELD_1", "t", "xyzabc" + DELIMITER + "datawave.IdentityDataType", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("LOAD_DATE", "e", "xyzabc", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("LOAD_DATE", "f", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(2L)); + assertContainsLoadDateTableEntry("FIELD_1", "FIELD_NAME" + DELIMITER + "shardIndex", "20140404" + DELIMITER + "xyzabc", encodeCount(2L)); + assertContainsLoadDateTableEntry("FIELD_1", "FIELD_NAME" + DELIMITER + "shardReverseIndex", "20140404" + DELIMITER + "xyzabc", encodeCount(2L)); } + /** + * Test ingesting multiple events that have the same load date and event date. + */ @Test - public void testCountsTwo() throws IOException { - setupMocks(); + public void testMultipleEventsForSameLoadDateAndEventDate() { + // Configure the field values. + givenFieldValue("FIELD_1", "HEY HO HEY HO"); + long loadDate = getMillis("20140404"); + givenFieldValue("LOAD_DATE", String.valueOf(loadDate)); + + // Configure the helper interface. + IngestHelper helper = createIngestHelper(); + helper.addDataType("FIELD_1", new IdentityDataType()); + + // Init the event metadata and add the event. + initEventMetadata(); + long eventDate = getMillis("20140402"); + eventMetadata.addEvent(helper, createMockEvent("xyzabc", eventDate, helper), fieldValues, loadDate); + eventMetadata.addEvent(helper, createMockEvent("xyzabc", eventDate, helper), fieldValues, loadDate); + + // Validate the resulting bulk entries. + collectBulkEntries(); + assertTotalBulkEntries(4); + assertContainsMetadataTableEntry("FIELD_1", "e", "xyzabc", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("FIELD_1", "f", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(2L)); + assertContainsMetadataTableEntry("LOAD_DATE", "e", "xyzabc", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("LOAD_DATE", "f", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(2L)); + } - helper.addIndexedField(FIELD_TO_COUNT); - RawRecordMetadata eventMetadata = new EventMetadata(null, METADATA_TABLE_NAME, LOADDATES_TABLE_NAME, INDEX_TABLE_NAME, RINDEX_TABLE_NAME, true); - eventMetadata.addEvent(helper, event, createEventFields(), getLoadDateAsMillis()); - eventMetadata.addEvent(helper, event, createEventFields(), getLoadDateAsMillis()); + /** + * Test ingesting multiple events that have the same load date, but different event dates. + */ + @Test + public void testMultipleEventsForSameLoadDateAndDifferentEventDate() { + // Configure the field values. + givenFieldValue("FIELD_1", "HEY HO HEY HO"); + long loadDate = getMillis("20140404"); + givenFieldValue("LOAD_DATE", String.valueOf(loadDate)); + + // Configure the helper interface. + IngestHelper helper = createIngestHelper(); + helper.addDataType("FIELD_1", new IdentityDataType()); + + // Init the event metadata and add the event. + initEventMetadata(); + long eventDate1 = getMillis("20140402"); + eventMetadata.addEvent(helper, createMockEvent("xyzabc", eventDate1, helper), fieldValues, loadDate); + long eventDate2 = getMillis("20140319"); + eventMetadata.addEvent(helper, createMockEvent("xyzabc", eventDate2, helper), fieldValues, loadDate); + + // Validate the resulting bulk entries. + collectBulkEntries(); + assertTotalBulkEntries(6); + assertContainsMetadataTableEntry("FIELD_1", "e", "xyzabc", eventDate1, NULL_VALUE); + assertContainsMetadataTableEntry("FIELD_1", "f", "xyzabc" + DELIMITER + "20140319", eventDate2, encodeCount(1L)); + assertContainsMetadataTableEntry("FIELD_1", "f", "xyzabc" + DELIMITER + "20140402", eventDate1, encodeCount(1L)); + assertContainsMetadataTableEntry("LOAD_DATE", "e", "xyzabc", eventDate1, NULL_VALUE); + assertContainsMetadataTableEntry("LOAD_DATE", "f", "xyzabc" + DELIMITER + "20140319", eventDate2, encodeCount(1L)); + assertContainsMetadataTableEntry("LOAD_DATE", "f", "xyzabc" + DELIMITER + "20140402", eventDate1, encodeCount(1L)); + } - assertFieldNameCountEquals(2L, INDEX_TABLE_NAME, FIELD_TO_COUNT, eventMetadata); + /** + * Test ingesting an event for a field that should be targeted for tokenization. + */ + @Test + public void testTermFrequencyForTokenization() { + // Configure the field values. + givenFieldValue("FIELD_1", "HEY HO HEY HO"); + long loadDate = getMillis("20140404"); + givenFieldValue("LOAD_DATE", String.valueOf(loadDate)); + + // Configure the helper interface. + ContentIngestHelper helper = createContentIngestHelper(); + helper.addDataType("FIELD_1", new IdentityDataType()); + + // Mark the field as a content index field. + helper.addContentIndexField("FIELD_1"); + + // Init the event metadata and add the event. + initEventMetadata(); + long eventDate = getMillis("20140402"); + eventMetadata.addEvent(helper, createMockEvent("xyzabc", eventDate, helper), fieldValues, loadDate); + + // Validate the resulting bulk entries. + collectBulkEntries(); + assertTotalBulkEntries(8); + assertContainsMetadataTableEntry("FIELD_1", "e", "xyzabc", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("FIELD_1", "f", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(1L)); + assertContainsMetadataTableEntry("FIELD_1_TOKEN", "t", "xyzabc" + DELIMITER + "datawave.IdentityDataType", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("FIELD_1_TOKEN", "tf", "xyzabc", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("FIELD_1_TOKEN", "i", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(1L)); + assertContainsMetadataTableEntry("LOAD_DATE", "e", "xyzabc", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("LOAD_DATE", "f", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(1L)); + assertContainsLoadDateTableEntry("FIELD_1_TOKEN", "FIELD_NAME" + DELIMITER + "shardIndex", "20140404" + DELIMITER + "xyzabc", encodeCount(1L)); + } - EasyMock.verify(event); + /** + * Test ingesting an event for a field that should be targeted for term frequencies. + */ + @Test + public void testTermFrequencyForLists() { + // Configure the field values. + givenFieldValue("FIELD_1", "HEY HO HEY HO"); + long loadDate = getMillis("20140404"); + givenFieldValue("LOAD_DATE", String.valueOf(loadDate)); + + // Configure the helper interface. + ContentIngestHelper helper = createContentIngestHelper(); + helper.addDataType("FIELD_1", new IdentityDataType()); + + // Mark the field as an index list field. + helper.addIndexListField("FIELD_1"); + + // Init the event metadata and add the event. + initEventMetadata(); + long eventDate = getMillis("20140402"); + eventMetadata.addEvent(helper, createMockEvent("xyzabc", eventDate, helper), fieldValues, loadDate); + + // Validate the resulting bulk entries. + collectBulkEntries(); + assertTotalBulkEntries(8); + assertContainsMetadataTableEntry("FIELD_1", "e", "xyzabc", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("FIELD_1", "f", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(1L)); + assertContainsMetadataTableEntry("FIELD_1", "t", "xyzabc" + DELIMITER + "datawave.IdentityDataType", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("FIELD_1", "tf", "xyzabc", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("FIELD_1", "i", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(1L)); + assertContainsMetadataTableEntry("LOAD_DATE", "e", "xyzabc", eventDate, NULL_VALUE); + assertContainsMetadataTableEntry("LOAD_DATE", "f", "xyzabc" + DELIMITER + "20140402", eventDate, encodeCount(1L)); + assertContainsLoadDateTableEntry("FIELD_1", "FIELD_NAME" + DELIMITER + "shardIndex", "20140404" + DELIMITER + "xyzabc", encodeCount(1L)); } + /** + * Test ingesting an event for a field that should be targeted for term frequencies. + */ @Test - public void testWithExclusions() throws IOException { - setupMocks(); - helper.addShardExclusionField(FIELD_TO_COUNT); - RawRecordMetadata eventMetadata = new EventMetadata(null, METADATA_TABLE_NAME, LOADDATES_TABLE_NAME, INDEX_TABLE_NAME, RINDEX_TABLE_NAME, true); - eventMetadata.addEvent(helper, event, createEventFields(), getLoadDateAsMillis()); + public void testMultipleFieldEvents() { + // Configure the field values. + givenFieldValue("FIELD_1", "HEY HO HEY HO"); + givenFieldValue("FIELD_2", "Follow the yellow brick road"); + givenFieldValue("FIELD_3", "542643"); + + long loadDate = getMillis("20140404"); + givenFieldValue("LOAD_DATE", String.valueOf(loadDate)); + + // Configure the helper interface. + IngestHelper helper = createIngestHelper(); + helper.addDataType("FIELD_1", new IdentityDataType()); + helper.addDataType("FIELD_2", new LcNoDiacriticsType()); + helper.addDataType("FIELD_3", new NumberType()); + + // Init the event metadata and add the event. + initEventMetadata(); + long eventDate1 = getMillis("20140402"); + eventMetadata.addEvent(helper, createMockEvent("xyzabc", eventDate1, helper), fieldValues, loadDate); + eventMetadata.addEvent(helper, createMockEvent("xyzabc", eventDate1, helper), fieldValues, loadDate); + eventMetadata.addEvent(helper, createMockEvent("ababa", eventDate1, helper), fieldValues, loadDate); + + long eventDate2 = getMillis("20140403"); + eventMetadata.addEvent(helper, createMockEvent("ididi", eventDate2, helper), fieldValues, loadDate); + + // Validate the resulting bulk entries. + collectBulkEntries(); + assertTotalBulkEntries(24); + assertContainsMetadataTableEntry("FIELD_1", "e", "xyzabc", eventDate1, NULL_VALUE); + assertContainsMetadataTableEntry("FIELD_1", "f", "xyzabc" + DELIMITER + "20140402", eventDate1, encodeCount(2L)); + assertContainsMetadataTableEntry("FIELD_1", "f", "ababa" + DELIMITER + "20140402", eventDate1, encodeCount(1L)); + assertContainsMetadataTableEntry("FIELD_1", "e", "ababa", eventDate1, NULL_VALUE); + assertContainsMetadataTableEntry("FIELD_1", "f", "ididi" + DELIMITER + "20140403", eventDate2, encodeCount(1L)); + assertContainsMetadataTableEntry("FIELD_1", "e", "ididi", eventDate2, NULL_VALUE); + assertContainsMetadataTableEntry("FIELD_2", "e", "xyzabc", eventDate1, NULL_VALUE); + assertContainsMetadataTableEntry("FIELD_2", "f", "xyzabc" + DELIMITER + "20140402", eventDate1, encodeCount(2L)); + assertContainsMetadataTableEntry("FIELD_2", "f", "ababa" + DELIMITER + "20140402", eventDate1, encodeCount(1L)); + assertContainsMetadataTableEntry("FIELD_2", "e", "ababa", eventDate1, NULL_VALUE); + assertContainsMetadataTableEntry("FIELD_2", "f", "ididi" + DELIMITER + "20140403", eventDate2, encodeCount(1L)); + assertContainsMetadataTableEntry("FIELD_2", "e", "ididi", eventDate2, NULL_VALUE); + assertContainsMetadataTableEntry("FIELD_3", "e", "xyzabc", eventDate1, NULL_VALUE); + assertContainsMetadataTableEntry("FIELD_3", "f", "xyzabc" + DELIMITER + "20140402", eventDate1, encodeCount(2L)); + assertContainsMetadataTableEntry("FIELD_3", "f", "ababa" + DELIMITER + "20140402", eventDate1, encodeCount(1L)); + assertContainsMetadataTableEntry("FIELD_3", "e", "ababa", eventDate1, NULL_VALUE); + assertContainsMetadataTableEntry("FIELD_3", "f", "ididi" + DELIMITER + "20140403", eventDate2, encodeCount(1L)); + assertContainsMetadataTableEntry("FIELD_3", "e", "ididi", eventDate2, NULL_VALUE); + assertContainsMetadataTableEntry("LOAD_DATE", "f", "xyzabc" + DELIMITER + "20140402", eventDate1, encodeCount(2L)); + assertContainsMetadataTableEntry("LOAD_DATE", "e", "xyzabc", eventDate1, NULL_VALUE); + assertContainsMetadataTableEntry("LOAD_DATE", "f", "ababa" + DELIMITER + "20140402", eventDate1, encodeCount(1L)); + assertContainsMetadataTableEntry("LOAD_DATE", "e", "ababa", eventDate1, NULL_VALUE); + assertContainsMetadataTableEntry("LOAD_DATE", "f", "ididi" + DELIMITER + "20140403", eventDate2, encodeCount(1L)); + assertContainsMetadataTableEntry("LOAD_DATE", "e", "ididi", eventDate2, NULL_VALUE); + } - Assert.assertFalse(assertEExists(FIELD_TO_COUNT, eventMetadata)); + // Return the given date as millis. + private long getMillis(String date) { + return LocalDate.parse(date, dateTimeFormatter).atStartOfDay(ZoneOffset.UTC).toInstant().toEpochMilli(); + } - EasyMock.verify(event); + // Return a new mock event. + private RawRecordContainer createMockEvent(String type, long date, IngestHelperInterface helper) { + RawRecordContainer event = niceMock(RawRecordContainer.class); + expect(event.getDataType()).andReturn(new Type(type, helper.getClass(), null, null, 4, null)).anyTimes(); + expect(event.getDate()).andReturn(date).anyTimes(); + replay(event); + return event; } - private void assertFieldNameCountEquals(long expectedCount, Text tableName, String fieldName, RawRecordMetadata eventMetadata) { - Text expectedColumnFamily = new Text(FIELD_NAME + RawRecordMetadata.DELIMITER + tableName); - Text expectedColumnQualifier = new Text(VALUE_FOR_LOAD_DATE + RawRecordMetadata.DELIMITER + DATA_TYPE); - assertCountEquals(expectedCount, fieldName, eventMetadata, LOADDATES_TABLE_NAME, expectedColumnFamily, expectedColumnQualifier); + // Add the given field and value to the field values. + private void givenFieldValue(String field, String value) { + this.fieldValues.put(field, new NormalizedFieldAndValue(field, value.getBytes())); } - private boolean assertEExists(String fieldName, RawRecordMetadata eventMetadata) { - Text expectedColumnFamily = new Text(E); - Text expectedColumnQualifier = new Text(DATA_TYPE); - return assertExists(fieldName, eventMetadata, METADATA_TABLE_NAME, expectedColumnFamily, expectedColumnQualifier); + /** + * Initialize the EventMetadata instance. + */ + private void initEventMetadata() { + this.eventMetadata = new EventMetadata(SHARD_TABLE_NAME, METADATA_TABLE_NAME, LOAD_DATE_TABLE_NAME, SHARD_INDEX_TABLE_NAME, + SHARD_REVERSE_INDEX_TABLE_NAME, true); } - private boolean assertIExists(String fieldName, RawRecordMetadata eventMetadata) { - Text expectedColumnFamily = new Text(I); - Text expectedColumnQualifier = new Text(DATA_TYPE); - return assertExists(fieldName, eventMetadata, METADATA_TABLE_NAME, expectedColumnFamily, expectedColumnQualifier); + // Return a new IngestHelper. + private IngestHelper createIngestHelper() { + return new IngestHelper(fieldValues); } - private void assertTfCountEquals(long expectedCount, String fieldName, RawRecordMetadata eventMetadata) { - Text expectedColumnFamily = new Text(TF); - Text expectedColumnQualifier = new Text(DATA_TYPE); - assertCountEquals(expectedCount, fieldName, eventMetadata, METADATA_TABLE_NAME, expectedColumnFamily, expectedColumnQualifier); + // Return a new ContentIngestHelper. + private ContentIngestHelper createContentIngestHelper() { + return new ContentIngestHelper(fieldValues); } - private boolean assertTExists(String fieldName, RawRecordMetadata eventMetadata) { - Text expectedColumnFamily = new Text(T); - Text expectedColumnQualifier = new Text(DATA_TYPE + RawRecordMetadata.DELIMITER + NUMBER_TYPE); - return assertExists(fieldName, eventMetadata, METADATA_TABLE_NAME, expectedColumnFamily, expectedColumnQualifier); + // Collect the bulk metadata entries from the event metadata instance. + private void collectBulkEntries() { + this.bulkMetadata = eventMetadata.getBulkMetadata(); } - private void assertTCountEquals(long expectedCount, String fieldName, RawRecordMetadata eventMetadata) { - Text expectedColumnFamily = new Text(T); - Text expectedColumnQualifier = new Text(DATA_TYPE + RawRecordMetadata.DELIMITER + NUMBER_TYPE); - assertCountEquals(expectedCount, fieldName, eventMetadata, METADATA_TABLE_NAME, expectedColumnFamily, expectedColumnQualifier); + // Assert the total number of bulk metadata entries. + private void assertTotalBulkEntries(int total) { + assertEquals(total, bulkMetadata.size()); } - private void assertCountEquals(long expectedCount, String rowId, RawRecordMetadata eventMetadata, Text tableName, Text expectedColumnFamily, - Text expectedColumnQualifier) { - BulkIngestKey expectedBulkIngestKey = createExpectedBulkIngestKey(expectedColumnFamily, tableName, rowId, expectedColumnQualifier); - Assert.assertTrue("Didn't have entry for " + expectedColumnFamily + ", " + rowId, assertContainsKey(eventMetadata, expectedBulkIngestKey)); - Iterator values = getCorrespondingValue(eventMetadata, expectedBulkIngestKey).iterator(); - Assert.assertTrue(values.hasNext()); + // Assert that the collected bulk metadata entries contain a matching bulk ingest key and value. + private void assertContainsMetadataTableEntry(String field, String columnFamily, String columnQualifier, long date, Value value) { + assertTrue(bulkMetadata.containsEntry(createBulkIngestKey(METADATA_TABLE_NAME, field, columnFamily, columnQualifier, date), value)); + } - byte[] value = values.next().get(); - if (expectedCount > 0) { - Assert.assertEquals(expectedCount, (long) SummingCombiner.VAR_LEN_ENCODER.decode(value)); + // Assert that the collected bulk metadata entries contain a matching bulk ingest key and value, ignoring the timestamp. + private void assertContainsLoadDateTableEntry(String field, String columnFamily, String columnQualifier, Value value) { + Key partialKey = new Key(new Text(field), new Text(columnFamily), new Text(columnQualifier)); + for (Map.Entry entry : bulkMetadata.entries()) { + BulkIngestKey ingestKey = entry.getKey(); + if (ingestKey.getTableName().equals(LOAD_DATE_TABLE_NAME) && ingestKey.getKey().equals(partialKey, PartialKey.ROW_COLFAM_COLQUAL) + && entry.getValue().equals(value)) { + return; + } } - - Assert.assertFalse(values.hasNext()); + Assert.fail("No match found"); } - private boolean assertExists(String rowId, RawRecordMetadata eventMetadata, Text tableName, Text expectedColumnFamily, Text expectedColumnQualifier) { - BulkIngestKey expectedBulkIngestKey = createExpectedBulkIngestKey(expectedColumnFamily, tableName, rowId, expectedColumnQualifier); - return assertContainsKey(eventMetadata, expectedBulkIngestKey); + // Return a new bulk ingest key with the given attributes. + private BulkIngestKey createBulkIngestKey(Text tableName, String fieldName, String columnFamily, String columnQualifier, long timestamp) { + Key key = new Key(new Text(fieldName), new Text(columnFamily), new Text(columnQualifier), timestamp); + return new BulkIngestKey(tableName, key); } - private void assertNonIndexedFieldNameIsMissing(RawRecordMetadata eventMetadata) { - Assert.assertFalse(assertContainsKey(eventMetadata, RINDEX_TABLE_NAME, FIELD_NAME_FOR_LOAD_DATE)); - Assert.assertFalse(assertContainsKey(eventMetadata, INDEX_TABLE_NAME, FIELD_NAME_FOR_LOAD_DATE)); + // Return the given count as an encoded value. + private Value encodeCount(long count) { + return new Value(SummingCombiner.VAR_LEN_ENCODER.encode(count)); } - private boolean assertContainsKey(RawRecordMetadata eventMetadata, Text columnFamily, String fieldName) { - BulkIngestKey expectedBulkIngestKey = createExpectedBulkIngestKey(columnFamily, fieldName); - return null != getCorrespondingValue(eventMetadata, expectedBulkIngestKey); - } + /** + * Ingest helper implementation for basic testing. + */ + private static class IngestHelper extends TestBaseIngestHelper { - private boolean assertContainsKey(RawRecordMetadata eventMetadata, BulkIngestKey expectedBulkIngestKey) { - return null != getCorrespondingValue(eventMetadata, expectedBulkIngestKey); - } + private final ArrayListMultimap> dataTypes = ArrayListMultimap.create(); - private Collection getCorrespondingValue(RawRecordMetadata eventMetadata, BulkIngestKey expectedBulkIngestKey) { - Multimap bulkMetadata = eventMetadata.getBulkMetadata(); - for (BulkIngestKey actualBulkIngestKey : bulkMetadata.keySet()) { - if (actualBulkIngestKey.getTableName().equals(expectedBulkIngestKey.getTableName()) - && actualBulkIngestKey.getKey().equals(expectedBulkIngestKey.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS)) { - return bulkMetadata.get(actualBulkIngestKey); - } + public IngestHelper(Multimap fieldValues) { + super(fieldValues); } - return null; - } - private void setupMocks() { - try { - EasyMock.expect(event.getDataType()).andReturn(new Type(DATA_TYPE, helper.getClass(), null, null, 4, null)).anyTimes(); - EasyMock.expect(event.getDate()).andReturn(new Date().getTime()).anyTimes(); - EasyMock.expect(event.getRawData()).andReturn(new byte[] {}).anyTimes(); - EasyMock.replay(event); - } catch (Exception e) { - throw new RuntimeException(e); // make the test fail if the mock wasn't created properly + @Override + public List> getDataTypes(String fieldName) { + return dataTypes.get(fieldName); } - } - private BulkIngestKey createExpectedBulkIngestKey(Text columnFamily, String fieldName) { - return createExpectedBulkIngestKey(columnFamily, LOADDATES_TABLE_NAME, fieldName, - new Text(VALUE_FOR_LOAD_DATE + RawRecordMetadata.DELIMITER + DATA_TYPE)); + private void addDataType(String field, datawave.data.type.Type dataType) { + dataTypes.put(field, dataType); + } } - private BulkIngestKey createExpectedBulkIngestKey(Text columnFamily, Text tableName, String fieldName, Text columnQualifier) { - Key k = new Key(new Text(fieldName), columnFamily, columnQualifier); - return new BulkIngestKey(new Text(tableName), k); - } + /** + * Ingest helper implementation for testing term frequency row generation. + */ + private static class ContentIngestHelper extends TestAbstractContentIngestHelper { - private Multimap createEventFields() { - Multimap eventFields = HashMultimap.create(); - addToEventFields(eventFields, FIELD_NAME_FOR_LOAD_DATE, "" + getLoadDateAsMillis()); - addToEventFields(eventFields, FIELD_TO_COUNT, "HEY HO HEY HO"); - return eventFields; - } + private final ArrayListMultimap> dataTypes = ArrayListMultimap.create(); + private final Set contentIndexFields = new HashSet<>(); + private final Set indexListFields = new HashSet<>(); - private long getLoadDateAsMillis() { - try { - return simpleDateFormat.parse(VALUE_FOR_LOAD_DATE).getTime(); - } catch (ParseException e) { - throw new RuntimeException(e); + public ContentIngestHelper(Multimap fieldValues) { + super(fieldValues); + } + + @Override + public boolean isContentIndexField(String field) { + return contentIndexFields.contains(field); + } + + private void addContentIndexField(String field) { + this.contentIndexFields.add(field); } - } - private void addToEventFields(Multimap eventFields, String fieldName, String value) { - NormalizedContentInterface loadDateContent = new NormalizedFieldAndValue(fieldName, value.getBytes()); - eventFields.put(fieldName, loadDateContent); + @Override + public boolean isIndexListField(String field) { + return indexListFields.contains(field); + } + + private void addIndexListField(String field) { + this.indexListFields.add(field); + } + + @Override + public List> getDataTypes(String fieldName) { + return dataTypes.get(fieldName); + } + + private void addDataType(String field, datawave.data.type.Type dataType) { + dataTypes.put(field, dataType); + } } } diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/metadata/MetadataCounterGroupTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/metadata/MetadataCounterGroupTest.java index 0d3aa7895ba..015bfd7e709 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/metadata/MetadataCounterGroupTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/metadata/MetadataCounterGroupTest.java @@ -38,12 +38,12 @@ private void helperTestTwoDataTypes(MetadataCounterGroup counters) { Assert.assertEquals(2, counters.getEntries().size()); - Iterator iterator = counters.getEntries().iterator(); - MetadataCounterGroup.CountAndKeyComponents first = iterator.next(); + Iterator iterator = counters.getEntries().iterator(); + MetadataCounterGroup.Components first = iterator.next(); Assert.assertEquals(dataType + "2", first.getDataType()); Assert.assertEquals(2, first.getCount()); - MetadataCounterGroup.CountAndKeyComponents second = iterator.next(); + MetadataCounterGroup.Components second = iterator.next(); Assert.assertEquals(dataType, second.getDataType()); Assert.assertEquals(1, second.getCount()); } @@ -53,7 +53,7 @@ private void assertOneEntryWithExpectedCount(MetadataCounterGroup counters, int Assert.assertEquals(expectedCount, getOnlyEntry(counters).getCount()); } - private MetadataCounterGroup.CountAndKeyComponents getOnlyEntry(MetadataCounterGroup counters) { + private MetadataCounterGroup.Components getOnlyEntry(MetadataCounterGroup counters) { return counters.getEntries().iterator().next(); } @@ -78,7 +78,7 @@ public void testCountsUpAndDownCorrectly() { public void testAssignments() { MetadataCounterGroup counters = new MetadataCounterGroup(COLUMN_FAMILY); counters.addToCount(1, dataType, fieldName, date); - MetadataCounterGroup.CountAndKeyComponents entry = getOnlyEntry(counters); + MetadataCounterGroup.Components entry = getOnlyEntry(counters); Assert.assertEquals(dataType, entry.getDataType()); Assert.assertEquals(fieldName, entry.getRowId()); Assert.assertEquals(date, entry.getDate()); diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/metadata/MetadataWithMostRecentDateTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/metadata/MetadataWithMostRecentDateTest.java index 62f0cf83004..f6cf5af1621 100644 --- a/warehouse/ingest-core/src/test/java/datawave/ingest/metadata/MetadataWithMostRecentDateTest.java +++ b/warehouse/ingest-core/src/test/java/datawave/ingest/metadata/MetadataWithMostRecentDateTest.java @@ -50,7 +50,7 @@ public void testCanTrackSeparately() { expected.add(new Tuple("sham", "wow2", "sham.wow.pick.up.Messes", 124L)); Collection actual = Lists.newArrayList(); - for (MetadataWithMostRecentDate.MostRecentEventDateAndKeyComponents m : metadata.entries()) { + for (MetadataWithMostRecentDate.Components m : metadata.entries()) { actual.add(new Tuple(m.getFieldName(), m.getDataType(), m.getNormalizerClassName(), m.getMostRecentDate())); } @@ -62,7 +62,7 @@ private void assertOneEntryWithExpectedDate(MetadataWithMostRecentDate counters, Assert.assertEquals(expectedDate, getOnlyEntry(counters).getMostRecentDate()); } - private MetadataWithMostRecentDate.MostRecentEventDateAndKeyComponents getOnlyEntry(MetadataWithMostRecentDate counters) { + private MetadataWithMostRecentDate.Components getOnlyEntry(MetadataWithMostRecentDate counters) { return counters.entries().iterator().next(); } @@ -70,7 +70,7 @@ private MetadataWithMostRecentDate.MostRecentEventDateAndKeyComponents getOnlyEn public void testAssignments() { MetadataWithMostRecentDate counters = new MetadataWithMostRecentDate(COLUMN_FAMILY); counters.createOrUpdate(fieldName, dataTypeName, normalizerClassName, date); - MetadataWithMostRecentDate.MostRecentEventDateAndKeyComponents entry = getOnlyEntry(counters); + MetadataWithMostRecentDate.Components entry = getOnlyEntry(counters); Assert.assertEquals(fieldName, entry.getFieldName()); Assert.assertEquals(dataTypeName, entry.getDataType()); Assert.assertEquals(normalizerClassName, entry.getNormalizerClassName()); diff --git a/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/TableConfigurationUtilTest.java b/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/TableConfigurationUtilTest.java index a31c878f7a4..8c6afcf9d63 100644 --- a/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/TableConfigurationUtilTest.java +++ b/warehouse/ingest-csv/src/test/java/datawave/ingest/csv/TableConfigurationUtilTest.java @@ -107,7 +107,7 @@ public void testCacheFile() throws AccumuloSecurityException, AccumuloException, Assert.assertEquals(0, tempCacheFile.length()); tcu.updateCacheFile(); - Assert.assertEquals(7215, tempCacheFile.length()); + Assert.assertEquals(7868, tempCacheFile.length()); tcu.serializeTableConfgurationIntoConf(conf); @@ -271,7 +271,7 @@ private void validateTCU(TableConfigurationUtil tcu, Configuration conf) throws Assert.assertEquals(10, shardIndexProps.size()); Map metaProps = tcu.getTableProperties("datawave.metadata"); - Assert.assertEquals(15, metaProps.size()); + Assert.assertEquals(23, metaProps.size()); tcu.setTableItersPrioritiesAndOpts(); @@ -284,8 +284,10 @@ private void validateTCU(TableConfigurationUtil tcu, Configuration conf) throws Assert.assertEquals("datawave.ingest.table.aggregator.GlobalIndexUidAggregator", shardIndexAggs.get(19).get("*")); Map> metaCombiners = tcu.getTableCombiners("datawave.metadata"); - Assert.assertEquals(3, metaCombiners.size()); + Assert.assertEquals(5, metaCombiners.size()); Assert.assertEquals("org.apache.accumulo.core.iterators.user.SummingCombiner", metaCombiners.get(10).get(TableConfigurationUtil.ITERATOR_CLASS_MARKER)); + Assert.assertEquals("org.apache.accumulo.core.iterators.user.SummingCombiner", metaCombiners.get(11).get(TableConfigurationUtil.ITERATOR_CLASS_MARKER)); + Assert.assertEquals("org.apache.accumulo.core.iterators.user.SummingCombiner", metaCombiners.get(12).get(TableConfigurationUtil.ITERATOR_CLASS_MARKER)); Assert.assertEquals("datawave.iterators.CountMetadataCombiner", metaCombiners.get(15).get(TableConfigurationUtil.ITERATOR_CLASS_MARKER)); Assert.assertEquals("datawave.iterators.EdgeMetadataCombiner", metaCombiners.get(19).get(TableConfigurationUtil.ITERATOR_CLASS_MARKER)); From efb4e285b85cd59ed6d3eba0e87887a3cd6cd0e2 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Tue, 13 Feb 2024 19:45:44 +0000 Subject: [PATCH 13/50] 6.7.0-SNAPSHOT --- common-test/pom.xml | 2 +- contrib/datawave-quickstart/docker/pom.xml | 2 +- core/pom.xml | 2 +- core/utils/pom.xml | 2 +- docs/pom.xml | 2 +- microservices/pom.xml | 2 +- microservices/services/pom.xml | 2 +- microservices/starters/pom.xml | 2 +- pom.xml | 2 +- warehouse/accumulo-extensions/pom.xml | 2 +- warehouse/assemble/datawave/pom.xml | 2 +- warehouse/assemble/pom.xml | 2 +- warehouse/assemble/webservice/pom.xml | 2 +- warehouse/common/pom.xml | 2 +- warehouse/core/pom.xml | 2 +- warehouse/data-dictionary-core/pom.xml | 2 +- warehouse/edge-dictionary-core/pom.xml | 2 +- warehouse/edge-model-configuration-core/pom.xml | 2 +- warehouse/index-stats/pom.xml | 2 +- warehouse/ingest-configuration/pom.xml | 2 +- warehouse/ingest-core/pom.xml | 2 +- warehouse/ingest-csv/pom.xml | 2 +- warehouse/ingest-json/pom.xml | 2 +- warehouse/ingest-nyctlc/pom.xml | 2 +- warehouse/ingest-scripts/pom.xml | 2 +- warehouse/ingest-wikipedia/pom.xml | 2 +- warehouse/metrics-core/pom.xml | 2 +- warehouse/ops-tools/config-compare/pom.xml | 2 +- warehouse/ops-tools/index-validation/pom.xml | 2 +- warehouse/ops-tools/pom.xml | 2 +- warehouse/pom.xml | 2 +- warehouse/query-core/pom.xml | 2 +- warehouse/regression-testing/pom.xml | 2 +- warehouse/ssdeep-common/pom.xml | 2 +- web-services/accumulo/pom.xml | 2 +- web-services/atom/pom.xml | 2 +- web-services/cached-results/pom.xml | 2 +- web-services/client/pom.xml | 2 +- web-services/common-util/pom.xml | 2 +- web-services/common/pom.xml | 2 +- web-services/deploy/application/pom.xml | 2 +- web-services/deploy/configuration/pom.xml | 2 +- web-services/deploy/docs/pom.xml | 2 +- web-services/deploy/pom.xml | 2 +- web-services/deploy/spring-framework-integration/pom.xml | 2 +- web-services/dictionary/pom.xml | 2 +- web-services/examples/client-login/pom.xml | 2 +- web-services/examples/http-client/pom.xml | 2 +- web-services/examples/jms-client/pom.xml | 2 +- web-services/examples/pom.xml | 2 +- web-services/examples/query-war/pom.xml | 2 +- web-services/map-reduce-embedded/pom.xml | 2 +- web-services/map-reduce-status/pom.xml | 2 +- web-services/map-reduce/pom.xml | 2 +- web-services/model/pom.xml | 2 +- web-services/modification/pom.xml | 2 +- web-services/pom.xml | 2 +- web-services/query-websocket/pom.xml | 2 +- web-services/query/pom.xml | 2 +- web-services/rest-api/pom.xml | 2 +- web-services/security/pom.xml | 2 +- web-services/web-root/pom.xml | 2 +- 62 files changed, 62 insertions(+), 62 deletions(-) diff --git a/common-test/pom.xml b/common-test/pom.xml index 237487670ed..7d0c176560a 100644 --- a/common-test/pom.xml +++ b/common-test/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-common-test ${project.artifactId} diff --git a/contrib/datawave-quickstart/docker/pom.xml b/contrib/datawave-quickstart/docker/pom.xml index 747a69c60a0..5af989617cf 100644 --- a/contrib/datawave-quickstart/docker/pom.xml +++ b/contrib/datawave-quickstart/docker/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT ../../../pom.xml quickstart diff --git a/core/pom.xml b/core/pom.xml index 9e634e1dff8..df1b7baeee6 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT gov.nsa.datawave.core datawave-core-parent diff --git a/core/utils/pom.xml b/core/utils/pom.xml index 62b5975d230..f412ce40ea8 100644 --- a/core/utils/pom.xml +++ b/core/utils/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.core datawave-core-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT gov.nsa.datawave.core datawave-utils-parent diff --git a/docs/pom.xml b/docs/pom.xml index cee55740648..c57c473f570 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-docs diff --git a/microservices/pom.xml b/microservices/pom.xml index 7839679494a..446236890f8 100644 --- a/microservices/pom.xml +++ b/microservices/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT gov.nsa.datawave.microservice datawave-microservice-build-parent diff --git a/microservices/services/pom.xml b/microservices/services/pom.xml index 8562435cdd2..42f8ffa058d 100644 --- a/microservices/services/pom.xml +++ b/microservices/services/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-build-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-microservice-service-build-parent pom diff --git a/microservices/starters/pom.xml b/microservices/starters/pom.xml index 598c30cb136..ed43f1da156 100644 --- a/microservices/starters/pom.xml +++ b/microservices/starters/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-build-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-microservice-starter-build-parent pom diff --git a/pom.xml b/pom.xml index 8410652244d..66b51533c93 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 gov.nsa.datawave datawave-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT pom DataWave DataWave is a Java-based ingest and query framework that leverages Apache Accumulo to provide fast, secure access to your data. diff --git a/warehouse/accumulo-extensions/pom.xml b/warehouse/accumulo-extensions/pom.xml index 2daa137daab..14f445611d2 100644 --- a/warehouse/accumulo-extensions/pom.xml +++ b/warehouse/accumulo-extensions/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-accumulo-extensions ${project.artifactId} diff --git a/warehouse/assemble/datawave/pom.xml b/warehouse/assemble/datawave/pom.xml index 921d2d4c257..fb24a4345cf 100644 --- a/warehouse/assemble/datawave/pom.xml +++ b/warehouse/assemble/datawave/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave assemble-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT assemble-datawave jar diff --git a/warehouse/assemble/pom.xml b/warehouse/assemble/pom.xml index 7524a3fb304..c70a132b21c 100644 --- a/warehouse/assemble/pom.xml +++ b/warehouse/assemble/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT assemble-parent pom diff --git a/warehouse/assemble/webservice/pom.xml b/warehouse/assemble/webservice/pom.xml index 4e9639eca08..5958c14f08b 100644 --- a/warehouse/assemble/webservice/pom.xml +++ b/warehouse/assemble/webservice/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave assemble-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT assemble-webservice ${project.artifactId} diff --git a/warehouse/common/pom.xml b/warehouse/common/pom.xml index c32c9f3954a..8991e037ffb 100644 --- a/warehouse/common/pom.xml +++ b/warehouse/common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-common ${project.artifactId} diff --git a/warehouse/core/pom.xml b/warehouse/core/pom.xml index 65c214a38d6..9f37e568dbf 100644 --- a/warehouse/core/pom.xml +++ b/warehouse/core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-core jar diff --git a/warehouse/data-dictionary-core/pom.xml b/warehouse/data-dictionary-core/pom.xml index 647d9ea6817..4cf6a46adb9 100644 --- a/warehouse/data-dictionary-core/pom.xml +++ b/warehouse/data-dictionary-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-data-dictionary-core jar diff --git a/warehouse/edge-dictionary-core/pom.xml b/warehouse/edge-dictionary-core/pom.xml index b1daa9c8355..dba8bac91a3 100644 --- a/warehouse/edge-dictionary-core/pom.xml +++ b/warehouse/edge-dictionary-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-edge-dictionary-core jar diff --git a/warehouse/edge-model-configuration-core/pom.xml b/warehouse/edge-model-configuration-core/pom.xml index 3972ac1baa3..c3aa3a868e8 100644 --- a/warehouse/edge-model-configuration-core/pom.xml +++ b/warehouse/edge-model-configuration-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-edge-model-configuration-core jar diff --git a/warehouse/index-stats/pom.xml b/warehouse/index-stats/pom.xml index 726c489112d..9d1af44b5fc 100644 --- a/warehouse/index-stats/pom.xml +++ b/warehouse/index-stats/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-index-stats jar diff --git a/warehouse/ingest-configuration/pom.xml b/warehouse/ingest-configuration/pom.xml index 481afd6b422..6298cf821b4 100644 --- a/warehouse/ingest-configuration/pom.xml +++ b/warehouse/ingest-configuration/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ingest-configuration diff --git a/warehouse/ingest-core/pom.xml b/warehouse/ingest-core/pom.xml index abb337f6e9b..ce13224d0e0 100644 --- a/warehouse/ingest-core/pom.xml +++ b/warehouse/ingest-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ingest-core jar diff --git a/warehouse/ingest-csv/pom.xml b/warehouse/ingest-csv/pom.xml index eca3a6b9f01..35b1c2df03c 100644 --- a/warehouse/ingest-csv/pom.xml +++ b/warehouse/ingest-csv/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ingest-csv jar diff --git a/warehouse/ingest-json/pom.xml b/warehouse/ingest-json/pom.xml index 70086727d52..5d7c4f1a8a7 100644 --- a/warehouse/ingest-json/pom.xml +++ b/warehouse/ingest-json/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ingest-json jar diff --git a/warehouse/ingest-nyctlc/pom.xml b/warehouse/ingest-nyctlc/pom.xml index e8e546e8d0a..fa661596a31 100644 --- a/warehouse/ingest-nyctlc/pom.xml +++ b/warehouse/ingest-nyctlc/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ingest-nyctlc jar diff --git a/warehouse/ingest-scripts/pom.xml b/warehouse/ingest-scripts/pom.xml index f59cf9997a2..0824161dbe5 100644 --- a/warehouse/ingest-scripts/pom.xml +++ b/warehouse/ingest-scripts/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ingest-scripts ${project.artifactId} diff --git a/warehouse/ingest-wikipedia/pom.xml b/warehouse/ingest-wikipedia/pom.xml index 4932211c2b3..fea39cd417e 100644 --- a/warehouse/ingest-wikipedia/pom.xml +++ b/warehouse/ingest-wikipedia/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ingest-wikipedia jar diff --git a/warehouse/metrics-core/pom.xml b/warehouse/metrics-core/pom.xml index 088538ca91a..0257b2bb37e 100644 --- a/warehouse/metrics-core/pom.xml +++ b/warehouse/metrics-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-metrics-core jar diff --git a/warehouse/ops-tools/config-compare/pom.xml b/warehouse/ops-tools/config-compare/pom.xml index 3d4308bf7c6..dde7abee524 100644 --- a/warehouse/ops-tools/config-compare/pom.xml +++ b/warehouse/ops-tools/config-compare/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-ops-tools-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ops-tools-config-compare diff --git a/warehouse/ops-tools/index-validation/pom.xml b/warehouse/ops-tools/index-validation/pom.xml index 073d3a3e1a1..ff10e398236 100644 --- a/warehouse/ops-tools/index-validation/pom.xml +++ b/warehouse/ops-tools/index-validation/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-ops-tools-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ops-tools-index-validation jar diff --git a/warehouse/ops-tools/pom.xml b/warehouse/ops-tools/pom.xml index c504c441b87..db58cb07060 100644 --- a/warehouse/ops-tools/pom.xml +++ b/warehouse/ops-tools/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ops-tools-parent pom diff --git a/warehouse/pom.xml b/warehouse/pom.xml index eb459015a01..f08999e6bf3 100644 --- a/warehouse/pom.xml +++ b/warehouse/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-warehouse-parent pom diff --git a/warehouse/query-core/pom.xml b/warehouse/query-core/pom.xml index a0cce52c8d1..78fc6678468 100644 --- a/warehouse/query-core/pom.xml +++ b/warehouse/query-core/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-query-core jar diff --git a/warehouse/regression-testing/pom.xml b/warehouse/regression-testing/pom.xml index f28530a25d6..125f4ac8cd9 100644 --- a/warehouse/regression-testing/pom.xml +++ b/warehouse/regression-testing/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-warehouse-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-regression-testing ${project.artifactId} diff --git a/warehouse/ssdeep-common/pom.xml b/warehouse/ssdeep-common/pom.xml index be6fa2b12e9..108a9b5aa7e 100644 --- a/warehouse/ssdeep-common/pom.xml +++ b/warehouse/ssdeep-common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT ../../pom.xml datawave-ssdeep-common diff --git a/web-services/accumulo/pom.xml b/web-services/accumulo/pom.xml index 679530f84d1..5ce18817d58 100644 --- a/web-services/accumulo/pom.xml +++ b/web-services/accumulo/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-accumulo ejb diff --git a/web-services/atom/pom.xml b/web-services/atom/pom.xml index db8b1827fb7..4a925cf581b 100644 --- a/web-services/atom/pom.xml +++ b/web-services/atom/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-atom ejb diff --git a/web-services/cached-results/pom.xml b/web-services/cached-results/pom.xml index c118ca94488..de70d9d80cb 100644 --- a/web-services/cached-results/pom.xml +++ b/web-services/cached-results/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-cached-results ejb diff --git a/web-services/client/pom.xml b/web-services/client/pom.xml index 32055265e76..d24e5838ec9 100644 --- a/web-services/client/pom.xml +++ b/web-services/client/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-client jar diff --git a/web-services/common-util/pom.xml b/web-services/common-util/pom.xml index 27c350a6afb..c75e95b7f1d 100644 --- a/web-services/common-util/pom.xml +++ b/web-services/common-util/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-common-util jar diff --git a/web-services/common/pom.xml b/web-services/common/pom.xml index 71c7b9d3394..5df8eaa097d 100644 --- a/web-services/common/pom.xml +++ b/web-services/common/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-common ejb diff --git a/web-services/deploy/application/pom.xml b/web-services/deploy/application/pom.xml index 50b4605315e..cfd7c0a4b8a 100644 --- a/web-services/deploy/application/pom.xml +++ b/web-services/deploy/application/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-deploy-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-deploy-application ear diff --git a/web-services/deploy/configuration/pom.xml b/web-services/deploy/configuration/pom.xml index 9909110089f..83357cc8af5 100644 --- a/web-services/deploy/configuration/pom.xml +++ b/web-services/deploy/configuration/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-deploy-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-deploy-configuration jar diff --git a/web-services/deploy/docs/pom.xml b/web-services/deploy/docs/pom.xml index ec416f50658..bf3beaf195f 100644 --- a/web-services/deploy/docs/pom.xml +++ b/web-services/deploy/docs/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-deploy-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-deploy-docs war diff --git a/web-services/deploy/pom.xml b/web-services/deploy/pom.xml index 03584195e2c..dc67c937f8e 100644 --- a/web-services/deploy/pom.xml +++ b/web-services/deploy/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT gov.nsa.datawave.webservices datawave-ws-deploy-parent diff --git a/web-services/deploy/spring-framework-integration/pom.xml b/web-services/deploy/spring-framework-integration/pom.xml index f6382a331dd..52459f2a6e4 100644 --- a/web-services/deploy/spring-framework-integration/pom.xml +++ b/web-services/deploy/spring-framework-integration/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-deploy-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT spring-framework-integration ${project.artifactId} diff --git a/web-services/dictionary/pom.xml b/web-services/dictionary/pom.xml index aeedafc99b1..20dde3cc373 100644 --- a/web-services/dictionary/pom.xml +++ b/web-services/dictionary/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-dictionary ejb diff --git a/web-services/examples/client-login/pom.xml b/web-services/examples/client-login/pom.xml index 64f4eaf15d6..7191338ed62 100644 --- a/web-services/examples/client-login/pom.xml +++ b/web-services/examples/client-login/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-examples-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-examples-client-login ejb diff --git a/web-services/examples/http-client/pom.xml b/web-services/examples/http-client/pom.xml index c31996fd974..bd9c22ef3de 100644 --- a/web-services/examples/http-client/pom.xml +++ b/web-services/examples/http-client/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-examples-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-examples-http-client jar diff --git a/web-services/examples/jms-client/pom.xml b/web-services/examples/jms-client/pom.xml index e5de977827c..c94d0813100 100644 --- a/web-services/examples/jms-client/pom.xml +++ b/web-services/examples/jms-client/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-examples-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-examples-jms-client jar diff --git a/web-services/examples/pom.xml b/web-services/examples/pom.xml index 0b78e6a15aa..c8108533252 100644 --- a/web-services/examples/pom.xml +++ b/web-services/examples/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-examples-parent pom diff --git a/web-services/examples/query-war/pom.xml b/web-services/examples/query-war/pom.xml index 7474a8940ff..3cb70f571a1 100644 --- a/web-services/examples/query-war/pom.xml +++ b/web-services/examples/query-war/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-examples-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-examples-query-war war diff --git a/web-services/map-reduce-embedded/pom.xml b/web-services/map-reduce-embedded/pom.xml index 70d702afab9..edd123de926 100644 --- a/web-services/map-reduce-embedded/pom.xml +++ b/web-services/map-reduce-embedded/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-map-reduce-embedded jar diff --git a/web-services/map-reduce-status/pom.xml b/web-services/map-reduce-status/pom.xml index 820fd21dfb1..bfdea7622ab 100644 --- a/web-services/map-reduce-status/pom.xml +++ b/web-services/map-reduce-status/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-map-reduce-status ejb diff --git a/web-services/map-reduce/pom.xml b/web-services/map-reduce/pom.xml index cef46fb20f2..5ca9823d953 100644 --- a/web-services/map-reduce/pom.xml +++ b/web-services/map-reduce/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-map-reduce ejb diff --git a/web-services/model/pom.xml b/web-services/model/pom.xml index 3671c77fd5d..d136098c61c 100644 --- a/web-services/model/pom.xml +++ b/web-services/model/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-model ejb diff --git a/web-services/modification/pom.xml b/web-services/modification/pom.xml index b902daef1c2..138d81fe4ae 100644 --- a/web-services/modification/pom.xml +++ b/web-services/modification/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-modification ejb diff --git a/web-services/pom.xml b/web-services/pom.xml index c4396ee22a3..c877836bfde 100644 --- a/web-services/pom.xml +++ b/web-services/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave datawave-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT gov.nsa.datawave.webservices datawave-ws-parent diff --git a/web-services/query-websocket/pom.xml b/web-services/query-websocket/pom.xml index d2df02a6bfd..ace51e49e85 100644 --- a/web-services/query-websocket/pom.xml +++ b/web-services/query-websocket/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-query-websocket war diff --git a/web-services/query/pom.xml b/web-services/query/pom.xml index 275dbacddd5..b91638faf34 100644 --- a/web-services/query/pom.xml +++ b/web-services/query/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-query ejb diff --git a/web-services/rest-api/pom.xml b/web-services/rest-api/pom.xml index b4d6f739e62..fa63ee40984 100644 --- a/web-services/rest-api/pom.xml +++ b/web-services/rest-api/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-rest-api war diff --git a/web-services/security/pom.xml b/web-services/security/pom.xml index df626f77218..b739d4aa839 100644 --- a/web-services/security/pom.xml +++ b/web-services/security/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-security ejb diff --git a/web-services/web-root/pom.xml b/web-services/web-root/pom.xml index 2bf46e1945b..ee0c71de60d 100644 --- a/web-services/web-root/pom.xml +++ b/web-services/web-root/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.webservices datawave-ws-parent - 6.6.0-SNAPSHOT + 6.7.0-SNAPSHOT datawave-ws-web-root war From 05a4a12d95027607a7419de2c3ac8b9ed2a3beb0 Mon Sep 17 00:00:00 2001 From: Christopher Gross Date: Wed, 14 Feb 2024 10:43:07 -0500 Subject: [PATCH 14/50] #2226 - fix javadoc warnings (#2232) * fix javadoc warnings --------- Co-authored-by: Christopher Gross cogross --- .../common/test/utils/ProcessUtils.java | 2 + .../java/datawave/edge/protobuf/EdgeData.java | 346 ++++++++ .../datawave/edge/util/EdgeValueHelper.java | 4 +- .../datawave/ingest/protobuf/TermWeight.java | 153 ++++ .../java/datawave/ingest/protobuf/Uid.java | 170 ++++ .../util/cache/watch/FileRuleWatcher.java | 18 +- .../util/cache/watch/FileSystemWatcher.java | 7 +- .../ingest/util/io/GzipDetectionUtil.java | 4 + .../filter/ConfigurableAgeOffFilter.java | 32 +- .../iterators/filter/RegexFilterBase.java | 1 + .../iterators/filter/TokenFilterBase.java | 3 +- .../iterators/filter/TokenSpecParser.java | 26 +- .../iterators/filter/TokenTtlTrie.java | 17 + .../filter/TokenizingFilterBase.java | 5 + .../iterators/filter/ageoff/AppliedRule.java | 16 +- .../filter/ageoff/DataTypeAgeOffFilter.java | 1 + .../filter/ageoff/FieldAgeOffFilter.java | 2 + .../iterators/filter/ageoff/FilterRule.java | 13 +- .../filter/ageoff/MaximumAgeOffFilter.java | 2 + .../datawave/mr/bulk/BulkInputFormat.java | 4 + .../mr/bulk/split/FileRangeSplit.java | 18 +- .../datawave/mr/bulk/split/RangeSplit.java | 5 + .../java/datawave/util/ObjectFactory.java | 3 + .../datawave/util/OperationEvaluator.java | 6 + .../src/main/java/datawave/util/TextUtil.java | 8 + .../java/datawave/util/timely/TcpClient.java | 1 + .../data/config/ingest/FilterIngest.java | 3 +- .../dateindex/DateIndexDataTypeHandler.java | 4 +- .../handler/error/ErrorDataTypeHandler.java | 2 +- .../error/ErrorShardedDataTypeHandler.java | 2 +- .../handler/shard/ShardedDataTypeHandler.java | 2 +- ...ndedContentIndexingColumnBasedHandler.java | 2 +- .../ingest/metadata/EventMetadata.java | 2 +- .../ingest/protobuf/RawRecordContainer.java | 338 ++++++++ .../config/AbstractTableConfigHelper.java | 5 + .../mapreduce/IngestMetricsMapper.java | 5 +- .../mapreduce/LoaderMetricsMapper.java | 7 +- .../common/grouping/DocumentGrouper.java | 11 +- .../query/common/grouping/GroupFields.java | 2 + .../query/function/AncestorRangeProvider.java | 3 +- .../datawave/query/jexl/JexlASTHelper.java | 107 ++- .../EdgeTableRangeBuildingVisitor.java | 2 +- .../visitors/IngestTypePruningVisitor.java | 5 +- .../jexl/visitors/QueryModelVisitor.java | 2 + .../CustomAnalyzerQueryNodeProcessor.java | 10 +- .../predicate/EventDataQueryFieldFilter.java | 6 + .../query/transformer/ExcerptTransform.java | 2 + .../datawave/query/util/DateIndexHelper.java | 2 +- .../ssdeep/BucketAccumuloKeyGenerator.java | 3 +- .../datawave/util/ssdeep/HashReverse.java | 19 +- .../datawave/util/ssdeep/IntegerEncoding.java | 18 +- .../util/ssdeep/SSDeepHashScorer.java | 13 +- .../results/cached/CachedResultsBean.java | 5 - .../webservice/query/QueryMessages.java | 430 +++++++++ .../webservice/query/QueryParametersImpl.java | 17 +- .../query/exception/ExceptionMessages.java | 60 ++ .../query/result/event/EventBase.java | 2 + .../result/event/ResponseObjectFactory.java | 2 +- .../webservice/result/ResponseMessages.java | 814 ++++++++++++++++++ .../cached/result/CachedresultMessages.java | 259 ++++++ .../security/util/WSAuthorizationsUtil.java | 1 + .../mr/state/MapReduceStatePersisterBean.java | 4 +- .../cache/CreatedQueryLogicCacheBean.java | 1 + .../webservice/query/logic/QueryLogic.java | 2 + .../query/metric/QueryMetricsWriter.java | 18 +- .../query/runner/QueryExecutorBean.java | 1 - .../simple/DatabaseUserService.java | 4 +- 67 files changed, 2970 insertions(+), 94 deletions(-) diff --git a/common-test/src/main/java/datawave/common/test/utils/ProcessUtils.java b/common-test/src/main/java/datawave/common/test/utils/ProcessUtils.java index c3b57af8df6..c8e3ea704f5 100644 --- a/common-test/src/main/java/datawave/common/test/utils/ProcessUtils.java +++ b/common-test/src/main/java/datawave/common/test/utils/ProcessUtils.java @@ -46,6 +46,8 @@ public static List buildApplicationCommandLine(String clzName, List * edge table count * + * + * @return if there is a count */ boolean hasCount(); @@ -27,6 +29,8 @@ public interface EdgeValueOrBuilder extends * 

              * edge table count
          *
    + * + * @return the count */ long getCount(); @@ -36,6 +40,8 @@ public interface EdgeValueOrBuilder extends * 
              * hour activity array (length 24)
          *
    + * + * @return the list of hours */ java.util.List getHoursList(); @@ -45,6 +51,8 @@ public interface EdgeValueOrBuilder extends * 
              * hour activity array (length 24)
          *
    + * + * @return the count of hours */ int getHoursCount(); @@ -54,6 +62,11 @@ public interface EdgeValueOrBuilder extends * 
              * hour activity array (length 24)
          *
    + * + * @param index + * the index + * + * @return the hours at the specific index */ long getHours(int index); @@ -65,6 +78,8 @@ public interface EdgeValueOrBuilder extends * packed means it won't appear in serialized data * that doesn't contain any hours fields. * + * + * @return list of durations */ java.util.List getDurationList(); @@ -76,6 +91,8 @@ public interface EdgeValueOrBuilder extends * packed means it won't appear in serialized data * that doesn't contain any hours fields. * + * + * @return duration count */ int getDurationCount(); @@ -87,6 +104,11 @@ public interface EdgeValueOrBuilder extends * packed means it won't appear in serialized data * that doesn't contain any hours fields. * + * + * @param index + * the index + * + * @return the duration at an index */ long getDuration(int index); @@ -96,6 +118,8 @@ public interface EdgeValueOrBuilder extends * 
              * Bitmask for tracking edge creation by the hour.
          *
    + * + * @return if there is an hour bitmask */ boolean hasHourBitmask(); @@ -105,61 +129,85 @@ public interface EdgeValueOrBuilder extends * 
              * Bitmask for tracking edge creation by the hour.
          *
    + * + * @return the hour bitmask */ int getHourBitmask(); /** * optional string sourceValue = 5; + * + * @return if the optional sourcevalue is set */ boolean hasSourceValue(); /** * optional string sourceValue = 5; + * + * @return the sourcevalue */ java.lang.String getSourceValue(); /** * optional string sourceValue = 5; + * + * @return the sourcevalue byte string */ com.google.protobuf.ByteString getSourceValueBytes(); /** * optional string sinkValue = 6; + * + * @return if there is a sinkvalue */ boolean hasSinkValue(); /** * optional string sinkValue = 6; + * + * @return the sinkvalue */ java.lang.String getSinkValue(); /** * optional string sinkValue = 6; + * + * @return the sinkvalue byte string */ com.google.protobuf.ByteString getSinkValueBytes(); /** * optional string loadDate = 7; + * + * @return if there is a loaddate */ boolean hasLoadDate(); /** * optional string loadDate = 7; + * + * @return the loaddate */ java.lang.String getLoadDate(); /** * optional string loadDate = 7; + * + * @return the loaddate bytestring */ com.google.protobuf.ByteString getLoadDateBytes(); /** * optional bool badActivity = 8; + * + * @return if there is bad activity */ boolean hasBadActivity(); /** * optional bool badActivity = 8; + * + * @return back activity value */ boolean getBadActivity(); @@ -169,6 +217,8 @@ public interface EdgeValueOrBuilder extends * 
              * a single uuid of an event that contributed to the edge
          *
    + * + * @return if there is a uuid for the edge value */ boolean hasUuid(); @@ -178,6 +228,8 @@ public interface EdgeValueOrBuilder extends * 
              * a single uuid of an event that contributed to the edge
          *
    + * + * @return the uuid */ datawave.edge.protobuf.EdgeData.EdgeValue.UUID getUuid(); @@ -187,6 +239,8 @@ public interface EdgeValueOrBuilder extends * 
              * a single uuid of an event that contributed to the edge
          *
    + * + * @return the uuidOrBuilder */ datawave.edge.protobuf.EdgeData.EdgeValue.UUIDOrBuilder getUuidOrBuilder(); @@ -196,6 +250,8 @@ public interface EdgeValueOrBuilder extends * 
              * a single uuid of an event that contributed to the edge in string format
          *
    + * + * @return if there is a uuidString */ boolean hasUuidString(); @@ -205,6 +261,8 @@ public interface EdgeValueOrBuilder extends * 
              * a single uuid of an event that contributed to the edge in string format
          *
    + * + * @return the uuid string */ java.lang.String getUuidString(); @@ -214,6 +272,8 @@ public interface EdgeValueOrBuilder extends * 
              * a single uuid of an event that contributed to the edge in string format
          *
    + * + * @return the uuid bytestring */ com.google.protobuf.ByteString getUuidStringBytes(); } @@ -413,21 +473,29 @@ public interface UUIDOrBuilder extends /** * required sint64 least_significant_bits = 1; + * + * @return if it has least significant bits */ boolean hasLeastSignificantBits(); /** * required sint64 least_significant_bits = 1; + * + * @return the least significant bits */ long getLeastSignificantBits(); /** * required sint64 most_significant_bits = 2; + * + * @return if it has most significant bits */ boolean hasMostSignificantBits(); /** * required sint64 most_significant_bits = 2; + * + * @return most significant bits */ long getMostSignificantBits(); } @@ -533,6 +601,8 @@ public com.google.protobuf.Parser getParserForType() { /** * required sint64 least_significant_bits = 1; + * + * @return if there are least significant bits */ public boolean hasLeastSignificantBits() { return ((bitField0_ & 0x00000001) == 0x00000001); @@ -540,6 +610,8 @@ public boolean hasLeastSignificantBits() { /** * required sint64 least_significant_bits = 1; + * + * @return the least significant bits */ public long getLeastSignificantBits() { return leastSignificantBits_; @@ -550,6 +622,8 @@ public long getLeastSignificantBits() { /** * required sint64 most_significant_bits = 2; + * + * @return if there are most significant bits */ public boolean hasMostSignificantBits() { return ((bitField0_ & 0x00000002) == 0x00000002); @@ -557,6 +631,8 @@ public boolean hasMostSignificantBits() { /** * required sint64 most_significant_bits = 2; + * + * @return the most significant bits */ public long getMostSignificantBits() { return mostSignificantBits_; @@ -829,6 +905,8 @@ public Builder mergeFrom(com.google.protobuf.CodedInputStream input, com.google. /** * required sint64 least_significant_bits = 1; + * + * @return if it has least significant bits */ public boolean hasLeastSignificantBits() { return ((bitField0_ & 0x00000001) == 0x00000001); @@ -836,6 +914,8 @@ public boolean hasLeastSignificantBits() { /** * required sint64 least_significant_bits = 1; + * + * @return the least significant bits */ public long getLeastSignificantBits() { return leastSignificantBits_; @@ -843,6 +923,11 @@ public long getLeastSignificantBits() { /** * required sint64 least_significant_bits = 1; + * + * @param value + * the value + * + * @return builder to set least significant bits */ public Builder setLeastSignificantBits(long value) { bitField0_ |= 0x00000001; @@ -853,6 +938,8 @@ public Builder setLeastSignificantBits(long value) { /** * required sint64 least_significant_bits = 1; + * + * @return builder to clear least significant bits */ public Builder clearLeastSignificantBits() { bitField0_ = (bitField0_ & ~0x00000001); @@ -865,6 +952,8 @@ public Builder clearLeastSignificantBits() { /** * required sint64 most_significant_bits = 2; + * + * @return if it has most significant bits */ public boolean hasMostSignificantBits() { return ((bitField0_ & 0x00000002) == 0x00000002); @@ -872,6 +961,8 @@ public boolean hasMostSignificantBits() { /** * required sint64 most_significant_bits = 2; + * + * @return the most significant bits */ public long getMostSignificantBits() { return mostSignificantBits_; @@ -879,6 +970,11 @@ public long getMostSignificantBits() { /** * required sint64 most_significant_bits = 2; + * + * @param value + * the value + * + * @return builder to set most significant bits */ public Builder setMostSignificantBits(long value) { bitField0_ |= 0x00000002; @@ -889,6 +985,8 @@ public Builder setMostSignificantBits(long value) { /** * required sint64 most_significant_bits = 2; + * + * @return builder to clear the most significant bits */ public Builder clearMostSignificantBits() { bitField0_ = (bitField0_ & ~0x00000002); @@ -918,6 +1016,8 @@ public Builder clearMostSignificantBits() { * 
              * edge table count
          *
    + * + * @return if it has a count */ public boolean hasCount() { return ((bitField0_ & 0x00000001) == 0x00000001); @@ -929,6 +1029,8 @@ public boolean hasCount() { * 
              * edge table count
          *
    + * + * @return the count */ public long getCount() { return count_; @@ -943,6 +1045,8 @@ public long getCount() { * 
              * hour activity array (length 24)
          *
    + * + * @return the hours list */ public java.util.List getHoursList() { return hours_; @@ -954,6 +1058,8 @@ public java.util.List getHoursList() { * 
              * hour activity array (length 24)
          *
    + * + * @return the size of the hours list */ public int getHoursCount() { return hours_.size(); @@ -965,6 +1071,11 @@ public int getHoursCount() { * 
              * hour activity array (length 24)
          *
    + * + * @param index + * the index + * + * @return the hours at the index */ public long getHours(int index) { return hours_.get(index); @@ -983,6 +1094,8 @@ public long getHours(int index) { * packed means it won't appear in serialized data * that doesn't contain any hours fields. * + * + * @return the duration list */ public java.util.List getDurationList() { return duration_; @@ -996,6 +1109,8 @@ public java.util.List getDurationList() { * packed means it won't appear in serialized data * that doesn't contain any hours fields. * + * + * @return the duration count */ public int getDurationCount() { return duration_.size(); @@ -1009,6 +1124,11 @@ public int getDurationCount() { * packed means it won't appear in serialized data * that doesn't contain any hours fields. * + * + * @param index + * the index + * + * @return the duration at the index */ public long getDuration(int index) { return duration_.get(index); @@ -1025,6 +1145,8 @@ public long getDuration(int index) { * 
              * Bitmask for tracking edge creation by the hour.
          *
    + * + * @return if there is an hour bitmask */ public boolean hasHourBitmask() { return ((bitField0_ & 0x00000002) == 0x00000002); @@ -1036,6 +1158,8 @@ public boolean hasHourBitmask() { * 
              * Bitmask for tracking edge creation by the hour.
          *
    + * + * @return the hour bitmask */ public int getHourBitmask() { return hourBitmask_; @@ -1046,6 +1170,8 @@ public int getHourBitmask() { /** * optional string sourceValue = 5; + * + * @return if there is a source value */ public boolean hasSourceValue() { return ((bitField0_ & 0x00000004) == 0x00000004); @@ -1053,6 +1179,8 @@ public boolean hasSourceValue() { /** * optional string sourceValue = 5; + * + * @return the source value */ public java.lang.String getSourceValue() { java.lang.Object ref = sourceValue_; @@ -1070,6 +1198,8 @@ public java.lang.String getSourceValue() { /** * optional string sourceValue = 5; + * + * @return the source value byte string */ public com.google.protobuf.ByteString getSourceValueBytes() { java.lang.Object ref = sourceValue_; @@ -1087,6 +1217,8 @@ public com.google.protobuf.ByteString getSourceValueBytes() { /** * optional string sinkValue = 6; + * + * @return if it has sink value */ public boolean hasSinkValue() { return ((bitField0_ & 0x00000008) == 0x00000008); @@ -1094,6 +1226,8 @@ public boolean hasSinkValue() { /** * optional string sinkValue = 6; + * + * @return the sink value */ public java.lang.String getSinkValue() { java.lang.Object ref = sinkValue_; @@ -1111,6 +1245,8 @@ public java.lang.String getSinkValue() { /** * optional string sinkValue = 6; + * + * @return the sink value byte string */ public com.google.protobuf.ByteString getSinkValueBytes() { java.lang.Object ref = sinkValue_; @@ -1128,6 +1264,8 @@ public com.google.protobuf.ByteString getSinkValueBytes() { /** * optional string loadDate = 7; + * + * @return if there is a load date */ public boolean hasLoadDate() { return ((bitField0_ & 0x00000010) == 0x00000010); @@ -1135,6 +1273,8 @@ public boolean hasLoadDate() { /** * optional string loadDate = 7; + * + * @return the load date */ public java.lang.String getLoadDate() { java.lang.Object ref = loadDate_; @@ -1152,6 +1292,8 @@ public java.lang.String getLoadDate() { /** * optional string loadDate = 7; + * + * @return the load date byte string */ public com.google.protobuf.ByteString getLoadDateBytes() { java.lang.Object ref = loadDate_; @@ -1169,6 +1311,8 @@ public com.google.protobuf.ByteString getLoadDateBytes() { /** * optional bool badActivity = 8; + * + * @return if there is bad activity */ public boolean hasBadActivity() { return ((bitField0_ & 0x00000020) == 0x00000020); @@ -1176,6 +1320,8 @@ public boolean hasBadActivity() { /** * optional bool badActivity = 8; + * + * @return bad activity */ public boolean getBadActivity() { return badActivity_; @@ -1190,6 +1336,8 @@ public boolean getBadActivity() { * 
              * a single uuid of an event that contributed to the edge
          *
    + * + * @return if it has uuid */ public boolean hasUuid() { return ((bitField0_ & 0x00000040) == 0x00000040); @@ -1201,6 +1349,8 @@ public boolean hasUuid() { * 
              * a single uuid of an event that contributed to the edge
          *
    + * + * @return the uuid */ public datawave.edge.protobuf.EdgeData.EdgeValue.UUID getUuid() { return uuid_; @@ -1212,6 +1362,8 @@ public datawave.edge.protobuf.EdgeData.EdgeValue.UUID getUuid() { * 
              * a single uuid of an event that contributed to the edge
          *
    + * + * @return the uuid or builder */ public datawave.edge.protobuf.EdgeData.EdgeValue.UUIDOrBuilder getUuidOrBuilder() { return uuid_; @@ -1226,6 +1378,8 @@ public datawave.edge.protobuf.EdgeData.EdgeValue.UUIDOrBuilder getUuidOrBuilder( * 
              * a single uuid of an event that contributed to the edge in string format
          *
    + * + * @return if there is uuid */ public boolean hasUuidString() { return ((bitField0_ & 0x00000080) == 0x00000080); @@ -1237,6 +1391,8 @@ public boolean hasUuidString() { * 
              * a single uuid of an event that contributed to the edge in string format
          *
    + * + * @return the uuid string */ public java.lang.String getUuidString() { java.lang.Object ref = uuidString_; @@ -1258,6 +1414,8 @@ public java.lang.String getUuidString() { * 
              * a single uuid of an event that contributed to the edge in string format
          *
    + * + * @return uuid byte string */ public com.google.protobuf.ByteString getUuidStringBytes() { java.lang.Object ref = uuidString_; @@ -1728,6 +1886,8 @@ public Builder mergeFrom(com.google.protobuf.CodedInputStream input, com.google. * 
                  * edge table count
              *
    + * + * @return if it has a count */ public boolean hasCount() { return ((bitField0_ & 0x00000001) == 0x00000001); @@ -1739,6 +1899,8 @@ public boolean hasCount() { * 
                  * edge table count
              *
    + * + * @return the count */ public long getCount() { return count_; @@ -1750,6 +1912,11 @@ public long getCount() { * 
                  * edge table count
              *
    + * + * @param value + * the count + * + * @return a builder to set the count */ public Builder setCount(long value) { bitField0_ |= 0x00000001; @@ -1764,6 +1931,8 @@ public Builder setCount(long value) { * 
                  * edge table count
              *
    + * + * @return a builder to clear the count */ public Builder clearCount() { bitField0_ = (bitField0_ & ~0x00000001); @@ -1787,6 +1956,8 @@ private void ensureHoursIsMutable() { * 
                  * hour activity array (length 24)
              *
    + * + * @return the hours list */ public java.util.List getHoursList() { return java.util.Collections.unmodifiableList(hours_); @@ -1798,6 +1969,8 @@ public java.util.List getHoursList() { * 
                  * hour activity array (length 24)
              *
    + * + * @return the size of the hours list */ public int getHoursCount() { return hours_.size(); @@ -1809,6 +1982,11 @@ public int getHoursCount() { * 
                  * hour activity array (length 24)
              *
    + * + * @param index + * the index + * + * @return the hours at the index */ public long getHours(int index) { return hours_.get(index); @@ -1820,6 +1998,13 @@ public long getHours(int index) { * 
                  * hour activity array (length 24)
              *
    + * + * @param index + * the index + * @param value + * the value + * + * @return builder to set the hours a the provided index to the value provided */ public Builder setHours(int index, long value) { ensureHoursIsMutable(); @@ -1834,6 +2019,11 @@ public Builder setHours(int index, long value) { * 
                  * hour activity array (length 24)
              *
    + * + * @param value + * the hours to add to the list + * + * @return builder to add the hours to the list */ public Builder addHours(long value) { ensureHoursIsMutable(); @@ -1848,6 +2038,11 @@ public Builder addHours(long value) { * 
                  * hour activity array (length 24)
              *
    + * + * @param values + * the values to add + * + * @return builder to add all the hours from the values parameter */ public Builder addAllHours(java.lang.Iterable values) { ensureHoursIsMutable(); @@ -1862,6 +2057,8 @@ public Builder addAllHours(java.lang.Iterable values) * 
                  * hour activity array (length 24)
              *
    + * + * @return builder to clear hours */ public Builder clearHours() { hours_ = java.util.Collections.emptyList(); @@ -1887,6 +2084,8 @@ private void ensureDurationIsMutable() { * packed means it won't appear in serialized data * that doesn't contain any hours fields. * + * + * @return the duration list */ public java.util.List getDurationList() { return java.util.Collections.unmodifiableList(duration_); @@ -1900,6 +2099,8 @@ public java.util.List getDurationList() { * packed means it won't appear in serialized data * that doesn't contain any hours fields. * + * + * @return the duration list size */ public int getDurationCount() { return duration_.size(); @@ -1913,6 +2114,11 @@ public int getDurationCount() { * packed means it won't appear in serialized data * that doesn't contain any hours fields. * + * + * @param index + * the index + * + * @return the duration at the index provided */ public long getDuration(int index) { return duration_.get(index); @@ -1926,6 +2132,13 @@ public long getDuration(int index) { * packed means it won't appear in serialized data * that doesn't contain any hours fields. * + * + * @param index + * the index + * @param value + * the value + * + * @return builder to set the duration to a value for provided index */ public Builder setDuration(int index, long value) { ensureDurationIsMutable(); @@ -1942,6 +2155,11 @@ public Builder setDuration(int index, long value) { * packed means it won't appear in serialized data * that doesn't contain any hours fields. * + * + * @param value + * the value + * + * @return builder to add a duration to the list */ public Builder addDuration(long value) { ensureDurationIsMutable(); @@ -1958,6 +2176,11 @@ public Builder addDuration(long value) { * packed means it won't appear in serialized data * that doesn't contain any hours fields. * + * + * @param values + * iterator of duration values to add + * + * @return builder to add durations from values passed in */ public Builder addAllDuration(java.lang.Iterable values) { ensureDurationIsMutable(); @@ -1974,6 +2197,8 @@ public Builder addAllDuration(java.lang.Iterable value * packed means it won't appear in serialized data * that doesn't contain any hours fields. * + * + * @return builder to clear durations */ public Builder clearDuration() { duration_ = java.util.Collections.emptyList(); @@ -1990,6 +2215,8 @@ public Builder clearDuration() { * 
                  * Bitmask for tracking edge creation by the hour.
              *
    + * + * @return if it has hour bitmask */ public boolean hasHourBitmask() { return ((bitField0_ & 0x00000008) == 0x00000008); @@ -2001,6 +2228,8 @@ public boolean hasHourBitmask() { * 
                  * Bitmask for tracking edge creation by the hour.
              *
    + * + * @return the hour bitmask */ public int getHourBitmask() { return hourBitmask_; @@ -2012,6 +2241,11 @@ public int getHourBitmask() { * 
                  * Bitmask for tracking edge creation by the hour.
              *
    + * + * @param value + * the value + * + * @return builder to set hour bitmask */ public Builder setHourBitmask(int value) { bitField0_ |= 0x00000008; @@ -2026,6 +2260,8 @@ public Builder setHourBitmask(int value) { * 
                  * Bitmask for tracking edge creation by the hour.
              *
    + * + * @return builder to clear hour bitmask */ public Builder clearHourBitmask() { bitField0_ = (bitField0_ & ~0x00000008); @@ -2038,6 +2274,8 @@ public Builder clearHourBitmask() { /** * optional string sourceValue = 5; + * + * @return if it has source value */ public boolean hasSourceValue() { return ((bitField0_ & 0x00000010) == 0x00000010); @@ -2045,6 +2283,8 @@ public boolean hasSourceValue() { /** * optional string sourceValue = 5; + * + * @return the source value */ public java.lang.String getSourceValue() { java.lang.Object ref = sourceValue_; @@ -2062,6 +2302,8 @@ public java.lang.String getSourceValue() { /** * optional string sourceValue = 5; + * + * @return source value byte string */ public com.google.protobuf.ByteString getSourceValueBytes() { java.lang.Object ref = sourceValue_; @@ -2076,6 +2318,11 @@ public com.google.protobuf.ByteString getSourceValueBytes() { /** * optional string sourceValue = 5; + * + * @param value + * the new source value + * + * @return builder to set the source value */ public Builder setSourceValue(java.lang.String value) { if (value == null) { @@ -2089,6 +2336,8 @@ public Builder setSourceValue(java.lang.String value) { /** * optional string sourceValue = 5; + * + * @return builder to clear source value */ public Builder clearSourceValue() { bitField0_ = (bitField0_ & ~0x00000010); @@ -2099,6 +2348,11 @@ public Builder clearSourceValue() { /** * optional string sourceValue = 5; + * + * @param value + * to set source value to + * + * @return builder to set source value to parameter passed in */ public Builder setSourceValueBytes(com.google.protobuf.ByteString value) { if (value == null) { @@ -2114,6 +2368,8 @@ public Builder setSourceValueBytes(com.google.protobuf.ByteString value) { /** * optional string sinkValue = 6; + * + * @return if there is a sink value */ public boolean hasSinkValue() { return ((bitField0_ & 0x00000020) == 0x00000020); @@ -2121,6 +2377,8 @@ public boolean hasSinkValue() { /** * optional string sinkValue = 6; + * + * @return the sink value */ public java.lang.String getSinkValue() { java.lang.Object ref = sinkValue_; @@ -2138,6 +2396,8 @@ public java.lang.String getSinkValue() { /** * optional string sinkValue = 6; + * + * @return the sink value byte string */ public com.google.protobuf.ByteString getSinkValueBytes() { java.lang.Object ref = sinkValue_; @@ -2152,6 +2412,11 @@ public com.google.protobuf.ByteString getSinkValueBytes() { /** * optional string sinkValue = 6; + * + * @param value + * the new sink value + * + * @return builder to set sink value */ public Builder setSinkValue(java.lang.String value) { if (value == null) { @@ -2165,6 +2430,8 @@ public Builder setSinkValue(java.lang.String value) { /** * optional string sinkValue = 6; + * + * @return builder to clear sink value */ public Builder clearSinkValue() { bitField0_ = (bitField0_ & ~0x00000020); @@ -2175,6 +2442,11 @@ public Builder clearSinkValue() { /** * optional string sinkValue = 6; + * + * @param value + * new sink value + * + * @return builder to set sink value to passed in parameter */ public Builder setSinkValueBytes(com.google.protobuf.ByteString value) { if (value == null) { @@ -2190,6 +2462,8 @@ public Builder setSinkValueBytes(com.google.protobuf.ByteString value) { /** * optional string loadDate = 7; + * + * @return if it has load date */ public boolean hasLoadDate() { return ((bitField0_ & 0x00000040) == 0x00000040); @@ -2197,6 +2471,8 @@ public boolean hasLoadDate() { /** * optional string loadDate = 7; + * + * @return load date */ public java.lang.String getLoadDate() { java.lang.Object ref = loadDate_; @@ -2214,6 +2490,8 @@ public java.lang.String getLoadDate() { /** * optional string loadDate = 7; + * + * @return load date byte string */ public com.google.protobuf.ByteString getLoadDateBytes() { java.lang.Object ref = loadDate_; @@ -2228,6 +2506,11 @@ public com.google.protobuf.ByteString getLoadDateBytes() { /** * optional string loadDate = 7; + * + * @param value + * the new load date + * + * @return builder to set load date to passed in parameter */ public Builder setLoadDate(java.lang.String value) { if (value == null) { @@ -2241,6 +2524,8 @@ public Builder setLoadDate(java.lang.String value) { /** * optional string loadDate = 7; + * + * @return builder to clear the load date */ public Builder clearLoadDate() { bitField0_ = (bitField0_ & ~0x00000040); @@ -2251,6 +2536,11 @@ public Builder clearLoadDate() { /** * optional string loadDate = 7; + * + * @param value + * the new load date value + * + * @return builder to set the load date to the passed in parameter */ public Builder setLoadDateBytes(com.google.protobuf.ByteString value) { if (value == null) { @@ -2266,6 +2556,8 @@ public Builder setLoadDateBytes(com.google.protobuf.ByteString value) { /** * optional bool badActivity = 8; + * + * @return if it has bad activity */ public boolean hasBadActivity() { return ((bitField0_ & 0x00000080) == 0x00000080); @@ -2273,6 +2565,8 @@ public boolean hasBadActivity() { /** * optional bool badActivity = 8; + * + * @return bad activity indicator */ public boolean getBadActivity() { return badActivity_; @@ -2280,6 +2574,11 @@ public boolean getBadActivity() { /** * optional bool badActivity = 8; + * + * @param value + * bad activity flag + * + * @return builder to set bad activity to passed in paramter */ public Builder setBadActivity(boolean value) { bitField0_ |= 0x00000080; @@ -2290,6 +2589,8 @@ public Builder setBadActivity(boolean value) { /** * optional bool badActivity = 8; + * + * @return builder to clear bad activity indicator */ public Builder clearBadActivity() { bitField0_ = (bitField0_ & ~0x00000080); @@ -2307,6 +2608,8 @@ public Builder clearBadActivity() { * 
                  * a single uuid of an event that contributed to the edge
              *
    + * + * @return if it has uuid */ public boolean hasUuid() { return ((bitField0_ & 0x00000100) == 0x00000100); @@ -2318,6 +2621,8 @@ public boolean hasUuid() { * 
                  * a single uuid of an event that contributed to the edge
              *
    + * + * @return the uuid */ public datawave.edge.protobuf.EdgeData.EdgeValue.UUID getUuid() { if (uuidBuilder_ == null) { @@ -2333,6 +2638,11 @@ public datawave.edge.protobuf.EdgeData.EdgeValue.UUID getUuid() { * 
                  * a single uuid of an event that contributed to the edge
              *
    + * + * @param value + * the new uuid + * + * @return builder to set the new uuid */ public Builder setUuid(datawave.edge.protobuf.EdgeData.EdgeValue.UUID value) { if (uuidBuilder_ == null) { @@ -2354,6 +2664,11 @@ public Builder setUuid(datawave.edge.protobuf.EdgeData.EdgeValue.UUID value) { * 
                  * a single uuid of an event that contributed to the edge
              *
    + * + * @param builderForValue + * the new uuid + * + * @return builder to set uuid to new value */ public Builder setUuid(datawave.edge.protobuf.EdgeData.EdgeValue.UUID.Builder builderForValue) { if (uuidBuilder_ == null) { @@ -2372,6 +2687,11 @@ public Builder setUuid(datawave.edge.protobuf.EdgeData.EdgeValue.UUID.Builder bu * 
                  * a single uuid of an event that contributed to the edge
              *
    + * + * @param value + * uuid to merge in + * + * @return builder to merge in new uuid */ public Builder mergeUuid(datawave.edge.protobuf.EdgeData.EdgeValue.UUID value) { if (uuidBuilder_ == null) { @@ -2394,6 +2714,8 @@ public Builder mergeUuid(datawave.edge.protobuf.EdgeData.EdgeValue.UUID value) { * 
                  * a single uuid of an event that contributed to the edge
              *
    + * + * @return builder to clear uuid */ public Builder clearUuid() { if (uuidBuilder_ == null) { @@ -2412,6 +2734,8 @@ public Builder clearUuid() { * 
                  * a single uuid of an event that contributed to the edge
              *
    + * + * @return builder to build uuid */ public datawave.edge.protobuf.EdgeData.EdgeValue.UUID.Builder getUuidBuilder() { bitField0_ |= 0x00000100; @@ -2425,6 +2749,8 @@ public datawave.edge.protobuf.EdgeData.EdgeValue.UUID.Builder getUuidBuilder() { * 
                  * a single uuid of an event that contributed to the edge
              *
    + * + * @return uuidorbuilder */ public datawave.edge.protobuf.EdgeData.EdgeValue.UUIDOrBuilder getUuidOrBuilder() { if (uuidBuilder_ != null) { @@ -2440,6 +2766,8 @@ public datawave.edge.protobuf.EdgeData.EdgeValue.UUIDOrBuilder getUuidOrBuilder( * 
                  * a single uuid of an event that contributed to the edge
              *
    + * + * @return uuidfieldbuilder */ private com.google.protobuf.SingleFieldBuilder getUuidFieldBuilder() { if (uuidBuilder_ == null) { @@ -2458,6 +2786,8 @@ private com.google.protobuf.SingleFieldBuilder * a single uuid of an event that contributed to the edge in string format * + * + * @return if it has uuid */ public boolean hasUuidString() { return ((bitField0_ & 0x00000200) == 0x00000200); @@ -2469,6 +2799,8 @@ public boolean hasUuidString() { * 
                  * a single uuid of an event that contributed to the edge in string format
              *
    + * + * @return uuid */ public java.lang.String getUuidString() { java.lang.Object ref = uuidString_; @@ -2490,6 +2822,8 @@ public java.lang.String getUuidString() { * 
                  * a single uuid of an event that contributed to the edge in string format
              *
    + * + * @return uuid byte string */ public com.google.protobuf.ByteString getUuidStringBytes() { java.lang.Object ref = uuidString_; @@ -2508,6 +2842,11 @@ public com.google.protobuf.ByteString getUuidStringBytes() { * 
                  * a single uuid of an event that contributed to the edge in string format
              *
    + * + * @param value + * new uuid + * + * @return builder to set uuid to passed in value */ public Builder setUuidString(java.lang.String value) { if (value == null) { @@ -2525,6 +2864,8 @@ public Builder setUuidString(java.lang.String value) { * 
                  * a single uuid of an event that contributed to the edge in string format
              *
    + * + * @return builder to clear uuid */ public Builder clearUuidString() { bitField0_ = (bitField0_ & ~0x00000200); @@ -2539,6 +2880,11 @@ public Builder clearUuidString() { * 
                  * a single uuid of an event that contributed to the edge in string format
              *
    + * + * @param value + * new uuid + * + * @return builder to set the uuid */ public Builder setUuidStringBytes(com.google.protobuf.ByteString value) { if (value == null) { diff --git a/warehouse/core/src/main/java/datawave/edge/util/EdgeValueHelper.java b/warehouse/core/src/main/java/datawave/edge/util/EdgeValueHelper.java index 6f4bc360aba..6c19fc7805f 100644 --- a/warehouse/core/src/main/java/datawave/edge/util/EdgeValueHelper.java +++ b/warehouse/core/src/main/java/datawave/edge/util/EdgeValueHelper.java @@ -120,8 +120,9 @@ public static byte[] getByteArrayForHour(int hour, boolean delete) { } /** - * Creates a Variable length encoded byte array of longs where the elements have the following meaning: *

    + * Creates a Variable length encoded byte array of longs where the elements have the following meaning: + *

    * * 
          * 		0   < 10 sec
@@ -136,6 +137,7 @@ public static byte[] getByteArrayForHour(int hour, boolean delete) {
      * 
    
      * The element at index N will be set if the duration matches the values above, otherwise it will be zero. The element will be set to 1 if deleteRecord is
      * false, else -1.
+     * 
    
      *
      * @param elapsed
      *            - variable length of time elapsed
diff --git a/warehouse/core/src/main/java/datawave/ingest/protobuf/TermWeight.java b/warehouse/core/src/main/java/datawave/ingest/protobuf/TermWeight.java
index be9ec12e58b..5313274d3e1 100644
--- a/warehouse/core/src/main/java/datawave/ingest/protobuf/TermWeight.java
+++ b/warehouse/core/src/main/java/datawave/ingest/protobuf/TermWeight.java
@@ -13,59 +13,90 @@ public interface InfoOrBuilder extends com.google.protobuf.MessageOrBuilder {
         // repeated uint32 termOffset = 3;
         /**
          * repeated uint32 termOffset = 3;
+         *
+         * @return list of term offsets
          */
         java.util.List getTermOffsetList();
 
         /**
          * repeated uint32 termOffset = 3;
+         *
+         * @return the number of term offsets in the list
          */
         int getTermOffsetCount();
 
         /**
          * repeated uint32 termOffset = 3;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the term offset at the provided index
          */
         int getTermOffset(int index);
 
         // repeated uint32 prevSkips = 4;
         /**
          * repeated uint32 prevSkips = 4;
+         *
+         * @return list of previous skips
          */
         java.util.List getPrevSkipsList();
 
         /**
          * repeated uint32 prevSkips = 4;
+         *
+         * @return the prev skips count
          */
         int getPrevSkipsCount();
 
         /**
          * repeated uint32 prevSkips = 4;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the prev skip at the provided index
          */
         int getPrevSkips(int index);
 
         // repeated uint32 score = 5;
         /**
          * repeated uint32 score = 5;
+         *
+         * @return list of scores
          */
         java.util.List getScoreList();
 
         /**
          * repeated uint32 score = 5;
+         *
+         * @return the number of scores in the list
          */
         int getScoreCount();
 
         /**
          * repeated uint32 score = 5;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the score at the provided index
          */
         int getScore(int index);
 
         // optional bool zeroOffsetMatch = 6 [default = true];
         /**
          * optional bool zeroOffsetMatch = 6 [default = true];
+         *
+         * @return if it has a zero offset
          */
         boolean hasZeroOffsetMatch();
 
         /**
          * optional bool zeroOffsetMatch = 6 [default = true];
+         *
+         * @return the zero offset match
          */
         boolean getZeroOffsetMatch();
     }
@@ -237,6 +268,8 @@ public com.google.protobuf.Parser getParserForType() {
 
         /**
          * repeated uint32 termOffset = 3;
+         *
+         * @return list of term offsets
          */
         public java.util.List getTermOffsetList() {
             return termOffset_;
@@ -244,6 +277,8 @@ public java.util.List getTermOffsetList() {
 
         /**
          * repeated uint32 termOffset = 3;
+         *
+         * @return the term offset count
          */
         public int getTermOffsetCount() {
             return termOffset_.size();
@@ -251,6 +286,11 @@ public int getTermOffsetCount() {
 
         /**
          * repeated uint32 termOffset = 3;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the term offset at the provided index
          */
         public int getTermOffset(int index) {
             return termOffset_.get(index);
@@ -262,6 +302,8 @@ public int getTermOffset(int index) {
 
         /**
          * repeated uint32 prevSkips = 4;
+         *
+         * @return the prev skips list
          */
         public java.util.List getPrevSkipsList() {
             return prevSkips_;
@@ -269,6 +311,8 @@ public java.util.List getPrevSkipsList() {
 
         /**
          * repeated uint32 prevSkips = 4;
+         *
+         * @return the prev scips count
          */
         public int getPrevSkipsCount() {
             return prevSkips_.size();
@@ -276,6 +320,11 @@ public int getPrevSkipsCount() {
 
         /**
          * repeated uint32 prevSkips = 4;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the prev skip at the provided index
          */
         public int getPrevSkips(int index) {
             return prevSkips_.get(index);
@@ -294,6 +343,8 @@ public java.util.List getScoreList() {
 
         /**
          * repeated uint32 score = 5;
+         *
+         * @return the size of the score list
          */
         public int getScoreCount() {
             return score_.size();
@@ -301,6 +352,11 @@ public int getScoreCount() {
 
         /**
          * repeated uint32 score = 5;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the score at the provided index
          */
         public int getScore(int index) {
             return score_.get(index);
@@ -312,6 +368,8 @@ public int getScore(int index) {
 
         /**
          * optional bool zeroOffsetMatch = 6 [default = true];
+         *
+         * @return if it has a zero offset
          */
         public boolean hasZeroOffsetMatch() {
             return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -319,6 +377,8 @@ public boolean hasZeroOffsetMatch() {
 
         /**
          * optional bool zeroOffsetMatch = 6 [default = true];
+         *
+         * @return the zero offset match
          */
         public boolean getZeroOffsetMatch() {
             return zeroOffsetMatch_;
@@ -649,6 +709,8 @@ private void ensureTermOffsetIsMutable() {
 
             /**
              * repeated uint32 termOffset = 3;
+             *
+             * @return list of term offsets
              */
             public java.util.List getTermOffsetList() {
                 return java.util.Collections.unmodifiableList(termOffset_);
@@ -656,6 +718,8 @@ public java.util.List getTermOffsetList() {
 
             /**
              * repeated uint32 termOffset = 3;
+             *
+             * @return the number of term offsets in the least
              */
             public int getTermOffsetCount() {
                 return termOffset_.size();
@@ -663,6 +727,11 @@ public int getTermOffsetCount() {
 
             /**
              * repeated uint32 termOffset = 3;
+             *
+             * @param index
+             *            the index
+             *
+             * @return the term offset at the provided index
              */
             public int getTermOffset(int index) {
                 return termOffset_.get(index);
@@ -670,6 +739,13 @@ public int getTermOffset(int index) {
 
             /**
              * repeated uint32 termOffset = 3;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the new term offset
+             *
+             * @return builder to set term offset at provided index
              */
             public Builder setTermOffset(int index, int value) {
                 ensureTermOffsetIsMutable();
@@ -680,6 +756,11 @@ public Builder setTermOffset(int index, int value) {
 
             /**
              * repeated uint32 termOffset = 3;
+             *
+             * @param value
+             *            the new term offset
+             *
+             * @return builder to add new term offset provided
              */
             public Builder addTermOffset(int value) {
                 ensureTermOffsetIsMutable();
@@ -690,6 +771,11 @@ public Builder addTermOffset(int value) {
 
             /**
              * repeated uint32 termOffset = 3;
+             *
+             * @param values
+             *            iterable of term offsets to add to the list
+             *
+             * @return builder to add term offsets provided
              */
             public Builder addAllTermOffset(java.lang.Iterable values) {
                 ensureTermOffsetIsMutable();
@@ -700,6 +786,8 @@ public Builder addAllTermOffset(java.lang.Iterable
 
             /**
              * repeated uint32 termOffset = 3;
+             *
+             * @return builder to clear term offsets
              */
             public Builder clearTermOffset() {
                 termOffset_ = java.util.Collections.emptyList();
@@ -720,6 +808,8 @@ private void ensurePrevSkipsIsMutable() {
 
             /**
              * repeated uint32 prevSkips = 4;
+             *
+             * @return the prev skip list
              */
             public java.util.List getPrevSkipsList() {
                 return java.util.Collections.unmodifiableList(prevSkips_);
@@ -727,6 +817,8 @@ public java.util.List getPrevSkipsList() {
 
             /**
              * repeated uint32 prevSkips = 4;
+             *
+             * @return the number of items in the prev skip list
              */
             public int getPrevSkipsCount() {
                 return prevSkips_.size();
@@ -734,6 +826,11 @@ public int getPrevSkipsCount() {
 
             /**
              * repeated uint32 prevSkips = 4;
+             *
+             * @param index
+             *            the index
+             *
+             * @return the prev skip at the provided index
              */
             public int getPrevSkips(int index) {
                 return prevSkips_.get(index);
@@ -741,6 +838,13 @@ public int getPrevSkips(int index) {
 
             /**
              * repeated uint32 prevSkips = 4;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            new prev skip value
+             *
+             * @return builder to set the prev skips value at the provided index
              */
             public Builder setPrevSkips(int index, int value) {
                 ensurePrevSkipsIsMutable();
@@ -751,6 +855,11 @@ public Builder setPrevSkips(int index, int value) {
 
             /**
              * repeated uint32 prevSkips = 4;
+             *
+             * @param value
+             *            new prev skips value to add
+             *
+             * @return builder to add a prev skips value
              */
             public Builder addPrevSkips(int value) {
                 ensurePrevSkipsIsMutable();
@@ -761,6 +870,11 @@ public Builder addPrevSkips(int value) {
 
             /**
              * repeated uint32 prevSkips = 4;
+             *
+             * @param values
+             *            iterable of perv skips to add
+             *
+             * @return builder to add iterable of values
              */
             public Builder addAllPrevSkips(java.lang.Iterable values) {
                 ensurePrevSkipsIsMutable();
@@ -771,6 +885,8 @@ public Builder addAllPrevSkips(java.lang.Iterable v
 
             /**
              * repeated uint32 prevSkips = 4;
+             *
+             * @return builder to clear prev skips
              */
             public Builder clearPrevSkips() {
                 prevSkips_ = java.util.Collections.emptyList();
@@ -791,6 +907,8 @@ private void ensureScoreIsMutable() {
 
             /**
              * repeated uint32 score = 5;
+             *
+             * @return list of scores
              */
             public java.util.List getScoreList() {
                 return java.util.Collections.unmodifiableList(score_);
@@ -805,6 +923,11 @@ public int getScoreCount() {
 
             /**
              * repeated uint32 score = 5;
+             *
+             * @param index
+             *            the index
+             *
+             * @return the score at the provided index
              */
             public int getScore(int index) {
                 return score_.get(index);
@@ -812,6 +935,13 @@ public int getScore(int index) {
 
             /**
              * repeated uint32 score = 5;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the new score value
+             *
+             * @return builder to set the score at the provided index
              */
             public Builder setScore(int index, int value) {
                 ensureScoreIsMutable();
@@ -822,6 +952,11 @@ public Builder setScore(int index, int value) {
 
             /**
              * repeated uint32 score = 5;
+             *
+             * @param value
+             *            the score to add
+             *
+             * @return return a builder to add the provided score
              */
             public Builder addScore(int value) {
                 ensureScoreIsMutable();
@@ -832,6 +967,11 @@ public Builder addScore(int value) {
 
             /**
              * repeated uint32 score = 5;
+             *
+             * @param values
+             *            iterable of scores to add
+             *
+             * @return builder to add the iterable of scores provided
              */
             public Builder addAllScore(java.lang.Iterable values) {
                 ensureScoreIsMutable();
@@ -842,6 +982,8 @@ public Builder addAllScore(java.lang.Iterable value
 
             /**
              * repeated uint32 score = 5;
+             *
+             * @return builder to clear the scores
              */
             public Builder clearScore() {
                 score_ = java.util.Collections.emptyList();
@@ -855,6 +997,8 @@ public Builder clearScore() {
 
             /**
              * optional bool zeroOffsetMatch = 6 [default = true];
+             *
+             * @return if it has zero offset match
              */
             public boolean hasZeroOffsetMatch() {
                 return ((bitField0_ & 0x00000008) == 0x00000008);
@@ -862,6 +1006,8 @@ public boolean hasZeroOffsetMatch() {
 
             /**
              * optional bool zeroOffsetMatch = 6 [default = true];
+             *
+             * @return the zero offset match
              */
             public boolean getZeroOffsetMatch() {
                 return zeroOffsetMatch_;
@@ -869,6 +1015,11 @@ public boolean getZeroOffsetMatch() {
 
             /**
              * optional bool zeroOffsetMatch = 6 [default = true];
+             *
+             * @param value
+             *            zero offset match value
+             *
+             * @return builder to set the zero offset match
              */
             public Builder setZeroOffsetMatch(boolean value) {
                 bitField0_ |= 0x00000008;
@@ -879,6 +1030,8 @@ public Builder setZeroOffsetMatch(boolean value) {
 
             /**
              * optional bool zeroOffsetMatch = 6 [default = true];
+             *
+             * @return builder to clear zero offset match
              */
             public Builder clearZeroOffsetMatch() {
                 bitField0_ = (bitField0_ & ~0x00000008);
diff --git a/warehouse/core/src/main/java/datawave/ingest/protobuf/Uid.java b/warehouse/core/src/main/java/datawave/ingest/protobuf/Uid.java
index db4bb08ae8b..d1c3e42f2be 100644
--- a/warehouse/core/src/main/java/datawave/ingest/protobuf/Uid.java
+++ b/warehouse/core/src/main/java/datawave/ingest/protobuf/Uid.java
@@ -13,64 +13,100 @@ public interface ListOrBuilder extends com.google.protobuf.MessageOrBuilder {
         // required bool IGNORE = 1;
         /**
          * required bool IGNORE = 1;
+         *
+         * @return if it has IGNORE
          */
         boolean hasIGNORE();
 
         /**
          * required bool IGNORE = 1;
+         *
+         * @return IGNORE flag
          */
         boolean getIGNORE();
 
         // required uint64 COUNT = 2;
         /**
          * required uint64 COUNT = 2;
+         *
+         * @return if it has COUNT
          */
         boolean hasCOUNT();
 
         /**
          * required uint64 COUNT = 2;
+         *
+         * @return the COUNT
          */
         long getCOUNT();
 
         // repeated string UID = 3;
         /**
          * repeated string UID = 3;
+         *
+         * @return list of UIDs
          */
         java.util.List getUIDList();
 
         /**
          * repeated string UID = 3;
+         *
+         * @return the number of UIDs in the list
          */
         int getUIDCount();
 
         /**
          * repeated string UID = 3;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the UID at the provided index
          */
         java.lang.String getUID(int index);
 
         /**
          * repeated string UID = 3;
+         *
+         * @param index
+         *            the index
+         *
+         * @return uid byte string
          */
         com.google.protobuf.ByteString getUIDBytes(int index);
 
         // repeated string REMOVEDUID = 4;
         /**
          * repeated string REMOVEDUID = 4;
+         *
+         * @return list of removed uids
          */
         java.util.List getREMOVEDUIDList();
 
         /**
          * repeated string REMOVEDUID = 4;
+         *
+         * @return number of removed uids in the list
          */
         int getREMOVEDUIDCount();
 
         /**
          * repeated string REMOVEDUID = 4;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the removed uid at the provided index
          */
         java.lang.String getREMOVEDUID(int index);
 
         /**
          * repeated string REMOVEDUID = 4;
+         *
+         * @param index
+         *            the index
+         *
+         * @return removed uid byte string at the provided index
          */
         com.google.protobuf.ByteString getREMOVEDUIDBytes(int index);
     }
@@ -197,6 +233,8 @@ public com.google.protobuf.Parser getParserForType() {
 
         /**
          * required bool IGNORE = 1;
+         *
+         * @return if it has IGNORE
          */
         public boolean hasIGNORE() {
             return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -204,6 +242,8 @@ public boolean hasIGNORE() {
 
         /**
          * required bool IGNORE = 1;
+         *
+         * @return IGNORE flag
          */
         public boolean getIGNORE() {
             return iGNORE_;
@@ -215,6 +255,8 @@ public boolean getIGNORE() {
 
         /**
          * required uint64 COUNT = 2;
+         *
+         * @return if it has COUNT
          */
         public boolean hasCOUNT() {
             return ((bitField0_ & 0x00000002) == 0x00000002);
@@ -222,6 +264,8 @@ public boolean hasCOUNT() {
 
         /**
          * required uint64 COUNT = 2;
+         *
+         * @return the COUNT
          */
         public long getCOUNT() {
             return cOUNT_;
@@ -233,6 +277,8 @@ public long getCOUNT() {
 
         /**
          * repeated string UID = 3;
+         *
+         * @return the uid list
          */
         public java.util.List getUIDList() {
             return uID_;
@@ -240,6 +286,8 @@ public java.util.List getUIDList() {
 
         /**
          * repeated string UID = 3;
+         *
+         * @return the uid count
          */
         public int getUIDCount() {
             return uID_.size();
@@ -247,6 +295,11 @@ public int getUIDCount() {
 
         /**
          * repeated string UID = 3;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the uid at the provided index
          */
         public java.lang.String getUID(int index) {
             return uID_.get(index);
@@ -254,6 +307,11 @@ public java.lang.String getUID(int index) {
 
         /**
          * repeated string UID = 3;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the uid byte string at the provided index
          */
         public com.google.protobuf.ByteString getUIDBytes(int index) {
             return uID_.getByteString(index);
@@ -265,6 +323,8 @@ public com.google.protobuf.ByteString getUIDBytes(int index) {
 
         /**
          * repeated string REMOVEDUID = 4;
+         *
+         * @return list of removed uids
          */
         public java.util.List getREMOVEDUIDList() {
             return rEMOVEDUID_;
@@ -272,6 +332,8 @@ public java.util.List getREMOVEDUIDList() {
 
         /**
          * repeated string REMOVEDUID = 4;
+         *
+         * @return the size of the removed uid list
          */
         public int getREMOVEDUIDCount() {
             return rEMOVEDUID_.size();
@@ -279,6 +341,11 @@ public int getREMOVEDUIDCount() {
 
         /**
          * repeated string REMOVEDUID = 4;
+         *
+         * @param index
+         *            the index
+         *
+         * @return removed uid at the provided index
          */
         public java.lang.String getREMOVEDUID(int index) {
             return rEMOVEDUID_.get(index);
@@ -286,6 +353,11 @@ public java.lang.String getREMOVEDUID(int index) {
 
         /**
          * repeated string REMOVEDUID = 4;
+         *
+         * @param index
+         *            the index
+         *
+         * @return removed uid byte string at provided index
          */
         public com.google.protobuf.ByteString getREMOVEDUIDBytes(int index) {
             return rEMOVEDUID_.getByteString(index);
@@ -612,6 +684,8 @@ public Builder mergeFrom(com.google.protobuf.CodedInputStream input, com.google.
 
             /**
              * required bool IGNORE = 1;
+             *
+             * @return if it has INGORE flag
              */
             public boolean hasIGNORE() {
                 return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -619,6 +693,8 @@ public boolean hasIGNORE() {
 
             /**
              * required bool IGNORE = 1;
+             *
+             * @return the IGNORE flag
              */
             public boolean getIGNORE() {
                 return iGNORE_;
@@ -626,6 +702,11 @@ public boolean getIGNORE() {
 
             /**
              * required bool IGNORE = 1;
+             *
+             * @param value
+             *            the IGNORE value
+             *
+             * @return builder to set IGNORE flag to provided value
              */
             public Builder setIGNORE(boolean value) {
                 bitField0_ |= 0x00000001;
@@ -636,6 +717,8 @@ public Builder setIGNORE(boolean value) {
 
             /**
              * required bool IGNORE = 1;
+             *
+             * @return builder to clear IGNORE flag
              */
             public Builder clearIGNORE() {
                 bitField0_ = (bitField0_ & ~0x00000001);
@@ -649,6 +732,8 @@ public Builder clearIGNORE() {
 
             /**
              * required uint64 COUNT = 2;
+             *
+             * @return if it has COUNT set
              */
             public boolean hasCOUNT() {
                 return ((bitField0_ & 0x00000002) == 0x00000002);
@@ -656,6 +741,8 @@ public boolean hasCOUNT() {
 
             /**
              * required uint64 COUNT = 2;
+             *
+             * @return the COUNT
              */
             public long getCOUNT() {
                 return cOUNT_;
@@ -663,6 +750,11 @@ public long getCOUNT() {
 
             /**
              * required uint64 COUNT = 2;
+             *
+             * @param value
+             *            the new COUNT value
+             *
+             * @return builder to set COUNT to provided value
              */
             public Builder setCOUNT(long value) {
                 bitField0_ |= 0x00000002;
@@ -673,6 +765,8 @@ public Builder setCOUNT(long value) {
 
             /**
              * required uint64 COUNT = 2;
+             *
+             * @return builder to clear the COUNT
              */
             public Builder clearCOUNT() {
                 bitField0_ = (bitField0_ & ~0x00000002);
@@ -693,6 +787,8 @@ private void ensureUIDIsMutable() {
 
             /**
              * repeated string UID = 3;
+             *
+             * @return the uid list
              */
             public java.util.List getUIDList() {
                 return java.util.Collections.unmodifiableList(uID_);
@@ -700,6 +796,8 @@ public java.util.List getUIDList() {
 
             /**
              * repeated string UID = 3;
+             *
+             * @return the uid list size
              */
             public int getUIDCount() {
                 return uID_.size();
@@ -707,6 +805,11 @@ public int getUIDCount() {
 
             /**
              * repeated string UID = 3;
+             *
+             * @param index
+             *            the index
+             *
+             * @return the uid at the provided index
              */
             public java.lang.String getUID(int index) {
                 return uID_.get(index);
@@ -714,6 +817,11 @@ public java.lang.String getUID(int index) {
 
             /**
              * repeated string UID = 3;
+             *
+             * @param index
+             *            the index
+             *
+             * @return uid byte string at the provided index
              */
             public com.google.protobuf.ByteString getUIDBytes(int index) {
                 return uID_.getByteString(index);
@@ -721,6 +829,13 @@ public com.google.protobuf.ByteString getUIDBytes(int index) {
 
             /**
              * repeated string UID = 3;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the new uid value
+             *
+             * @return builder to set the uid at the proved index
              */
             public Builder setUID(int index, java.lang.String value) {
                 if (value == null) {
@@ -734,6 +849,11 @@ public Builder setUID(int index, java.lang.String value) {
 
             /**
              * repeated string UID = 3;
+             *
+             * @param value
+             *            the new uid to add to the list
+             *
+             * @return builder to add the provided uid
              */
             public Builder addUID(java.lang.String value) {
                 if (value == null) {
@@ -747,6 +867,11 @@ public Builder addUID(java.lang.String value) {
 
             /**
              * repeated string UID = 3;
+             *
+             * @param values
+             *            iterable of uids to add to the list
+             *
+             * @return builder to add the new uid values
              */
             public Builder addAllUID(java.lang.Iterable values) {
                 ensureUIDIsMutable();
@@ -757,6 +882,8 @@ public Builder addAllUID(java.lang.Iterable values) {
 
             /**
              * repeated string UID = 3;
+             *
+             * @return builder to clear uid list
              */
             public Builder clearUID() {
                 uID_ = com.google.protobuf.LazyStringArrayList.EMPTY;
@@ -767,6 +894,11 @@ public Builder clearUID() {
 
             /**
              * repeated string UID = 3;
+             *
+             * @param value
+             *            the uid byte string
+             *
+             * @return builder to add uid byte string provided
              */
             public Builder addUIDBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -790,6 +922,8 @@ private void ensureREMOVEDUIDIsMutable() {
 
             /**
              * repeated string REMOVEDUID = 4;
+             *
+             * @return the removed uid list
              */
             public java.util.List getREMOVEDUIDList() {
                 return java.util.Collections.unmodifiableList(rEMOVEDUID_);
@@ -797,6 +931,8 @@ public java.util.List getREMOVEDUIDList() {
 
             /**
              * repeated string REMOVEDUID = 4;
+             *
+             * @return the size of the removed uid list
              */
             public int getREMOVEDUIDCount() {
                 return rEMOVEDUID_.size();
@@ -804,6 +940,11 @@ public int getREMOVEDUIDCount() {
 
             /**
              * repeated string REMOVEDUID = 4;
+             *
+             * @param index
+             *            the index
+             *
+             * @return get the removed uid at the provided index
              */
             public java.lang.String getREMOVEDUID(int index) {
                 return rEMOVEDUID_.get(index);
@@ -811,6 +952,11 @@ public java.lang.String getREMOVEDUID(int index) {
 
             /**
              * repeated string REMOVEDUID = 4;
+             *
+             * @param index
+             *            the index
+             *
+             * @return byte string ofremoved uid at the provided index
              */
             public com.google.protobuf.ByteString getREMOVEDUIDBytes(int index) {
                 return rEMOVEDUID_.getByteString(index);
@@ -818,6 +964,13 @@ public com.google.protobuf.ByteString getREMOVEDUIDBytes(int index) {
 
             /**
              * repeated string REMOVEDUID = 4;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the new removed uid value
+             *
+             * @return builder to set the removed uid at the provided index
              */
             public Builder setREMOVEDUID(int index, java.lang.String value) {
                 if (value == null) {
@@ -831,6 +984,11 @@ public Builder setREMOVEDUID(int index, java.lang.String value) {
 
             /**
              * repeated string REMOVEDUID = 4;
+             *
+             * @param value
+             *            the removed uid value to add to the list
+             *
+             * @return builder to add removed uid provided
              */
             public Builder addREMOVEDUID(java.lang.String value) {
                 if (value == null) {
@@ -844,6 +1002,11 @@ public Builder addREMOVEDUID(java.lang.String value) {
 
             /**
              * repeated string REMOVEDUID = 4;
+             *
+             * @param values
+             *            iterator of uids to add to removed uid list
+             *
+             * @return builder to add all values to removed uid list
              */
             public Builder addAllREMOVEDUID(java.lang.Iterable values) {
                 ensureREMOVEDUIDIsMutable();
@@ -854,6 +1017,8 @@ public Builder addAllREMOVEDUID(java.lang.Iterable values) {
 
             /**
              * repeated string REMOVEDUID = 4;
+             *
+             * @return builder to clear removed uid list
              */
             public Builder clearREMOVEDUID() {
                 rEMOVEDUID_ = com.google.protobuf.LazyStringArrayList.EMPTY;
@@ -864,6 +1029,11 @@ public Builder clearREMOVEDUID() {
 
             /**
              * repeated string REMOVEDUID = 4;
+             *
+             * @param value
+             *            byte string value
+             *
+             * @return builder to add removed uid byte string value
              */
             public Builder addREMOVEDUIDBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
diff --git a/warehouse/core/src/main/java/datawave/ingest/util/cache/watch/FileRuleWatcher.java b/warehouse/core/src/main/java/datawave/ingest/util/cache/watch/FileRuleWatcher.java
index e786683a80b..7981d1f56ff 100644
--- a/warehouse/core/src/main/java/datawave/ingest/util/cache/watch/FileRuleWatcher.java
+++ b/warehouse/core/src/main/java/datawave/ingest/util/cache/watch/FileRuleWatcher.java
@@ -31,7 +31,7 @@
 import datawave.iterators.filter.ageoff.FilterRule;
 
 /**
- *
+ * File Rule Watch
  */
 public class FileRuleWatcher extends FileSystemWatcher> {
 
@@ -41,9 +41,13 @@ public class FileRuleWatcher extends FileSystemWatcher> {
 
     /**
      * @param fs
+     *            file system
      * @param filePath
+     *            path to the file
      * @param configuredDiff
+     *            configured diff
      * @throws IOException
+     *             if there is a problem reading the file
      */
     public FileRuleWatcher(FileSystem fs, Path filePath, long configuredDiff) throws IOException {
         this(fs, filePath, configuredDiff, null);
@@ -51,10 +55,15 @@ public FileRuleWatcher(FileSystem fs, Path filePath, long configuredDiff) throws
 
     /**
      * @param fs
+     *            file system
      * @param filePath
+     *            path to the file
      * @param configuredDiff
+     *            configured diff
      * @param iterEnv
+     *            iterator environment
      * @throws IOException
+     *             if there is a problem reading the file
      */
     public FileRuleWatcher(FileSystem fs, Path filePath, long configuredDiff, IteratorEnvironment iterEnv) throws IOException {
         super(fs, filePath, configuredDiff);
@@ -63,8 +72,11 @@ public FileRuleWatcher(FileSystem fs, Path filePath, long configuredDiff, Iterat
 
     /**
      * @param filePath
+     *            path to the file
      * @param configuredDiff
+     *            configured diff
      * @throws IOException
+     *             if there is an error reading the file
      */
     public FileRuleWatcher(Path filePath, long configuredDiff) throws IOException {
         this(filePath, configuredDiff, null);
@@ -72,9 +84,13 @@ public FileRuleWatcher(Path filePath, long configuredDiff) throws IOException {
 
     /**
      * @param filePath
+     *            the path to the file
      * @param configuredDiff
+     *            configured diff
      * @param iterEnv
+     *            iterator environment
      * @throws IOException
+     *             if there is an error reading the file
      */
     public FileRuleWatcher(Path filePath, long configuredDiff, IteratorEnvironment iterEnv) throws IOException {
         super(filePath.getFileSystem(new Configuration()), filePath, configuredDiff);
diff --git a/warehouse/core/src/main/java/datawave/ingest/util/cache/watch/FileSystemWatcher.java b/warehouse/core/src/main/java/datawave/ingest/util/cache/watch/FileSystemWatcher.java
index 10834a3d561..ffbf077effa 100644
--- a/warehouse/core/src/main/java/datawave/ingest/util/cache/watch/FileSystemWatcher.java
+++ b/warehouse/core/src/main/java/datawave/ingest/util/cache/watch/FileSystemWatcher.java
@@ -11,7 +11,7 @@
 import org.apache.log4j.Logger;
 
 /**
- *
+ * File System Watch
  */
 public abstract class FileSystemWatcher extends Reloadable {
 
@@ -34,9 +34,12 @@ public FileSystemWatcher(FileSystem fs, Path filePath, long configuredDiff) thro
 
     /**
      * @param filePath
-     * @return
+     *            file path
+     * @return checksum
      * @throws NoSuchAlgorithmException
+     *             if the algorithm doesn't excist
      * @throws IOException
+     *             when an IO error occurs
      */
     private byte[] checksumFile(Path filePath) throws NoSuchAlgorithmException, IOException {
 
diff --git a/warehouse/core/src/main/java/datawave/ingest/util/io/GzipDetectionUtil.java b/warehouse/core/src/main/java/datawave/ingest/util/io/GzipDetectionUtil.java
index 67d87d9c6f0..234b6b787d0 100644
--- a/warehouse/core/src/main/java/datawave/ingest/util/io/GzipDetectionUtil.java
+++ b/warehouse/core/src/main/java/datawave/ingest/util/io/GzipDetectionUtil.java
@@ -19,6 +19,7 @@ public class GzipDetectionUtil {
      *            the file to check for GZIP Compression.
      * @return whether or not the given File is GZIP compressed.
      * @throws IOException
+     *             if there is an error reading the file
      */
     public static boolean isCompressed(final File file) throws IOException {
         final FileInputStream fis = new FileInputStream(file);
@@ -44,6 +45,9 @@ public static boolean isCompressed(final File file) throws IOException {
      * @param inputStream
      *            the {@link InputStream} to check.
      * @return an {@link InputStream} to read from.
+     *
+     * @throws IOException
+     *             if there is an error reading the file
      */
     public static InputStream decompressTream(InputStream inputStream) throws IOException {
         PushbackInputStream pushBack = new PushbackInputStream(inputStream, 2); // need to check first two bytes
diff --git a/warehouse/core/src/main/java/datawave/iterators/filter/ConfigurableAgeOffFilter.java b/warehouse/core/src/main/java/datawave/iterators/filter/ConfigurableAgeOffFilter.java
index a2fdde80d9b..56b0db040bd 100644
--- a/warehouse/core/src/main/java/datawave/iterators/filter/ConfigurableAgeOffFilter.java
+++ b/warehouse/core/src/main/java/datawave/iterators/filter/ConfigurableAgeOffFilter.java
@@ -52,15 +52,17 @@
  * {@code AgeOffConfigParams.FILTER_CONFIG} {@code AgeOffConfigParams.TTL_SHORT_CIRCUIT} can be optionally used to short circuit invoking the filters and will
  * allow all records younger thatn that interval to be passed through. The units definition is used for both {@code AgeOffConfigParams.TTL} and
  * {@code AgeOffConfigParams.TTL_SHORT_CIRCUIT}.
- *
+ * 
    
  *
  * 
    
  * The filtering rules are stored in a configuration file, which may be stored in the local file system, or in HDFS. If it is stored in the local filesystem,
  * then it must be available on all of the tablet servers' filesystems. The configuration file should be specified as a full URL such as
  * {@code file:///opt/accumulo/config/configFilter.xml} or {@code hdfs://config/filters/configFilter.xml}.
+ * 
    
  *
  * 
    
  * The TTL Units may be the following values:
+ * 
    
  * 
      
  * 
    • {@code ms} - milliseconds
  * 
    • {@code s} - seconds
@@ -71,8 +73,7 @@
  *
  * 
      
  * Sample Configuration File:
- *
- * 
      
+ * 
      
  *
  * 
        * <ageoffConfiguration>
@@ -208,6 +209,11 @@ public SortedKeyValueIterator deepCopy(IteratorEnvironment env) {
 
     /**
      * initialize the object via some other configurable age off filter.
+     *
+     * @param other
+     *            another filter to base this one off
+     *
+     * @return the configurable age off filter
      */
     protected ConfigurableAgeOffFilter initialize(ConfigurableAgeOffFilter other) {
 
@@ -230,11 +236,20 @@ protected ConfigurableAgeOffFilter initialize(ConfigurableAgeOffFilter other) {
      * Initialize this object with a set of string parameters representing the configuration options for this iterator.
      *
      * @param ttl
+     *            time to live
      * @param ttlUnits
+     *            time to live units
+     *
+     * @param ttlShortCircuitStr
+     *            time to live short circuit string
      * @param scanStart
+     *            scan start time
      * @param fileName
+     *            file name
      * @throws IOException
+     *             if error reading the file
      * @throws IllegalArgumentException
+     *             if illegal arguments passed
      */
     protected void initialize(final String ttl, final String ttlUnits, final String ttlShortCircuitStr, final long scanStart, final String fileName)
                     throws IllegalArgumentException, IOException {
@@ -297,9 +312,13 @@ protected void initialize(final String ttl, final String ttlUnits, final String
     /**
      * Used to initialize the default parameters used by this implementation of {@code Filter}, as well as the sub-filters specified in the configuration file.
      *
+     * @param source
+     *            the source key values
      * @param options
      *            {@code Map} object contain the configuration parameters for this {@code Filter} implementation. The parameters required are
      *            specified in the {@code AgeOffConfigParams.TTL}, {@code AgeOffConfigParams.TTL_UNITS}, and {@code AgeOffConfigParams.FILTER_CONFIG}.
+     * @param env
+     *            the iterator environment
      * @see org.apache.accumulo.core.iterators.Filter#init(SortedKeyValueIterator, Map, IteratorEnvironment)
      */
     @Override
@@ -330,7 +349,9 @@ public void init(SortedKeyValueIterator source, Map op
      * 
    
      *
      * @param options
+     *            map of options
      * @param env
+     *            the iterator environment
      * @return true only if we should disable filtering
      */
     private boolean shouldDisableForNonFullCompaction(Map options, IteratorEnvironment env) {
@@ -370,7 +391,9 @@ private boolean shouldDisableForNonFullCompaction(Map options, It
      * 
      *
      * @param options
+     *            map of options
      * @param env
+     *            the iterator environment
      * @return true only if we should disable filtering
      */
     private boolean shouldDisableForNonUserCompaction(Map options, IteratorEnvironment env) {
@@ -402,12 +425,13 @@ private boolean shouldDisableForNonUserCompaction(Map options, It
     }
 
     /**
-     * This method instantiates the the necessary implementations of the {@code Filter} interface, as they are defined in the configuration file specified by
+     * This method instantiates the necessary implementations of the {@code Filter} interface, as they are defined in the configuration file specified by
      * {@code this.filename}.
      *
      * @throws IllegalArgumentException
      *             if there is an error in the configuration file
      * @throws IOException
+     *             if there is an error reading the configuration file
      */
     private void initFilterRules() throws IllegalArgumentException, IOException {
         // filename
diff --git a/warehouse/core/src/main/java/datawave/iterators/filter/RegexFilterBase.java b/warehouse/core/src/main/java/datawave/iterators/filter/RegexFilterBase.java
index 52e18edfb82..ba94dcbb953 100644
--- a/warehouse/core/src/main/java/datawave/iterators/filter/RegexFilterBase.java
+++ b/warehouse/core/src/main/java/datawave/iterators/filter/RegexFilterBase.java
@@ -109,6 +109,7 @@ public void init(FilterOptions options) {
      * @param options
      *            {@code Map} object containing the TTL, TTL_UNITS, and MATCHPATTERN for the filter rule.
      * @param iterEnv
+     *            iterator environment
      * @see datawave.iterators.filter.AgeOffConfigParams
      */
     @Override
diff --git a/warehouse/core/src/main/java/datawave/iterators/filter/TokenFilterBase.java b/warehouse/core/src/main/java/datawave/iterators/filter/TokenFilterBase.java
index ddb99fda6b0..dd963b4b557 100644
--- a/warehouse/core/src/main/java/datawave/iterators/filter/TokenFilterBase.java
+++ b/warehouse/core/src/main/java/datawave/iterators/filter/TokenFilterBase.java
@@ -105,11 +105,12 @@ public void init(FilterOptions options) {
     }
 
     /**
-     * Required by the {@code FilterRule} interface. Used to initialize the the {@code FilterRule} implementation
+     * Required by the {@code FilterRule} interface. Used to initialize the {@code FilterRule} implementation
      *
      * @param options
      *            {@code Map} object containing the TTL, TTL_UNITS, and MATCHPATTERN for the filter rule.
      * @param iterEnv
+     *            iterator environment
      * @see datawave.iterators.filter.AgeOffConfigParams
      */
     @Override
diff --git a/warehouse/core/src/main/java/datawave/iterators/filter/TokenSpecParser.java b/warehouse/core/src/main/java/datawave/iterators/filter/TokenSpecParser.java
index 603ae37cf8a..a46c0893109 100644
--- a/warehouse/core/src/main/java/datawave/iterators/filter/TokenSpecParser.java
+++ b/warehouse/core/src/main/java/datawave/iterators/filter/TokenSpecParser.java
@@ -9,12 +9,24 @@
 
 public abstract class TokenSpecParser {
     /**
-     * Add a new token with its TTL to the the structure.
+     * Add a new token with its TTL to the structure.
+     *
+     * @param token
+     *            byte array token
+     * @param ttl
+     *            time to live
+     *
+     * @return a TokenSpecParser child object
      */
     public abstract B addToken(byte[] token, long ttl);
 
     /**
      * Parse additional token configurations from a string.
+     *
+     * @param configuration
+     *            figuration string for the parser
+     *
+     * @return TokenSpecParser child object
      */
     public B parse(String configuration) {
         ParserState parser = new ParserState(configuration);
@@ -129,6 +141,8 @@ private List tokenize(String input) {
 
         /**
          * Return the next token without advancing.
+         *
+         * @return the next token
          */
         protected ParseToken peek() {
             if (nextTokenPos >= parseTokens.size()) {
@@ -139,6 +153,11 @@ protected ParseToken peek() {
 
         /**
          * Consume the next token, assuming it's of the specified type, and return its content.
+         *
+         * @param type
+         *            the parse token type
+         *
+         * @return the content of the next token
          */
         protected String expect(ParseTokenType type) {
             ParseToken next = peek();
@@ -153,6 +172,9 @@ protected String expect(ParseTokenType type) {
 
         /**
          * Parse the entire input and add it to the TtlTrieBuilder.
+         *
+         * @param builder
+         *            token spec parser
          */
         protected void parseTo(TokenSpecParser builder) {
             ParseToken initialToken;
@@ -175,6 +197,8 @@ protected void parseTo(TokenSpecParser builder) {
 
         /**
          * Read a string literal.
+         *
+         * @return the parsed string literal
          */
         protected String parseStrliteral() {
             ParseToken token = peek();
diff --git a/warehouse/core/src/main/java/datawave/iterators/filter/TokenTtlTrie.java b/warehouse/core/src/main/java/datawave/iterators/filter/TokenTtlTrie.java
index 691e85040fd..7967e24e9df 100644
--- a/warehouse/core/src/main/java/datawave/iterators/filter/TokenTtlTrie.java
+++ b/warehouse/core/src/main/java/datawave/iterators/filter/TokenTtlTrie.java
@@ -38,6 +38,11 @@ public int size() {
 
     /**
      * Scan the specified string for tokens, returning the ttl of the best priority token found, or null if no tokens were found.
+     *
+     * @param rawString
+     *            the raw string
+     *
+     * @return the ttl for the best priority token found
      */
     public Long scan(byte[] rawString) {
         int bestPriority = Integer.MAX_VALUE;
@@ -111,6 +116,11 @@ public int size() {
 
         /**
          * Set the delimiter set.
+         *
+         * @param delimiters
+         *            the delimiter set
+         *
+         * @return builder to set the delimiters
          */
         public Builder setDelimiters(byte[] delimiters) {
             this.delimiters.clear();
@@ -122,6 +132,13 @@ public Builder setDelimiters(byte[] delimiters) {
 
         /**
          * Add a token to the TtlTrie under construction, along with the TTL value the specified token should be associated with.
+         *
+         * @param token
+         *            the token to add
+         * @param ttl
+         *            time to live
+         *
+         * @return builder to add provided token
          */
         @Override
         public Builder addToken(byte[] token, long ttl) {
diff --git a/warehouse/core/src/main/java/datawave/iterators/filter/TokenizingFilterBase.java b/warehouse/core/src/main/java/datawave/iterators/filter/TokenizingFilterBase.java
index b3d4cb45b2d..b2078cd0502 100644
--- a/warehouse/core/src/main/java/datawave/iterators/filter/TokenizingFilterBase.java
+++ b/warehouse/core/src/main/java/datawave/iterators/filter/TokenizingFilterBase.java
@@ -60,6 +60,11 @@ public abstract class TokenizingFilterBase extends AppliedRule {
     /**
      * Return a list of delimiters for scans. While the default is to pull this information out of the {@code <delimiters>} tag in the configuration,
      * subclasses may wish to override this to provide fixed delimiter sets.
+     *
+     * @param options
+     *            filter options
+     *
+     * @return list of delimiters for scans
      */
     public byte[] getDelimiters(FilterOptions options) {
         String delimiters = options.getOption(DELIMITERS_TAG);
diff --git a/warehouse/core/src/main/java/datawave/iterators/filter/ageoff/AppliedRule.java b/warehouse/core/src/main/java/datawave/iterators/filter/ageoff/AppliedRule.java
index 9784525cd57..18919877b2f 100644
--- a/warehouse/core/src/main/java/datawave/iterators/filter/ageoff/AppliedRule.java
+++ b/warehouse/core/src/main/java/datawave/iterators/filter/ageoff/AppliedRule.java
@@ -59,7 +59,14 @@ public void init(FilterOptions options, IteratorEnvironment iterEnv) {
         this.iterEnv = iterEnv;
     }
 
-    /** Perform initialization in support of a deepCopy, copying any expensive state from the parent. */
+    /**
+     * Perform initialization in support of a deepCopy, copying any expensive state from the parent.
+     *
+     * @param newOptions
+     *            filter options
+     * @param parentCopy
+     *            appolied rule of parent copy
+     **/
     protected void deepCopyInit(FilterOptions newOptions, AppliedRule parentCopy) {
         init(newOptions, iterEnv);
     }
@@ -115,7 +122,10 @@ public FilterRule deepCopy(AgeOffPeriod period, IteratorEnvironment iterEnv) {
 
     /**
      * @param scanStart
-     * @return
+     *            scan start time
+     * @param iterEnv
+     *            iterator environment
+     * @return new filter rule
      */
     public FilterRule deepCopy(long scanStart, IteratorEnvironment iterEnv) {
         AppliedRule newFilter;
@@ -136,7 +146,7 @@ public FilterRule deepCopy(long scanStart, IteratorEnvironment iterEnv) {
     }
 
     /**
-     * @return
+     * @return the age off period
      */
     protected AgeOffPeriod getPeriod() {
         return ageOffPeriod;
diff --git a/warehouse/core/src/main/java/datawave/iterators/filter/ageoff/DataTypeAgeOffFilter.java b/warehouse/core/src/main/java/datawave/iterators/filter/ageoff/DataTypeAgeOffFilter.java
index 8acf643d18b..74963cbc2b3 100644
--- a/warehouse/core/src/main/java/datawave/iterators/filter/ageoff/DataTypeAgeOffFilter.java
+++ b/warehouse/core/src/main/java/datawave/iterators/filter/ageoff/DataTypeAgeOffFilter.java
@@ -241,6 +241,7 @@ public void init(FilterOptions options) {
      * @param options
      *            {@code Map} object containing the TTL, TTL_UNITS, and MATCHPATTERN for the filter rule.
      * @param iterEnv
+     *            iterator environment
      * @see datawave.iterators.filter.AgeOffConfigParams
      */
     public void init(FilterOptions options, IteratorEnvironment iterEnv) {
diff --git a/warehouse/core/src/main/java/datawave/iterators/filter/ageoff/FieldAgeOffFilter.java b/warehouse/core/src/main/java/datawave/iterators/filter/ageoff/FieldAgeOffFilter.java
index 91d7130a749..ecfcd546e56 100644
--- a/warehouse/core/src/main/java/datawave/iterators/filter/ageoff/FieldAgeOffFilter.java
+++ b/warehouse/core/src/main/java/datawave/iterators/filter/ageoff/FieldAgeOffFilter.java
@@ -235,6 +235,8 @@ public void init(FilterOptions options) {
      * @param options
      *            {@code Map} object containing the TTL, TTL_UNITS, and MATCHPATTERN for the filter rule.
      * @param iterEnv
+     *            iterator environment
+     *
      * @see datawave.iterators.filter.AgeOffConfigParams
      */
     public void init(FilterOptions options, IteratorEnvironment iterEnv) {
diff --git a/warehouse/core/src/main/java/datawave/iterators/filter/ageoff/FilterRule.java b/warehouse/core/src/main/java/datawave/iterators/filter/ageoff/FilterRule.java
index e98790fc336..2916d833daa 100644
--- a/warehouse/core/src/main/java/datawave/iterators/filter/ageoff/FilterRule.java
+++ b/warehouse/core/src/main/java/datawave/iterators/filter/ageoff/FilterRule.java
@@ -10,7 +10,7 @@
  */
 public interface FilterRule {
     /**
-     * Used to initialize the the {@code FilterRule} implementation
+     * Used to initialize the {@code FilterRule} implementation
      *
      * @param options
      *            {@code Map} object
@@ -18,17 +18,21 @@ public interface FilterRule {
     void init(FilterOptions options);
 
     /**
-     * Used to initialize the the {@code FilterRule} implementation
+     * Used to initialize the {@code FilterRule} implementation
      *
      * @param options
      *            {@code Map} object
      * @param iterEnv
+     *            iterator environment
      */
     void init(FilterOptions options, IteratorEnvironment iterEnv);
 
     /**
      * Used to test a {@code Key/Value} pair, and returns {@code true} if it is accepted
      *
+     * @param iter
+     *            key/value iterator
+     *
      * @return {@code boolean} value.
      */
     boolean accept(SortedKeyValueIterator iter);
@@ -37,7 +41,10 @@ public interface FilterRule {
 
     /**
      * @param scanStart
-     * @return
+     *            index to start scan
+     * @param iterEnv
+     *            the iterator environment
+     * @return a copy
      */
     FilterRule deepCopy(long scanStart, IteratorEnvironment iterEnv);
 
diff --git a/warehouse/core/src/main/java/datawave/iterators/filter/ageoff/MaximumAgeOffFilter.java b/warehouse/core/src/main/java/datawave/iterators/filter/ageoff/MaximumAgeOffFilter.java
index b4fe4b7c9ed..04d9535a3fa 100644
--- a/warehouse/core/src/main/java/datawave/iterators/filter/ageoff/MaximumAgeOffFilter.java
+++ b/warehouse/core/src/main/java/datawave/iterators/filter/ageoff/MaximumAgeOffFilter.java
@@ -83,6 +83,8 @@ public void init(FilterOptions options) {
      * @param options
      *            {@code Map} object containing the TTL, TTL_UNITS, and MATCHPATTERN for the filter rule.
      * @param iterEnv
+     *            iterator environment
+     *
      * @see datawave.iterators.filter.AgeOffConfigParams
      */
     public void init(FilterOptions options, IteratorEnvironment iterEnv) {
diff --git a/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java b/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java
index b8d7490609c..4370cd6099d 100644
--- a/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java
+++ b/warehouse/core/src/main/java/datawave/mr/bulk/BulkInputFormat.java
@@ -318,7 +318,9 @@ public static void setRanges(Job job, Collection ranges) {
      * Specify the working directory, fs.getWorkingDirectory() doesn't work with ViewFS.
      *
      * @param conf
+     *            the Hadoop configuration object
      * @param path
+     *            path to the Hadoop working directory
      */
     public static void setWorkingDirectory(Configuration conf, String path) {
         conf.set(WORKING_DIRECTORY, path);
@@ -667,6 +669,7 @@ protected static String getUsername(Configuration conf) {
      *            the Hadoop configuration object
      * @return the BASE64-encoded password
      * @throws IOException
+     *             if there is an error reading the file
      * @see #setInputInfo(Job, String, byte[], String, Authorizations)
      */
     protected static byte[] getPassword(Configuration conf) throws IOException {
@@ -804,6 +807,7 @@ public static List getIteratorOptions(Configuration conf
          * @param scanner
          *            the scanner to configure
          * @throws AccumuloException
+         *             if there is an error with Accumulo
          */
         protected void setupIterators(Configuration conf, BatchScanner scanner) throws AccumuloException {
             List iterators = getIterators(conf);
diff --git a/warehouse/core/src/main/java/datawave/mr/bulk/split/FileRangeSplit.java b/warehouse/core/src/main/java/datawave/mr/bulk/split/FileRangeSplit.java
index c9b99925725..8c5107fe9d8 100644
--- a/warehouse/core/src/main/java/datawave/mr/bulk/split/FileRangeSplit.java
+++ b/warehouse/core/src/main/java/datawave/mr/bulk/split/FileRangeSplit.java
@@ -56,17 +56,29 @@ public boolean isSame(FileRangeSplit other) {
         return file.equals(other.file) && start == other.start && length == other.length;
     }
 
-    /** The file containing this split's data. */
+    /**
+     * The file containing this split's data.
+     *
+     * @return the path
+     */
     public Path getPath() {
         return file;
     }
 
-    /** The position of the first byte in the file to process. */
+    /**
+     * The position of the first byte in the file to process.
+     *
+     * @return the position of the first byte
+     */
     public long getStart() {
         return start;
     }
 
-    /** The number of bytes in the file to process. */
+    /**
+     * The number of bytes in the file to process.
+     *
+     * @return the number of bytes
+     */
     @Override
     public long getLength() {
         return length;
diff --git a/warehouse/core/src/main/java/datawave/mr/bulk/split/RangeSplit.java b/warehouse/core/src/main/java/datawave/mr/bulk/split/RangeSplit.java
index 763857d5c89..9cc7518e6de 100644
--- a/warehouse/core/src/main/java/datawave/mr/bulk/split/RangeSplit.java
+++ b/warehouse/core/src/main/java/datawave/mr/bulk/split/RangeSplit.java
@@ -80,7 +80,9 @@ private void updateProgressDepth() {
      * will set the startByte, endByte, and range class members.
      *
      * @param start
+     *            start of the range
      * @param end
+     *            end of the range
      */
     private void updateSignificantBytes(ByteSequence start, ByteSequence end) {
         // we want to compare enough bytes that we get a resolution of .1 percent or less
@@ -118,7 +120,9 @@ private void updateSignificantBytes(ByteSequence start, ByteSequence end) {
      * distance. Then the distance is compared to the range to return the overall percentage.
      *
      * @param start
+     *            start of the range
      * @param end
+     *            end of the range
      * @return the progress [0.0, 1.0]
      */
     public float getProgress(ByteSequence start, ByteSequence end) {
@@ -140,6 +144,7 @@ public float getProgress(ByteSequence start, ByteSequence end) {
      * use, and then use the method above to get the actuall progress value.
      *
      * @param currentKey
+     *            the current key
      * @return the progress [0.0, 1.0]
      */
     public float getProgress(Key currentKey) {
diff --git a/warehouse/core/src/main/java/datawave/util/ObjectFactory.java b/warehouse/core/src/main/java/datawave/util/ObjectFactory.java
index 0177bf6343b..db8e2bfc982 100644
--- a/warehouse/core/src/main/java/datawave/util/ObjectFactory.java
+++ b/warehouse/core/src/main/java/datawave/util/ObjectFactory.java
@@ -134,6 +134,9 @@ public static Object create(String className, Object... args) {
     /**
      * Return primitive for a Primitive wrapper, i.e. int --> Integer
      *
+     * @param clazz
+     *            the class
+     *
      * @return the class requested
      */
     public static Class getPrim(Class clazz) {
diff --git a/warehouse/core/src/main/java/datawave/util/OperationEvaluator.java b/warehouse/core/src/main/java/datawave/util/OperationEvaluator.java
index 1e6c16edfa8..4cb164c8e5c 100644
--- a/warehouse/core/src/main/java/datawave/util/OperationEvaluator.java
+++ b/warehouse/core/src/main/java/datawave/util/OperationEvaluator.java
@@ -401,6 +401,8 @@ public static boolean compare(Date left, Date right, String operator) {
      * 
  • >= returns whether the left is greater than or equal to the right
    • 
      * 
      *
+     * @param 
+     *            comparable class
      * @param left
      *            the left side of the expression
      * @param right
@@ -442,12 +444,16 @@ public static > boolean compare(T left, T right, String
      * 
  • >= returns whether the left is greater than or equal to the right
    • 
      * 
      *
+     * @param 
+     *            comparable class
      * @param left
      *            the left side of the expression
      * @param right
      *            the right side of the expression
      * @param operator
      *            the logical operator
+     * @param comparator
+     *            the comparator
      * @return true if the logical expression evaluates to true, or false otherwise
      */
     public static  boolean compare(T left, T right, String operator, Comparator comparator) {
diff --git a/warehouse/core/src/main/java/datawave/util/TextUtil.java b/warehouse/core/src/main/java/datawave/util/TextUtil.java
index 387b44a3688..33e03e24b8a 100644
--- a/warehouse/core/src/main/java/datawave/util/TextUtil.java
+++ b/warehouse/core/src/main/java/datawave/util/TextUtil.java
@@ -56,6 +56,13 @@ public static void textAppendNoNull(Text t, String s) {
 
     /**
      * Appends the UTF-8 bytes of the given string to the given {@link Text}
+     *
+     * @param t
+     *            the Text to which append
+     * @param s
+     *            the String to append
+     * @param replaceBadChar
+     *            flag to replace bad characters
      */
     public static void textAppendNoNull(Text t, String s, boolean replaceBadChar) {
         try {
@@ -90,6 +97,7 @@ public static byte[] toUtf8(String string) {
      * Converts a UTF-8 encoded byte array back into a String.
      *
      * @param bytes
+     *            UTF8 string of bytes
      * @return string
      */
     public static String fromUtf8(byte[] bytes) {
diff --git a/warehouse/core/src/main/java/datawave/util/timely/TcpClient.java b/warehouse/core/src/main/java/datawave/util/timely/TcpClient.java
index d4cf86bdd40..b0bf0f0f8f0 100644
--- a/warehouse/core/src/main/java/datawave/util/timely/TcpClient.java
+++ b/warehouse/core/src/main/java/datawave/util/timely/TcpClient.java
@@ -41,6 +41,7 @@ public void open() throws IOException {
      * @param metric
      *            newline terminated string representation of Timely metric
      * @throws IOException
+     *             an error occurs
      */
     public synchronized void write(String metric) throws IOException {
         if (connect() != 0) {
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/FilterIngest.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/FilterIngest.java
index f731f05fad7..c1c5faa6123 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/FilterIngest.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/FilterIngest.java
@@ -5,8 +5,9 @@
 import datawave.ingest.data.config.NormalizedContentInterface;
 
 /**
- * A marker interface that demonstrates an {@link IngestHelperInterface} must apply a filter to the event fields.
  * 
    
+ * A marker interface that demonstrates an {@link IngestHelperInterface} must apply a filter to the event fields.
+ * 
    
  *
  * @see datawave.ingest.data.config.ingest.IngestFieldFilter
  */
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java
index f1c87192b97..964634a4282 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/dateindex/DateIndexDataTypeHandler.java
@@ -49,7 +49,7 @@
  * This class creates the following Mutations or Key/Values: 
    
  * 
    
  * 
- * 
+ * 
  * 
  * 
  * 
@@ -70,7 +70,7 @@
  *
  * 
    
  * The table with the name specified by {@link #DATEINDEX_TNAME} will be the date index table.
- *
+ * 
    
  *
  *
  * @param 
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java
index 305f4796031..9aa7ea8a117 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorDataTypeHandler.java
@@ -47,7 +47,7 @@
  * This class creates the following Mutations or Key/Values: 
    
  * 
    
  * 
    DataTypeHandler
    Schema TypeUse
    
- * 
+ * 
  * 
  * 
  * 
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java
index 78df2c3230d..53fda2881a3 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/error/ErrorShardedDataTypeHandler.java
@@ -50,7 +50,7 @@
  * This class creates the following Mutations or Key/Values: 
    
  * 
    
  * 
    ErrorDataType
    Schema TypeUse
    
- * 
+ * 
  * 
  * 
  * 
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java
index ef1d6517216..35a28cc004a 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java
@@ -58,7 +58,7 @@
  * This class creates the following Mutations or Key/Values: 
    
  * 
    
  * 
    ErrorShardedDataType
    Schema TypeUse
    
- * 
+ * 
  * 
  * 
  * 
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/tokenize/ExtendedContentIndexingColumnBasedHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/tokenize/ExtendedContentIndexingColumnBasedHandler.java
index f2d532e6550..8141959ae98 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/tokenize/ExtendedContentIndexingColumnBasedHandler.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/tokenize/ExtendedContentIndexingColumnBasedHandler.java
@@ -68,7 +68,7 @@
  * This class creates the following Mutations or Key/Values in addition to those created by the {@link ShardedDataTypeHandler}: 
    
  * 
    
  * 
    SgardedDataType
    Schema TypeUse
    
- * 
+ * 
  * 
  * 
  * 
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/metadata/EventMetadata.java b/warehouse/ingest-core/src/main/java/datawave/ingest/metadata/EventMetadata.java
index 39bb38ea42a..549752787c4 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/metadata/EventMetadata.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/metadata/EventMetadata.java
@@ -36,7 +36,7 @@
  * 
    
  * 
    
  * 
    ExtendedContentIndexingColumnBased
    Schema TypeUse
    
- * 
+ * 
  * 
  * 
  * 
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/protobuf/RawRecordContainer.java b/warehouse/ingest-core/src/main/java/datawave/ingest/protobuf/RawRecordContainer.java
index 3d97c6a78af..ab4b53314c9 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/protobuf/RawRecordContainer.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/protobuf/RawRecordContainer.java
@@ -13,156 +13,226 @@ public interface DataOrBuilder extends com.google.protobuf.MessageOrBuilder {
         // optional uint64 date = 1;
         /**
          * optional uint64 date = 1;
+         *
+         * @return if it has date
          */
         boolean hasDate();
 
         /**
          * optional uint64 date = 1;
+         *
+         * @return the date
          */
         long getDate();
 
         // optional string dataType = 2;
         /**
          * optional string dataType = 2;
+         *
+         * @return if it has data type
          */
         boolean hasDataType();
 
         /**
          * optional string dataType = 2;
+         *
+         * @return the data type
          */
         java.lang.String getDataType();
 
         /**
          * optional string dataType = 2;
+         *
+         * @return the data type byte string
          */
         com.google.protobuf.ByteString getDataTypeBytes();
 
         // optional string uid = 3;
         /**
          * optional string uid = 3;
+         *
+         * @return if it has uid
          */
         boolean hasUid();
 
         /**
          * optional string uid = 3;
+         *
+         * @return uid
          */
         java.lang.String getUid();
 
         /**
          * optional string uid = 3;
+         *
+         * @return uid byte string
          */
         com.google.protobuf.ByteString getUidBytes();
 
         // repeated string errors = 4;
         /**
          * repeated string errors = 4;
+         *
+         * @return list of errors
          */
         java.util.List getErrorsList();
 
         /**
          * repeated string errors = 4;
+         *
+         * @return number of errors
          */
         int getErrorsCount();
 
         /**
          * repeated string errors = 4;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the error at the provided index
          */
         java.lang.String getErrors(int index);
 
         /**
          * repeated string errors = 4;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the error byte string at the provided index
          */
         com.google.protobuf.ByteString getErrorsBytes(int index);
 
         // optional bytes visibility = 5;
         /**
          * optional bytes visibility = 5;
+         *
+         * @return if it has visibility
          */
         boolean hasVisibility();
 
         /**
          * optional bytes visibility = 5;
+         *
+         * @return the visibility
          */
         com.google.protobuf.ByteString getVisibility();
 
         // optional string rawFileName = 6;
         /**
          * optional string rawFileName = 6;
+         *
+         * @return if it has raw file name
          */
         boolean hasRawFileName();
 
         /**
          * optional string rawFileName = 6;
+         *
+         * @return the raw file name
          */
         java.lang.String getRawFileName();
 
         /**
          * optional string rawFileName = 6;
+         *
+         * @return the raw file name byte string
          */
         com.google.protobuf.ByteString getRawFileNameBytes();
 
         // optional uint64 rawRecordNumber = 7;
         /**
          * optional uint64 rawRecordNumber = 7;
+         *
+         * @return if it has raw record number
          */
         boolean hasRawRecordNumber();
 
         /**
          * optional uint64 rawRecordNumber = 7;
+         *
+         * @return the raw record number
          */
         long getRawRecordNumber();
 
         // repeated string uuids = 8;
         /**
          * repeated string uuids = 8;
+         *
+         * @return list of uuids
          */
         java.util.List getUuidsList();
 
         /**
          * repeated string uuids = 8;
+         *
+         * @return the number of uuids in the list
          */
         int getUuidsCount();
 
         /**
          * repeated string uuids = 8;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the uuid at the provided index
          */
         java.lang.String getUuids(int index);
 
         /**
          * repeated string uuids = 8;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the uuid byte string at the provided index
          */
         com.google.protobuf.ByteString getUuidsBytes(int index);
 
         // optional bytes rawData = 9;
         /**
          * optional bytes rawData = 9;
+         *
+         * @return if it has raw data
          */
         boolean hasRawData();
 
         /**
          * optional bytes rawData = 9;
+         *
+         * @return the raw data byte string
          */
         com.google.protobuf.ByteString getRawData();
 
         // optional bool requiresMasking = 10;
         /**
          * optional bool requiresMasking = 10;
+         *
+         * @return if has requires masking
          */
         boolean hasRequiresMasking();
 
         /**
          * optional bool requiresMasking = 10;
+         *
+         * @return if it requires masking
          */
         boolean getRequiresMasking();
 
         // optional uint64 rawFileTimeStamp = 11;
         /**
          * optional uint64 rawFileTimeStamp = 11;
+         *
+         * @return if it has raw file timestamp
          */
         boolean hasRawFileTimeStamp();
 
         /**
          * optional uint64 rawFileTimeStamp = 11;
+         *
+         * @return the raw file timestamp
          */
         long getRawFileTimeStamp();
     }
@@ -324,6 +394,8 @@ public com.google.protobuf.Parser getParserForType() {
 
         /**
          * optional uint64 date = 1;
+         *
+         * @return if it has date
          */
         public boolean hasDate() {
             return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -331,6 +403,8 @@ public boolean hasDate() {
 
         /**
          * optional uint64 date = 1;
+         *
+         * @return the date
          */
         public long getDate() {
             return date_;
@@ -342,6 +416,8 @@ public long getDate() {
 
         /**
          * optional string dataType = 2;
+         *
+         * @return if it has data type
          */
         public boolean hasDataType() {
             return ((bitField0_ & 0x00000002) == 0x00000002);
@@ -349,6 +425,8 @@ public boolean hasDataType() {
 
         /**
          * optional string dataType = 2;
+         *
+         * @return the data type
          */
         public java.lang.String getDataType() {
             java.lang.Object ref = dataType_;
@@ -366,6 +444,8 @@ public java.lang.String getDataType() {
 
         /**
          * optional string dataType = 2;
+         *
+         * @return the data type byte string
          */
         public com.google.protobuf.ByteString getDataTypeBytes() {
             java.lang.Object ref = dataType_;
@@ -384,6 +464,8 @@ public com.google.protobuf.ByteString getDataTypeBytes() {
 
         /**
          * optional string uid = 3;
+         *
+         * @return if it has uid
          */
         public boolean hasUid() {
             return ((bitField0_ & 0x00000004) == 0x00000004);
@@ -391,6 +473,8 @@ public boolean hasUid() {
 
         /**
          * optional string uid = 3;
+         *
+         * @return the uid
          */
         public java.lang.String getUid() {
             java.lang.Object ref = uid_;
@@ -408,6 +492,8 @@ public java.lang.String getUid() {
 
         /**
          * optional string uid = 3;
+         *
+         * @return the uid byte string
          */
         public com.google.protobuf.ByteString getUidBytes() {
             java.lang.Object ref = uid_;
@@ -426,6 +512,8 @@ public com.google.protobuf.ByteString getUidBytes() {
 
         /**
          * repeated string errors = 4;
+         *
+         * @return the errors list
          */
         public java.util.List getErrorsList() {
             return errors_;
@@ -433,6 +521,8 @@ public java.util.List getErrorsList() {
 
         /**
          * repeated string errors = 4;
+         *
+         * @return the number of errors in the list
          */
         public int getErrorsCount() {
             return errors_.size();
@@ -440,6 +530,11 @@ public int getErrorsCount() {
 
         /**
          * repeated string errors = 4;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the error at the provided index
          */
         public java.lang.String getErrors(int index) {
             return errors_.get(index);
@@ -447,6 +542,11 @@ public java.lang.String getErrors(int index) {
 
         /**
          * repeated string errors = 4;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the error byte string for the provided index
          */
         public com.google.protobuf.ByteString getErrorsBytes(int index) {
             return errors_.getByteString(index);
@@ -458,6 +558,8 @@ public com.google.protobuf.ByteString getErrorsBytes(int index) {
 
         /**
          * optional bytes visibility = 5;
+         *
+         * @return if it has visibility
          */
         public boolean hasVisibility() {
             return ((bitField0_ & 0x00000008) == 0x00000008);
@@ -465,6 +567,8 @@ public boolean hasVisibility() {
 
         /**
          * optional bytes visibility = 5;
+         *
+         * @return the visibility byte string
          */
         public com.google.protobuf.ByteString getVisibility() {
             return visibility_;
@@ -476,6 +580,8 @@ public com.google.protobuf.ByteString getVisibility() {
 
         /**
          * optional string rawFileName = 6;
+         *
+         * @return if it has raw file name
          */
         public boolean hasRawFileName() {
             return ((bitField0_ & 0x00000010) == 0x00000010);
@@ -483,6 +589,8 @@ public boolean hasRawFileName() {
 
         /**
          * optional string rawFileName = 6;
+         *
+         * @return the raw file name
          */
         public java.lang.String getRawFileName() {
             java.lang.Object ref = rawFileName_;
@@ -500,6 +608,8 @@ public java.lang.String getRawFileName() {
 
         /**
          * optional string rawFileName = 6;
+         *
+         * @return the raw file name byte string
          */
         public com.google.protobuf.ByteString getRawFileNameBytes() {
             java.lang.Object ref = rawFileName_;
@@ -518,6 +628,8 @@ public com.google.protobuf.ByteString getRawFileNameBytes() {
 
         /**
          * optional uint64 rawRecordNumber = 7;
+         *
+         * @return if it has raw record number
          */
         public boolean hasRawRecordNumber() {
             return ((bitField0_ & 0x00000020) == 0x00000020);
@@ -525,6 +637,8 @@ public boolean hasRawRecordNumber() {
 
         /**
          * optional uint64 rawRecordNumber = 7;
+         *
+         * @return the raw record number
          */
         public long getRawRecordNumber() {
             return rawRecordNumber_;
@@ -536,6 +650,8 @@ public long getRawRecordNumber() {
 
         /**
          * repeated string uuids = 8;
+         *
+         * @return the uuids list
          */
         public java.util.List getUuidsList() {
             return uuids_;
@@ -543,6 +659,8 @@ public java.util.List getUuidsList() {
 
         /**
          * repeated string uuids = 8;
+         *
+         * @return the number of uuids in the list
          */
         public int getUuidsCount() {
             return uuids_.size();
@@ -550,6 +668,11 @@ public int getUuidsCount() {
 
         /**
          * repeated string uuids = 8;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the uuid at the provided index
          */
         public java.lang.String getUuids(int index) {
             return uuids_.get(index);
@@ -557,6 +680,11 @@ public java.lang.String getUuids(int index) {
 
         /**
          * repeated string uuids = 8;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the uuid byte string at the provided index
          */
         public com.google.protobuf.ByteString getUuidsBytes(int index) {
             return uuids_.getByteString(index);
@@ -568,6 +696,8 @@ public com.google.protobuf.ByteString getUuidsBytes(int index) {
 
         /**
          * optional bytes rawData = 9;
+         *
+         * @return if it has raw data
          */
         public boolean hasRawData() {
             return ((bitField0_ & 0x00000040) == 0x00000040);
@@ -575,6 +705,8 @@ public boolean hasRawData() {
 
         /**
          * optional bytes rawData = 9;
+         *
+         * @return raw data byte string
          */
         public com.google.protobuf.ByteString getRawData() {
             return rawData_;
@@ -586,6 +718,8 @@ public com.google.protobuf.ByteString getRawData() {
 
         /**
          * optional bool requiresMasking = 10;
+         *
+         * @return if it has requires masking bit set
          */
         public boolean hasRequiresMasking() {
             return ((bitField0_ & 0x00000080) == 0x00000080);
@@ -593,6 +727,8 @@ public boolean hasRequiresMasking() {
 
         /**
          * optional bool requiresMasking = 10;
+         *
+         * @return if it requires masking
          */
         public boolean getRequiresMasking() {
             return requiresMasking_;
@@ -604,6 +740,8 @@ public boolean getRequiresMasking() {
 
         /**
          * optional uint64 rawFileTimeStamp = 11;
+         *
+         * @return if it has raw file timestamp
          */
         public boolean hasRawFileTimeStamp() {
             return ((bitField0_ & 0x00000100) == 0x00000100);
@@ -611,6 +749,8 @@ public boolean hasRawFileTimeStamp() {
 
         /**
          * optional uint64 rawFileTimeStamp = 11;
+         *
+         * @return the raw file timestamp
          */
         public long getRawFileTimeStamp() {
             return rawFileTimeStamp_;
@@ -1040,6 +1180,8 @@ public Builder mergeFrom(com.google.protobuf.CodedInputStream input, com.google.
 
             /**
              * optional uint64 date = 1;
+             *
+             * @return if it has date
              */
             public boolean hasDate() {
                 return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -1047,6 +1189,8 @@ public boolean hasDate() {
 
             /**
              * optional uint64 date = 1;
+             *
+             * @return the date
              */
             public long getDate() {
                 return date_;
@@ -1054,6 +1198,11 @@ public long getDate() {
 
             /**
              * optional uint64 date = 1;
+             *
+             * @param value
+             *            the new date value
+             *
+             * @return builder to set the date to the new value
              */
             public Builder setDate(long value) {
                 bitField0_ |= 0x00000001;
@@ -1064,6 +1213,8 @@ public Builder setDate(long value) {
 
             /**
              * optional uint64 date = 1;
+             *
+             * @return builder to clear the date
              */
             public Builder clearDate() {
                 bitField0_ = (bitField0_ & ~0x00000001);
@@ -1077,6 +1228,8 @@ public Builder clearDate() {
 
             /**
              * optional string dataType = 2;
+             *
+             * @return if it has data type
              */
             public boolean hasDataType() {
                 return ((bitField0_ & 0x00000002) == 0x00000002);
@@ -1084,6 +1237,8 @@ public boolean hasDataType() {
 
             /**
              * optional string dataType = 2;
+             *
+             * @return the data type
              */
             public java.lang.String getDataType() {
                 java.lang.Object ref = dataType_;
@@ -1098,6 +1253,8 @@ public java.lang.String getDataType() {
 
             /**
              * optional string dataType = 2;
+             *
+             * @return the data type byte string
              */
             public com.google.protobuf.ByteString getDataTypeBytes() {
                 java.lang.Object ref = dataType_;
@@ -1112,6 +1269,11 @@ public com.google.protobuf.ByteString getDataTypeBytes() {
 
             /**
              * optional string dataType = 2;
+             *
+             * @param value
+             *            the new data type
+             *
+             * @return builder to set the data type to the new value
              */
             public Builder setDataType(java.lang.String value) {
                 if (value == null) {
@@ -1125,6 +1287,8 @@ public Builder setDataType(java.lang.String value) {
 
             /**
              * optional string dataType = 2;
+             *
+             * @return builder to clear the data type
              */
             public Builder clearDataType() {
                 bitField0_ = (bitField0_ & ~0x00000002);
@@ -1135,6 +1299,11 @@ public Builder clearDataType() {
 
             /**
              * optional string dataType = 2;
+             *
+             * @param value
+             *            data type byte string
+             *
+             * @return builder to set data type to new value
              */
             public Builder setDataTypeBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -1151,6 +1320,8 @@ public Builder setDataTypeBytes(com.google.protobuf.ByteString value) {
 
             /**
              * optional string uid = 3;
+             *
+             * @return if it has uid
              */
             public boolean hasUid() {
                 return ((bitField0_ & 0x00000004) == 0x00000004);
@@ -1158,6 +1329,8 @@ public boolean hasUid() {
 
             /**
              * optional string uid = 3;
+             *
+             * @return the uid
              */
             public java.lang.String getUid() {
                 java.lang.Object ref = uid_;
@@ -1172,6 +1345,8 @@ public java.lang.String getUid() {
 
             /**
              * optional string uid = 3;
+             *
+             * @return the uid byte string
              */
             public com.google.protobuf.ByteString getUidBytes() {
                 java.lang.Object ref = uid_;
@@ -1186,6 +1361,11 @@ public com.google.protobuf.ByteString getUidBytes() {
 
             /**
              * optional string uid = 3;
+             *
+             * @param value
+             *            the new uid value
+             *
+             * @return builder to set uid to new value
              */
             public Builder setUid(java.lang.String value) {
                 if (value == null) {
@@ -1199,6 +1379,8 @@ public Builder setUid(java.lang.String value) {
 
             /**
              * optional string uid = 3;
+             *
+             * @return builder to clear the uid
              */
             public Builder clearUid() {
                 bitField0_ = (bitField0_ & ~0x00000004);
@@ -1209,6 +1391,11 @@ public Builder clearUid() {
 
             /**
              * optional string uid = 3;
+             *
+             * @param value
+             *            new uid byte string
+             *
+             * @return builder to set uid to the provided byte string
              */
             public Builder setUidBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -1232,6 +1419,8 @@ private void ensureErrorsIsMutable() {
 
             /**
              * repeated string errors = 4;
+             *
+             * @return list of errors
              */
             public java.util.List getErrorsList() {
                 return java.util.Collections.unmodifiableList(errors_);
@@ -1239,6 +1428,8 @@ public java.util.List getErrorsList() {
 
             /**
              * repeated string errors = 4;
+             *
+             * @return the number of errors in the list
              */
             public int getErrorsCount() {
                 return errors_.size();
@@ -1246,6 +1437,11 @@ public int getErrorsCount() {
 
             /**
              * repeated string errors = 4;
+             *
+             * @param index
+             *            the index
+             *
+             * @return the error at the provided index
              */
             public java.lang.String getErrors(int index) {
                 return errors_.get(index);
@@ -1253,6 +1449,11 @@ public java.lang.String getErrors(int index) {
 
             /**
              * repeated string errors = 4;
+             *
+             * @param index
+             *            the index
+             *
+             * @return error byte string for provided index
              */
             public com.google.protobuf.ByteString getErrorsBytes(int index) {
                 return errors_.getByteString(index);
@@ -1260,6 +1461,14 @@ public com.google.protobuf.ByteString getErrorsBytes(int index) {
 
             /**
              * repeated string errors = 4;
+             *
+             * @param index
+             *            the index
+             *
+             * @param value
+             *            the new error string to add at the provided index
+             *
+             * @return builder to set the error at the provided index
              */
             public Builder setErrors(int index, java.lang.String value) {
                 if (value == null) {
@@ -1273,6 +1482,11 @@ public Builder setErrors(int index, java.lang.String value) {
 
             /**
              * repeated string errors = 4;
+             *
+             * @param value
+             *            the new error to add to the list of errors
+             *
+             * @return builder to add the error to the list
              */
             public Builder addErrors(java.lang.String value) {
                 if (value == null) {
@@ -1286,6 +1500,11 @@ public Builder addErrors(java.lang.String value) {
 
             /**
              * repeated string errors = 4;
+             *
+             * @param values
+             *            iterable of errors to add to the list
+             *
+             * @return builder to add errors to the list
              */
             public Builder addAllErrors(java.lang.Iterable values) {
                 ensureErrorsIsMutable();
@@ -1296,6 +1515,8 @@ public Builder addAllErrors(java.lang.Iterable values) {
 
             /**
              * repeated string errors = 4;
+             *
+             * @return builder to clear error list
              */
             public Builder clearErrors() {
                 errors_ = com.google.protobuf.LazyStringArrayList.EMPTY;
@@ -1306,6 +1527,11 @@ public Builder clearErrors() {
 
             /**
              * repeated string errors = 4;
+             *
+             * @param value
+             *            error byte string to add
+             *
+             * @return builder to add new error byte string
              */
             public Builder addErrorsBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -1322,6 +1548,8 @@ public Builder addErrorsBytes(com.google.protobuf.ByteString value) {
 
             /**
              * optional bytes visibility = 5;
+             *
+             * @return if it has visibility
              */
             public boolean hasVisibility() {
                 return ((bitField0_ & 0x00000010) == 0x00000010);
@@ -1329,6 +1557,8 @@ public boolean hasVisibility() {
 
             /**
              * optional bytes visibility = 5;
+             *
+             * @return visibility
              */
             public com.google.protobuf.ByteString getVisibility() {
                 return visibility_;
@@ -1336,6 +1566,11 @@ public com.google.protobuf.ByteString getVisibility() {
 
             /**
              * optional bytes visibility = 5;
+             *
+             * @param value
+             *            new visibility value
+             *
+             * @return builder to set visibility
              */
             public Builder setVisibility(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -1349,6 +1584,8 @@ public Builder setVisibility(com.google.protobuf.ByteString value) {
 
             /**
              * optional bytes visibility = 5;
+             *
+             * @return builder to clear visibility
              */
             public Builder clearVisibility() {
                 bitField0_ = (bitField0_ & ~0x00000010);
@@ -1362,6 +1599,8 @@ public Builder clearVisibility() {
 
             /**
              * optional string rawFileName = 6;
+             *
+             * @return if it has raw file name
              */
             public boolean hasRawFileName() {
                 return ((bitField0_ & 0x00000020) == 0x00000020);
@@ -1369,6 +1608,8 @@ public boolean hasRawFileName() {
 
             /**
              * optional string rawFileName = 6;
+             *
+             * @return raw file name
              */
             public java.lang.String getRawFileName() {
                 java.lang.Object ref = rawFileName_;
@@ -1383,6 +1624,8 @@ public java.lang.String getRawFileName() {
 
             /**
              * optional string rawFileName = 6;
+             *
+             * @return raw file name byte string
              */
             public com.google.protobuf.ByteString getRawFileNameBytes() {
                 java.lang.Object ref = rawFileName_;
@@ -1397,6 +1640,11 @@ public com.google.protobuf.ByteString getRawFileNameBytes() {
 
             /**
              * optional string rawFileName = 6;
+             *
+             * @param value
+             *            the new raw file name
+             *
+             * @return builder to set raw file name
              */
             public Builder setRawFileName(java.lang.String value) {
                 if (value == null) {
@@ -1410,6 +1658,8 @@ public Builder setRawFileName(java.lang.String value) {
 
             /**
              * optional string rawFileName = 6;
+             *
+             * @return builder to clear raw file name
              */
             public Builder clearRawFileName() {
                 bitField0_ = (bitField0_ & ~0x00000020);
@@ -1420,6 +1670,11 @@ public Builder clearRawFileName() {
 
             /**
              * optional string rawFileName = 6;
+             *
+             * @param value
+             *            raw file name byte string
+             *
+             * @return builder to set raw file name to provided byte string
              */
             public Builder setRawFileNameBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -1436,6 +1691,8 @@ public Builder setRawFileNameBytes(com.google.protobuf.ByteString value) {
 
             /**
              * optional uint64 rawRecordNumber = 7;
+             *
+             * @return if it has raw record number
              */
             public boolean hasRawRecordNumber() {
                 return ((bitField0_ & 0x00000040) == 0x00000040);
@@ -1443,6 +1700,8 @@ public boolean hasRawRecordNumber() {
 
             /**
              * optional uint64 rawRecordNumber = 7;
+             *
+             * @return raw record number
              */
             public long getRawRecordNumber() {
                 return rawRecordNumber_;
@@ -1450,6 +1709,11 @@ public long getRawRecordNumber() {
 
             /**
              * optional uint64 rawRecordNumber = 7;
+             *
+             * @param value
+             *            the new raw record number
+             *
+             * @return builder to set raw record number to provided value
              */
             public Builder setRawRecordNumber(long value) {
                 bitField0_ |= 0x00000040;
@@ -1460,6 +1724,8 @@ public Builder setRawRecordNumber(long value) {
 
             /**
              * optional uint64 rawRecordNumber = 7;
+             *
+             * @return builder to clear the raw record number
              */
             public Builder clearRawRecordNumber() {
                 bitField0_ = (bitField0_ & ~0x00000040);
@@ -1480,6 +1746,8 @@ private void ensureUuidsIsMutable() {
 
             /**
              * repeated string uuids = 8;
+             *
+             * @return uuids list
              */
             public java.util.List getUuidsList() {
                 return java.util.Collections.unmodifiableList(uuids_);
@@ -1487,6 +1755,8 @@ public java.util.List getUuidsList() {
 
             /**
              * repeated string uuids = 8;
+             *
+             * @return size of uuids list
              */
             public int getUuidsCount() {
                 return uuids_.size();
@@ -1494,6 +1764,11 @@ public int getUuidsCount() {
 
             /**
              * repeated string uuids = 8;
+             *
+             * @param index
+             *            the index
+             *
+             * @return the uuid at the provided index
              */
             public java.lang.String getUuids(int index) {
                 return uuids_.get(index);
@@ -1501,6 +1776,11 @@ public java.lang.String getUuids(int index) {
 
             /**
              * repeated string uuids = 8;
+             *
+             * @param index
+             *            the index
+             *
+             * @return the uuid byte string for the provided index
              */
             public com.google.protobuf.ByteString getUuidsBytes(int index) {
                 return uuids_.getByteString(index);
@@ -1508,6 +1788,14 @@ public com.google.protobuf.ByteString getUuidsBytes(int index) {
 
             /**
              * repeated string uuids = 8;
+             *
+             * @param index
+             *            the index
+             *
+             * @param value
+             *            the value to replace at the index
+             *
+             * @return builder to set uuid at the provided index to the provided value
              */
             public Builder setUuids(int index, java.lang.String value) {
                 if (value == null) {
@@ -1521,6 +1809,11 @@ public Builder setUuids(int index, java.lang.String value) {
 
             /**
              * repeated string uuids = 8;
+             *
+             * @param value
+             *            uuid to add ot the list
+             *
+             * @return builder to add provided uuid
              */
             public Builder addUuids(java.lang.String value) {
                 if (value == null) {
@@ -1534,6 +1827,11 @@ public Builder addUuids(java.lang.String value) {
 
             /**
              * repeated string uuids = 8;
+             *
+             * @param values
+             *            iterable of uuids to add
+             *
+             * @return builder to add all provided uuids
              */
             public Builder addAllUuids(java.lang.Iterable values) {
                 ensureUuidsIsMutable();
@@ -1544,6 +1842,8 @@ public Builder addAllUuids(java.lang.Iterable values) {
 
             /**
              * repeated string uuids = 8;
+             *
+             * @return builder to clear uuids
              */
             public Builder clearUuids() {
                 uuids_ = com.google.protobuf.LazyStringArrayList.EMPTY;
@@ -1554,6 +1854,11 @@ public Builder clearUuids() {
 
             /**
              * repeated string uuids = 8;
+             *
+             * @param value
+             *            uuid byte string
+             *
+             * @return builder to add uuid byte string provided
              */
             public Builder addUuidsBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -1570,6 +1875,8 @@ public Builder addUuidsBytes(com.google.protobuf.ByteString value) {
 
             /**
              * optional bytes rawData = 9;
+             *
+             * @return if it has raw data
              */
             public boolean hasRawData() {
                 return ((bitField0_ & 0x00000100) == 0x00000100);
@@ -1577,6 +1884,8 @@ public boolean hasRawData() {
 
             /**
              * optional bytes rawData = 9;
+             *
+             * @return raw data byte string
              */
             public com.google.protobuf.ByteString getRawData() {
                 return rawData_;
@@ -1584,6 +1893,11 @@ public com.google.protobuf.ByteString getRawData() {
 
             /**
              * optional bytes rawData = 9;
+             *
+             * @param value
+             *            raw data byte string
+             *
+             * @return builder to set raw data
              */
             public Builder setRawData(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -1597,6 +1911,8 @@ public Builder setRawData(com.google.protobuf.ByteString value) {
 
             /**
              * optional bytes rawData = 9;
+             *
+             * @return builder to clear raw data
              */
             public Builder clearRawData() {
                 bitField0_ = (bitField0_ & ~0x00000100);
@@ -1610,6 +1926,8 @@ public Builder clearRawData() {
 
             /**
              * optional bool requiresMasking = 10;
+             *
+             * @return if it has required masking set
              */
             public boolean hasRequiresMasking() {
                 return ((bitField0_ & 0x00000200) == 0x00000200);
@@ -1617,6 +1935,8 @@ public boolean hasRequiresMasking() {
 
             /**
              * optional bool requiresMasking = 10;
+             *
+             * @return requires masking flag
              */
             public boolean getRequiresMasking() {
                 return requiresMasking_;
@@ -1624,6 +1944,11 @@ public boolean getRequiresMasking() {
 
             /**
              * optional bool requiresMasking = 10;
+             *
+             * @param value
+             *            requires masking flag
+             *
+             * @return builder to set requires masking flag
              */
             public Builder setRequiresMasking(boolean value) {
                 bitField0_ |= 0x00000200;
@@ -1634,6 +1959,8 @@ public Builder setRequiresMasking(boolean value) {
 
             /**
              * optional bool requiresMasking = 10;
+             *
+             * @return builder to clear requires masking flag
              */
             public Builder clearRequiresMasking() {
                 bitField0_ = (bitField0_ & ~0x00000200);
@@ -1647,6 +1974,8 @@ public Builder clearRequiresMasking() {
 
             /**
              * optional uint64 rawFileTimeStamp = 11;
+             *
+             * @return if it has raw file timestamp
              */
             public boolean hasRawFileTimeStamp() {
                 return ((bitField0_ & 0x00000400) == 0x00000400);
@@ -1654,6 +1983,8 @@ public boolean hasRawFileTimeStamp() {
 
             /**
              * optional uint64 rawFileTimeStamp = 11;
+             *
+             * @return raw file timestamp
              */
             public long getRawFileTimeStamp() {
                 return rawFileTimeStamp_;
@@ -1661,6 +1992,11 @@ public long getRawFileTimeStamp() {
 
             /**
              * optional uint64 rawFileTimeStamp = 11;
+             *
+             * @param value
+             *            new raw file timestamp
+             *
+             * @return builder to set raw file timestamp
              */
             public Builder setRawFileTimeStamp(long value) {
                 bitField0_ |= 0x00000400;
@@ -1671,6 +2007,8 @@ public Builder setRawFileTimeStamp(long value) {
 
             /**
              * optional uint64 rawFileTimeStamp = 11;
+             *
+             * @return builder to clear raw file timestamp
              */
             public Builder clearRawFileTimeStamp() {
                 bitField0_ = (bitField0_ & ~0x00000400);
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/AbstractTableConfigHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/AbstractTableConfigHelper.java
index 4a32386c104..670b0924d4e 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/AbstractTableConfigHelper.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/table/config/AbstractTableConfigHelper.java
@@ -110,6 +110,11 @@ protected void setAggregatorConfigurationIfNecessary(String tableName, List generateInitialTableProperties(Configuration config, String tableName) {
diff --git a/warehouse/metrics-core/src/main/java/datawave/metrics/mapreduce/IngestMetricsMapper.java b/warehouse/metrics-core/src/main/java/datawave/metrics/mapreduce/IngestMetricsMapper.java
index 610f65bc246..8c1df3f1caa 100644
--- a/warehouse/metrics-core/src/main/java/datawave/metrics/mapreduce/IngestMetricsMapper.java
+++ b/warehouse/metrics-core/src/main/java/datawave/metrics/mapreduce/IngestMetricsMapper.java
@@ -27,8 +27,9 @@
 import datawave.metrics.mapreduce.util.TypeNameConverter;
 
 /**
- * A map task to import Bulk Ingest metrics. Given a Counters object from a Bulk Ingest job, this mapper will format the data into a table structure similar to:
  * 
    
+ * A map task to import Bulk Ingest metrics. Given a Counters object from a Bulk Ingest job, this mapper will format the data into a table structure similar to:
+ * 
    
  *
  * 
      * {@code
@@ -41,7 +42,7 @@
  * 
    
  * 
    
  * There are no values stored- all information is a part of the key (row, column family, column qualifier).
- *
+ * 
    
  */
 public class IngestMetricsMapper extends Mapper {
 
diff --git a/warehouse/metrics-core/src/main/java/datawave/metrics/mapreduce/LoaderMetricsMapper.java b/warehouse/metrics-core/src/main/java/datawave/metrics/mapreduce/LoaderMetricsMapper.java
index 3188e1c8d24..8fb2349e070 100644
--- a/warehouse/metrics-core/src/main/java/datawave/metrics/mapreduce/LoaderMetricsMapper.java
+++ b/warehouse/metrics-core/src/main/java/datawave/metrics/mapreduce/LoaderMetricsMapper.java
@@ -16,19 +16,20 @@
 import org.apache.hadoop.mapreduce.Mapper;
 
 /**
+ * 
    
  * A Map task that imports bulk loader metrics into Accumulo. The inputs are metrics files from MapFileLoader and it places the metrics into an accumulo table
  * with the structure:
- * 
    
+ * 
    
  *
  * 
      *  Row             Colf            Colq               Value
  * ---------------------------------------------------------
  * |end time        |directory      |start time        | job counters file
  * 
    
- * 
    
  *
+ * 
    
  * Because there is no actual tag for the data, there does not need to be a Structure convenience class that standardizes the strings used to organize data.
- *
+ * 
    
  */
 public class LoaderMetricsMapper extends Mapper {
 
diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/DocumentGrouper.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/DocumentGrouper.java
index af98c779c7d..45ed00968f1 100644
--- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/DocumentGrouper.java
+++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/DocumentGrouper.java
@@ -30,12 +30,13 @@
 import datawave.query.attributes.TypeAttribute;
 
 /**
+ * 
    
  * This class provides the primary functionality needed to group documents and aggregate field values within identified groups (regardless if done server or
  * client-side).
- * 
    
+ * 
    
  * 
    
  * Grouping
- * 
    
+ * 
    
  * Grouping fields across documents will result in groupings of distinct value groupings for each specified field to group, as well as the total number of times
  * each particular grouping combination was seen. Fields to group by can be specified by the following options:
  * 
      
@@ -56,13 +57,11 @@
  * matches cannot be determined for values of fields that do not have a context, and as such they will be combined with each possible grouping, effectively a
  * cartesian product. Direct matches are prioritized and found first before indirect matches are combined with them.
  * 
      
- * 
      
  * Aggregation
- * 
      
+ * 
      
  * Once all valid groupings have been identified and counted, aggregation can be performed on the values of any specified fields for each grouping. The
  * aggregation fields can differ from the group-by fields. The following aggregation operations are supported:
- * 
      
- * 
      
+ *
  * SUM: Sum up all the values for specified fields across groupings. This operation is limited to fields with numerical values. Fields may be
  * specified via:
  * 
        
diff --git a/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupFields.java b/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupFields.java
index 196db28e23f..c7dd7c9684b 100644
--- a/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupFields.java
+++ b/warehouse/query-core/src/main/java/datawave/query/common/grouping/GroupFields.java
@@ -322,6 +322,8 @@ private Set deconstructIdentifiers(Set set) {
      *
      * @param modelMap
      *            the map to retrieve alternative field mappings from
+     * @param reverseModelMap
+     *            the reverse model map
      */
     public void remapFields(Multimap modelMap, Map reverseModelMap) {
         this.groupByFields = remap(this.groupByFields, modelMap);
diff --git a/warehouse/query-core/src/main/java/datawave/query/function/AncestorRangeProvider.java b/warehouse/query-core/src/main/java/datawave/query/function/AncestorRangeProvider.java
index d5e96e7af98..5895e9b93ef 100644
--- a/warehouse/query-core/src/main/java/datawave/query/function/AncestorRangeProvider.java
+++ b/warehouse/query-core/src/main/java/datawave/query/function/AncestorRangeProvider.java
@@ -6,8 +6,9 @@
 import datawave.query.tld.TLD;
 
 /**
- * For use in the {@link datawave.query.ancestor.AncestorQueryIterator}
  * 
        
+ * For use in the {@link datawave.query.ancestor.AncestorQueryIterator}
+ * 
        
  *
  */
 public class AncestorRangeProvider extends DocumentRangeProvider {
diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/JexlASTHelper.java b/warehouse/query-core/src/main/java/datawave/query/jexl/JexlASTHelper.java
index a5f3dc3300d..b2859b9293a 100644
--- a/warehouse/query-core/src/main/java/datawave/query/jexl/JexlASTHelper.java
+++ b/warehouse/query-core/src/main/java/datawave/query/jexl/JexlASTHelper.java
@@ -128,6 +128,7 @@ public class JexlASTHelper {
      *            string representation of a query
      * @return a fully parsed and flattened query tree
      * @throws ParseException
+     *             when unable to parse Jexl Query
      */
     public static ASTJexlScript parseAndFlattenJexlQuery(String query) throws ParseException {
         ASTJexlScript script = parseJexlQuery(query);
@@ -142,6 +143,7 @@ public static ASTJexlScript parseAndFlattenJexlQuery(String query) throws ParseE
      *            The query string in JEXL syntax to parse
      * @return Root node of the query parse tree.
      * @throws ParseException
+     *             when unable to parse Jexl Query
      */
     public static ASTJexlScript parseJexlQuery(String query) throws ParseException {
         // Instantiate a parser and visitor
@@ -246,8 +248,10 @@ private static ASTJexlScript parseQueryWithBackslashes(String query, Parser pars
      * Fetch the literal off of the grandchild. Returns null if there is no literal
      *
      * @param node
-     * @return
+     *            to get the literal of
+     * @return literal node
      * @throws NoSuchElementException
+     *             if child doesn't exist
      */
     public static JexlNode getLiteral(JexlNode node) throws NoSuchElementException {
         node = dereference(node);
@@ -276,7 +280,8 @@ public static JexlNode getLiteral(JexlNode node) throws NoSuchElementException {
      * Helper method to determine if the child is a literal
      *
      * @param child
-     * @return
+     *            the node to check
+     * @return true if the child is a literal
      */
     public static boolean isLiteral(final JexlNode child) {
         //  @formatter:off
@@ -291,8 +296,10 @@ public static boolean isLiteral(final JexlNode child) {
      * Fetch the literal off of the grandchild. Throws an exception if there is no literal
      *
      * @param node
-     * @return
+     *            the node
+     * @return the literal
      * @throws NoSuchElementException
+     *             if the child doesn't exist
      */
     @SuppressWarnings("rawtypes")
     public static Object getLiteralValue(JexlNode node) throws NoSuchElementException {
@@ -316,7 +323,8 @@ public static Object getLiteralValue(JexlNode node) throws NoSuchElementExceptio
      * Fetch the literal off of the grandchild safely. Return null if there's an exception.
      *
      * @param node
-     * @return
+     *            the node
+     * @return the literal value
      */
     @SuppressWarnings("rawtypes")
     public static Object getLiteralValueSafely(JexlNode node) {
@@ -332,8 +340,10 @@ public static Object getLiteralValueSafely(JexlNode node) {
      * identifier will be deconstructed
      *
      * @param node
+     *            the node
      * @return the deconstructed identifier
      * @throws NoSuchElementException
+     *             if the child doesn't exist
      */
     public static String getIdentifier(JexlNode node) throws NoSuchElementException {
         return getIdentifier(node, true);
@@ -343,9 +353,12 @@ public static String getIdentifier(JexlNode node) throws NoSuchElementException
      * Fetch the identifier off of the grandchild, removing a leading {@link #IDENTIFIER_PREFIX} if present. Throws an exception if there is no identifier
      *
      * @param node
+     *            the node
      * @param deconstruct
+     *            flag to turn on/off option
      * @return the identifier, deconstructed if requested
      * @throws NoSuchElementException
+     *             if the child doesn't exist
      */
     public static String getIdentifier(JexlNode node, boolean deconstruct) throws NoSuchElementException {
         if (null != node && 2 == node.jjtGetNumChildren()) {
@@ -511,6 +524,7 @@ public static JexlNode dereferenceSafely(JexlNode node) {
      * This is the opposite of dereference in that this will climb back up reference expression nodes that only contain one child.
      *
      * @param node
+     *            a JexlNode
      * @return the parent reference/referenceexpression or this node
      */
     public static JexlNode rereference(JexlNode node) {
@@ -580,7 +594,8 @@ public static String deconstructIdentifier(ASTIdentifier identifier) {
      * Remove the {@link #IDENTIFIER_PREFIX} from the beginning of a fieldName if it exists
      *
      * @param fieldName
-     * @return
+     *            fieldName to deconstruct
+     * @return the deconstructed identifier
      */
     public static String deconstructIdentifier(String fieldName) {
         return deconstructIdentifier(fieldName, false);
@@ -591,8 +606,10 @@ public static String deconstructIdentifier(String fieldName) {
      * a fieldName if it exists
      *
      * @param fieldName
+     *            fieldName to deconstruct
      * @param includeGroupingContext
-     * @return
+     *            flag for including/excluding grouping context
+     * @return the deconstructed identifier
      */
     public static String deconstructIdentifier(String fieldName, boolean includeGroupingContext) {
         if (fieldName != null && fieldName.length() > 1) {
@@ -619,7 +636,8 @@ public static String deconstructIdentifier(String fieldName, boolean includeGrou
      * Rebuild the identifier with the {@link #IDENTIFIER_PREFIX} if the identifier starts with an invalid character per the Jexl IDENTIFIER definition
      *
      * @param fieldName
-     * @return
+     *            the identifier to rebuild
+     * @return the rebuilt identifier
      */
     public static String rebuildIdentifier(String fieldName) {
         return rebuildIdentifier(fieldName, false);
@@ -629,8 +647,10 @@ public static String rebuildIdentifier(String fieldName) {
      * Rebuild the identifier with the {@link #IDENTIFIER_PREFIX} if the identifier starts with an invalid character per the Jexl IDENTIFIER definition
      *
      * @param fieldName
+     *            the fieldname to rebuild
      * @param includeGroupingContext
-     * @return
+     *            flag to include/exclude grouping context
+     * @return the rebuilt identifier
      */
     public static String rebuildIdentifier(String fieldName, boolean includeGroupingContext) {
         // fieldName may be null if it is from a Function node
@@ -740,6 +760,7 @@ private static int numNegations(JexlNode node) {
      * Iterate through provided node and its children, then return a list of nodes that are an instance of ASTEQNode.
      *
      * @param node
+     *            a JexlNode
      * @return List of ASTEQNode nodes.
      */
     public static List getEQNodes(JexlNode node) {
@@ -754,7 +775,9 @@ public static List getEQNodes(JexlNode node) {
      * Check if the provided node is an instance of ASTEQNode. If yes, then add the node to the provided list.
      *
      * @param node
+     *            a JexlNode
      * @param eqNodes
+     *            List of nodes to potentially add the provided node to
      */
     private static void getEQNodes(JexlNode node, List eqNodes) {
         if (node instanceof ASTEQNode) {
@@ -770,6 +793,7 @@ private static void getEQNodes(JexlNode node, List eqNodes) {
      * Iterate through provided node and its children, then return a list of nodes that are an instance of ASTERNode.
      *
      * @param node
+     *            a JexlNode
      * @return List of ASTERNode nodes.
      */
     public static List getERNodes(JexlNode node) {
@@ -784,7 +808,9 @@ public static List getERNodes(JexlNode node) {
      * Check if the provided node is an instance of ASTERNode. If yes, then add the node to the provided list.
      *
      * @param node
+     *            a JexlNode
      * @param erNodes
+     *            List of nodes to potentially add the provided node to
      */
     private static void getERNodes(JexlNode node, List erNodes) {
         if (node instanceof ASTERNode) {
@@ -802,7 +828,12 @@ private static void getERNodes(JexlNode node, List erNodes) {
      * @see org.apache.commons.jexl3.parser.JexlNode
      *
      * @param node
+     *            a JexlNode
      * @param typeKey
+     *            class type to check node against
+     * @param 
+     *            class extending JexlNode
+     *
      * @return List of nodes matching provided class.
      */
     public static  List getNodesOfType(JexlNode node, Class typeKey) {
@@ -819,8 +850,13 @@ public static  List getNodesOfType(JexlNode node, Class
+     *            class extending JexlNode
      */
     @SuppressWarnings("unchecked")
     private static  void getNodesOfType(JexlNode node, List nodes, Class typeKey) {
@@ -874,6 +910,8 @@ private static Map getAssignments(JexlNode node, Map getBigDecimalBoundedRange(JexlNode[] chil
      * @see org.apache.commons.jexl3.parser.ASTOrNode
      *
      * @param node
+     *            a JexlNode
      * @return True if any ancestor is an instance of ASTOrNode. If not, then False.
      */
     public static boolean isDescendantOfOr(JexlNode node) {
@@ -1260,6 +1299,7 @@ public static boolean isDescendantOfOr(JexlNode node) {
      * @see org.apache.commons.jexl3.parser.ASTNotNode
      *
      * @param node
+     *            a JexlNode
      * @return True if any ancestor is an instance of ASTNotNode. If not, then False.
      */
     public static boolean isDescendantOfNot(JexlNode node) {
@@ -1282,6 +1322,7 @@ public static boolean isDescendantOfNot(JexlNode node) {
      * @see org.apache.commons.jexl3.parser.ASTAndNode
      *
      * @param node
+     *            a JexlNode
      * @return True if any ancestor is an instance of ASTAndNode. If not, then False.
      */
     public static boolean isDescendantOfAnd(JexlNode node) {
@@ -1304,7 +1345,12 @@ public static boolean isDescendantOfAnd(JexlNode node) {
      * @see org.apache.commons.jexl3.parser.JexlNode
      *
      * @param node
+     *            a JexlNode
      * @param typeKey
+     *            class type to match against
+     * @param 
+     *            class extending JexlNode
+     *
      * @return True if any ancestor is an instance of the provided class. If not, then False.
      */
     public static  boolean isDescendantOfNodeType(JexlNode node, Class typeKey) {
@@ -1328,7 +1374,12 @@ public static  boolean isDescendantOfNodeType(JexlNode node,
      * @see org.apache.commons.jexl3.parser.JexlNode
      *
      * @param node
+     *            a JexlNode
      * @param typeKey
+     *            class type to match against
+     * @param 
+     *            class extending JexlNode
+     *
      * @return True if any descendant is an instance of the provided class. If not, then False.
      */
     public static  boolean descendantsContainNodeType(JexlNode node, Class typeKey) {
@@ -1351,8 +1402,10 @@ public static  boolean descendantsContainNodeType(JexlNode n
      * Performs an order-dependent AST equality check
      *
      * @param one
+     *            a JexlNode
      * @param two
-     * @return
+     *            a JexlNode
+     * @return true if equal, false if not
      */
     public static boolean equals(JexlNode one, JexlNode two) {
         // If we have the same object or they're both null, they're equal
@@ -1410,25 +1463,31 @@ public static boolean equals(JexlNode one, JexlNode two) {
      * 
      *
      * @param node
-     *            - a JexlNode.
-     * @return - a key for the node.
+     *            a JexlNode.
+     * @return a key for the node.
      */
     public static String nodeToKey(JexlNode node) {
         return JexlStringBuildingVisitor.buildQueryWithoutParse(node, true);
     }
 
     /**
+     * 
        
      * When at an operand, this method will find the first Identifier and replace its {image} value with the supplied {String}. This is intended to be used when
      * the query model is being supplied and we want to replace the field name in some expression.
+     * 
        
      * 
        
      * This method returns a new operand node with an updated {Identifier}.
-     * 
        
+     * 
        
      * If neither of the operand's children are an {Identifier}, then an {IllegalArgumentException} is thrown.
      *
-     * @param 
      * @param operand
+     *            a JexlNode
      * @param field
-     * @return
+     *            the new field name
+     * @param 
+     *            class extending JexlNode
+     *
+     * @return node with field set
      */
     public static  T setField(T operand, String field) {
         ASTIdentifier identifier = findIdentifier(operand);
@@ -1499,7 +1558,8 @@ public static List wrapInParens(List
      * keywords
      *
      * @param node
-     * @return
+     *            a JexlNode
+     * @return if it is literal or not
      */
     public static boolean isLiteral(Object node) {
         if (null == node) {
@@ -1516,7 +1576,8 @@ public static boolean isLiteral(Object node) {
      * Check if the provided JexlNode is an ASTEQNode and is of the form `identifier eq literal`
      *
      * @param node
-     * @return
+     *            a JexlNode
+     * @return if it is a literal
      */
     public static boolean isLiteralEquality(JexlNode node) {
         Preconditions.checkNotNull(node);
@@ -1534,8 +1595,10 @@ public static boolean isLiteralEquality(JexlNode node) {
      * Determine if the given ASTEQNode is indexed based off of the Multimap of String fieldname to TextNormalizer.
      *
      * @param node
+     *            a JexlNode
      * @param config
-     * @return
+     *            query configuration
+     * @return if it is indexed
      */
     public static boolean isIndexed(JexlNode node, ShardQueryConfiguration config) {
         Preconditions.checkNotNull(config);
@@ -1561,9 +1624,12 @@ public static boolean isIndexed(JexlNode node, ShardQueryConfiguration config) {
      * Return the selectivity of the node's identifier, or IndexStatsClient.DEFAULT_VALUE if there's an error getting the selectivity
      *
      * @param node
+     *            a JexlNode
      * @param config
+     *            query configuration
      * @param stats
-     * @return
+     *            index stats client
+     * @return the selectivity of the node's identifier
      */
     public static Double getNodeSelectivity(JexlNode node, ShardQueryConfiguration config, IndexStatsClient stats) {
         List idents = getIdentifiers(node);
@@ -1580,9 +1646,12 @@ public static Double getNodeSelectivity(JexlNode node, ShardQueryConfiguration c
      * Return the selectivity of the node's identifier, or IndexStatsClient.DEFAULT_VALUE if there's an error getting the selectivity
      *
      * @param fieldNames
+     *            Set of field names
      * @param config
+     *            shard query configuration
      * @param stats
-     * @return
+     *            the IndexStatsClient
+     * @return the selectivity of the node's identifier
      */
     public static Double getNodeSelectivity(Set fieldNames, ShardQueryConfiguration config, IndexStatsClient stats) {
 
diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/EdgeTableRangeBuildingVisitor.java b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/EdgeTableRangeBuildingVisitor.java
index 71efe38781c..3365160bef0 100644
--- a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/EdgeTableRangeBuildingVisitor.java
+++ b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/EdgeTableRangeBuildingVisitor.java
@@ -43,6 +43,7 @@
  * The high level design of how the parsing works is as follows: There are two data structures that are built during the traversal to keep track of information.
  * Once the traversal is complete and the data structures are built they are sent to the VisitationContext which then builds the ranges and normalized query and
  * gets returned.
        
+ * 
        
  * 
        
  * 
        
  * All data is expected to be passed up since this is a depth first search nothing will be passed down
        
@@ -70,7 +71,6 @@
  * ranges/normalized query.
        
  * 
        
  * 
        
- * 
        
  * The 3 basic rules that are enforced are:
        
  * for equivalence expressions (== =~)
        
  * 1) cant and like identifiers
        
diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypePruningVisitor.java b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypePruningVisitor.java
index 91d64146091..b182d73da3c 100644
--- a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypePruningVisitor.java
+++ b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypePruningVisitor.java
@@ -50,14 +50,17 @@
 import datawave.query.util.TypeMetadata;
 
 /**
+ * 
        
  * This visitor addresses the case when multiple ingest types share some but not all fields in a query
+ * 
        
  * 
        
  * Consider the query (A AND B) where term A maps to ingest type 1 and term B maps to ingest type 2. No document will ever satisfy this
  * intersection. Thus, this should prune to zero terms.
+ * 
        
  * 
        
  * Consider the query (A AND (B OR C)) where term A and term B map to ingest type 1 and term C maps to ingest type 2. In this case term C should be
  * pruned from the nested union leaving the intersection (A AND B)
- * 
        
+ * 
        
  */
 public class IngestTypePruningVisitor extends BaseVisitor {
     private static final Logger log = Logger.getLogger(IngestTypePruningVisitor.class);
diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/QueryModelVisitor.java b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/QueryModelVisitor.java
index fb233260c36..0fb2589ec9b 100644
--- a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/QueryModelVisitor.java
+++ b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/QueryModelVisitor.java
@@ -474,6 +474,8 @@ private static boolean getIdentifiers(JexlNode node, List identif
      *
      * @param field
      *            the field to be expanded
+     * @param noExpansionFields
+     *            fields to not expand on
      * @return true if the field is excluded
      */
     public static boolean isFieldExcluded(String field, Set noExpansionFields) {
diff --git a/warehouse/query-core/src/main/java/datawave/query/language/processor/lucene/CustomAnalyzerQueryNodeProcessor.java b/warehouse/query-core/src/main/java/datawave/query/language/processor/lucene/CustomAnalyzerQueryNodeProcessor.java
index dc34494109a..82a30c5be99 100644
--- a/warehouse/query-core/src/main/java/datawave/query/language/processor/lucene/CustomAnalyzerQueryNodeProcessor.java
+++ b/warehouse/query-core/src/main/java/datawave/query/language/processor/lucene/CustomAnalyzerQueryNodeProcessor.java
@@ -50,8 +50,9 @@
 import datawave.query.language.parser.jexl.LuceneToJexlQueryParser;
 
 /**
- * Applies tokenization to {@link TextableQueryNode} objects using a configured Lucene {@link Analyzer}.
  * 
        
+ * Applies tokenization to {@link TextableQueryNode} objects using a configured Lucene {@link Analyzer}.
+ * 
        
  *
  * Uses the {@link Analyzer} specified in the the {@link ConfigurationKeys#ANALYZER} attribute of the {@link QueryConfigHandler} to process non-wildcard
  * {@link FieldQueryNode}s for fields listed in tokenizedFields.
@@ -62,19 +63,20 @@
  * The text of each {@link TextableQueryNode} is processed using the {@link Analyzer} to generate tokens. If the analyzer returns one or more terms that are not
  * identical to the input, the processor generates an {@link OrQueryNode} containing the original query node and a new {@link QuotedFieldQueryNode} or
  * {@link SlopQueryNode} depending on the nature of the original query node and whether useSlopForTokenizedTerms is false.
- *
+ * 
        
  * There are three primary cases where tokenization will be applied to input query terms - single terms (e.g: wi-fi), phrases (e.g: "portable wi-fi"), and
  * phrases with slop (e.g: "portable wi-fi"~3). In the case of single term input, tokenization will produce a phrase with slop equals to the number of positions
  * in the original query if useSlopForTokenizedTerms is set to true, otherwise a phrase without slop will be produced. In the case of
  * phrase input, a new phrase query will be generated with the new tokens. In t he case of a phrase with slop, a new phrase with slop will be generated and an
  * attempt will be made to adjust the slop based on the number of additional tokens generated. For exa mple, in the case of the slop query above, the new query
  * will be "portable wi fi"~4 because an additional token was generated based on the split of 'wi' and 'fi' into two separate tokens.
- *
+ * 
        
+ * 
        
  * FieldQueryNodes with empty fields are considered 'unfielded' and will be tokenized if unfieldedTokenized is true. The
  * skipTokenizeUnfieldedFields can be used in this case to indicate that a node should be treated as un-fielded but not tokenized. When this
  * processor encounters such a field in a node, it will not tokenize the text of that node and will set that node's field to an empty string so that downstream
  * processors will treat the node as if it is un-fielded.
- * 
        
+ * 
        
  *
  * @see Analyzer
  * @see TokenStream
diff --git a/warehouse/query-core/src/main/java/datawave/query/predicate/EventDataQueryFieldFilter.java b/warehouse/query-core/src/main/java/datawave/query/predicate/EventDataQueryFieldFilter.java
index 7e46b468f58..29d19c50bcc 100644
--- a/warehouse/query-core/src/main/java/datawave/query/predicate/EventDataQueryFieldFilter.java
+++ b/warehouse/query-core/src/main/java/datawave/query/predicate/EventDataQueryFieldFilter.java
@@ -35,6 +35,11 @@ public EventDataQueryFieldFilter(EventDataQueryFieldFilter other) {
 
     /**
      * Initialize filter with an empty projection
+     *
+     * @param projections
+     *            the projection
+     * @param projectionType
+     *            the projection type
      */
     public EventDataQueryFieldFilter(Set projections, Projection.ProjectionType projectionType) {
         this.keyProjection = new KeyProjection(projections, projectionType);
@@ -44,6 +49,7 @@ public EventDataQueryFieldFilter(Set projections, Projection.ProjectionT
      * Initiate from a KeyProjection
      *
      * @param projection
+     *            the projection
      */
     public EventDataQueryFieldFilter(KeyProjection projection) {
         this.keyProjection = projection;
diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/ExcerptTransform.java b/warehouse/query-core/src/main/java/datawave/query/transformer/ExcerptTransform.java
index ae6e0d2118b..e196023c98e 100644
--- a/warehouse/query-core/src/main/java/datawave/query/transformer/ExcerptTransform.java
+++ b/warehouse/query-core/src/main/java/datawave/query/transformer/ExcerptTransform.java
@@ -309,6 +309,8 @@ private Set getExcerpts(PhraseIndexes phraseIndexes) {
      *            the end index of the excerpt
      * @param range
      *            the range to use when seeking
+     * @param hitTermValues
+     *            the term values to match
      * @return the excerpt
      */
     private String getExcerpt(String field, int start, int end, Range range, String hitTermValues) {
diff --git a/warehouse/query-core/src/main/java/datawave/query/util/DateIndexHelper.java b/warehouse/query-core/src/main/java/datawave/query/util/DateIndexHelper.java
index 9cdabcc4632..615123b1b52 100644
--- a/warehouse/query-core/src/main/java/datawave/query/util/DateIndexHelper.java
+++ b/warehouse/query-core/src/main/java/datawave/query/util/DateIndexHelper.java
@@ -43,7 +43,7 @@
  * Helper class to fetch data from the date index.
  * 
        
  * 
    EventMetadata
    Schema TypeUse
    
- * 
+ * 
  * 
  * 
  * 
diff --git a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/BucketAccumuloKeyGenerator.java b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/BucketAccumuloKeyGenerator.java
index ae76cc999ea..cb1646af1ee 100644
--- a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/BucketAccumuloKeyGenerator.java
+++ b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/BucketAccumuloKeyGenerator.java
@@ -70,7 +70,8 @@ public BucketAccumuloKeyGenerator(int bucketCount, int bucketEncodingBase, int b
      * column family is the integer chunk size and the column qualifier is the original ssdeep hash bytes. The generated value is always empty.
      *
      * @param t
-     * @return
+     *            tuple
+     * @return Accumulo key/value pair when unable to produce an accumulo key/value pair
      */
     public ImmutablePair call(ImmutablePair t) {
         int rowSize = t.getKey().getChunk().length() + bucketEncoding.getLength() + chunkEncoding.getLength();
diff --git a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/HashReverse.java b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/HashReverse.java
index 0ccd630035d..4aad9bef04b 100644
--- a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/HashReverse.java
+++ b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/HashReverse.java
@@ -56,8 +56,11 @@ public static int getPrefixIndex(final String hash, final int length) {
      * character in the array of acceptable base64 characters.
      *
      * @param hash
+     *            the hash string
      * @param length
-     * @return
+     *            length of the prefix
+     *
+     * @return prefix index
      */
     public static int getPrefixIndex(final byte[] hash, final int length) {
         int result = 0;
@@ -83,13 +86,23 @@ public static int getPrefixIndex(final byte[] hash, final int length) {
      * Return the max possible value for the provided prefix length
      *
      * @param length
-     * @return
+     *            length of the prefix
+     *
+     * @return the prefix
      */
     public static int getPrefixMax(final int length) {
         return getPrefixIndex(MAX_HASH, length);
     }
 
-    /** Utility to generate splits for the ssdeep table based on a prefix of 2 - 64^64 (4096) splits in size */
+    /**
+     * Utility to generate splits for the ssdeep table based on a prefix of 2 - 64^64 (4096) splits in size
+     *
+     * @param args
+     *            main arguments passed in
+     *
+     * @throws IOException
+     *             if there is a problem splitting the files
+     */
     public static void main(String[] args) throws IOException {
         int len = LEXICAL_B64_TABLE.length;
         byte[] output = new byte[2];
diff --git a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/IntegerEncoding.java b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/IntegerEncoding.java
index 42acaf9f3b8..f3ff4393e1b 100644
--- a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/IntegerEncoding.java
+++ b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/IntegerEncoding.java
@@ -46,7 +46,11 @@ public IntegerEncoding(int base, int length) {
         this.limit = (int) calculatedLimit; // truncation is fine here.
     }
 
-    /** Return the maximum value this encoder can encode */
+    /**
+     * Return the maximum value this encoder can encode
+     *
+     * @return the limit
+     */
     public int getLimit() {
         return limit;
     }
@@ -55,13 +59,20 @@ public int getLength() {
         return length;
     }
 
-    /** Encode the provided value, return a string result */
+    /**
+     * Encode the provided value, return a string result
+     *
+     * @param value
+     *            integer value to encode
+     *
+     * @return the encoded string
+     */
     public String encode(int value) {
         return new String(encodeToBytes(value, new byte[length], 0));
     }
 
     /**
-     * encode the provided value, writing the result to the provided buffer starting offset
+     * Encode the provided value, writing the result to the provided buffer starting offset
      *
      * @param value
      *            the value to encode
@@ -114,6 +125,7 @@ public static byte[] encodeBaseTenDigitBytes(int value) {
      *
      * @param encodedValue
      *            the string to decode
+     *
      * @return the decoded result
      */
     public int decode(String encodedValue) {
diff --git a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepHashScorer.java b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepHashScorer.java
index 54db7467262..610e3360d22 100644
--- a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepHashScorer.java
+++ b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepHashScorer.java
@@ -18,8 +18,8 @@ public SSDeepHashScorer(int maxRepeatedCharacters) {
     }
 
     /**
-     * Compare two ssdeep hashes, returning a score between 0 to 100 that indicates similarity. A score of 0 means that the items are not similar at all whereas
-     * a score of 100 indicates a high degree of similarity.
+     * Compare two ssdeep hashes, returning a score between 0 and 100 that indicates similarity. A score of 0 means that the items are not similar at all
+     * whereas a score of 100 indicates a high degree of similarity.
      *
      * @param signature1
      *            the first object to be compared.
@@ -72,6 +72,15 @@ public int apply(SSDeepHash signature1, SSDeepHash signature2) {
     /**
      * This is the low level chunk scoring algorithm. It takes two chunks and scores them on a scale of 0-100 where 0 is a terrible match and 100 is a great
      * match. The chunkSize is used to cope with very small messages.
+     *
+     * @param s1
+     *            chunk 1
+     * @param s2
+     *            chunk 2
+     * @param chunkSize
+     *            size of the chunk
+     *
+     * @return score of 0-100 depending on matching calculation
      */
     private static int scoreChunks(final String s1, final String s2, final long chunkSize) {
         final int len1 = s1.length();
diff --git a/web-services/cached-results/src/main/java/datawave/webservice/results/cached/CachedResultsBean.java b/web-services/cached-results/src/main/java/datawave/webservice/results/cached/CachedResultsBean.java
index 64b7897b6e9..cd7697272cc 100644
--- a/web-services/cached-results/src/main/java/datawave/webservice/results/cached/CachedResultsBean.java
+++ b/web-services/cached-results/src/main/java/datawave/webservice/results/cached/CachedResultsBean.java
@@ -738,7 +738,6 @@ else if (maxValueLength > maxLength) {
      *            a query id
      * @return List of attribute names that can be used in subsequent queries
      *
-     * @return {@code datawave.webservice.result.GenericResponse}
      * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user by specifying a chain of DNs of the identities to proxy
      * @RequestHeader X-ProxiedIssuersChain required when using X-ProxiedEntitiesChain, specify one issuer DN per subject DN listed in X-ProxiedEntitiesChain
      * @ResponseHeader X-OperationTimeInMS time spent on the server performing the operation, does not account for network or result serialization
@@ -862,7 +861,6 @@ protected void persistBatch(PreparedStatement ps) throws SQLException {
      *            additional name that this query can be retrieved by
      * @return name of the view for this query, use it as the table name in the SQL query
      *
-     * @return {@code datawave.webservice.result.GenericResponse}
      * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user by specifying a chain of DNs of the identities to proxy
      * @RequestHeader X-ProxiedIssuersChain required when using X-ProxiedEntitiesChain, specify one issuer DN per subject DN listed in X-ProxiedEntitiesChain
      * @ResponseHeader X-OperationTimeInMS time spent on the server performing the operation, does not account for network or result serialization
@@ -1096,7 +1094,6 @@ public Future loadAndCreateAsync(MultivaluedMap}
      * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user by specifying a chain of DNs of the identities to proxy
      * @RequestHeader X-ProxiedIssuersChain required when using X-ProxiedEntitiesChain, specify one issuer DN per subject DN listed in X-ProxiedEntitiesChain
      * @ResponseHeader X-OperationTimeInMS time spent on the server performing the operation, does not account for network or result serialization
@@ -1473,7 +1470,6 @@ public CachedResultsResponse update(@PathParam("queryId") @Required("queryId") S
      *            user defined id for this query
      * @return previous page of results
      *
-     * @return datawave.webservice.result.BaseQueryResponse
      * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user by specifying a chain of DNs of the identities to proxy
      * @RequestHeader X-ProxiedIssuersChain required when using X-ProxiedEntitiesChain, specify one issuer DN per subject DN listed in X-ProxiedEntitiesChain
      * @RequestHeader query-session-id session id value used for load balancing purposes. query-session-id can be placed in the request in a Cookie header or as
@@ -1681,7 +1677,6 @@ public CachedResultsResponse reset(@PathParam("queryId") @Required("queryId") St
      *            user defined id for this query
      * @return a page of results
      *
-     * @return datawave.webservice.result.BaseQueryResponse
      * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user by specifying a chain of DNs of the identities to proxy
      * @RequestHeader X-ProxiedIssuersChain required when using X-ProxiedEntitiesChain, specify one issuer DN per subject DN listed in X-ProxiedEntitiesChain
      * @RequestHeader query-session-id session id value used for load balancing purposes. query-session-id can be placed in the request in a Cookie header or as
diff --git a/web-services/client/src/main/java/datawave/webservice/query/QueryMessages.java b/web-services/client/src/main/java/datawave/webservice/query/QueryMessages.java
index 1c6fdace158..e9e69f40b92 100644
--- a/web-services/client/src/main/java/datawave/webservice/query/QueryMessages.java
+++ b/web-services/client/src/main/java/datawave/webservice/query/QueryMessages.java
@@ -13,172 +13,239 @@ public interface QueryImplOrBuilder extends com.google.protobuf.MessageLiteOrBui
         // required string query_logic_name = 1;
         /**
          * required string query_logic_name = 1;
+         *
+         * @return if it has query logic name
          */
         boolean hasQueryLogicName();
 
         /**
          * required string query_logic_name = 1;
+         *
+         * @return query logic name
          */
         java.lang.String getQueryLogicName();
 
         /**
          * required string query_logic_name = 1;
+         *
+         * @return query logic name byte string
          */
         com.google.protobuf.ByteString getQueryLogicNameBytes();
 
         // required string id = 2;
         /**
          * required string id = 2;
+         *
+         * @return if it has id
          */
         boolean hasId();
 
         /**
          * required string id = 2;
+         *
+         * @return the id
          */
         java.lang.String getId();
 
         /**
          * required string id = 2;
+         *
+         * @return id byte string
          */
         com.google.protobuf.ByteString getIdBytes();
 
         // optional string query_name = 3;
         /**
          * optional string query_name = 3;
+         *
+         * @return if it has query name
          */
         boolean hasQueryName();
 
         /**
          * optional string query_name = 3;
+         *
+         * @return query name
          */
         java.lang.String getQueryName();
 
         /**
          * optional string query_name = 3;
+         *
+         * @return query name byte string
          */
         com.google.protobuf.ByteString getQueryNameBytes();
 
         // required string user_DN = 4;
         /**
          * required string user_DN = 4;
+         *
+         * @return if it has user dn
          */
         boolean hasUserDN();
 
         /**
          * required string user_DN = 4;
+         *
+         * @return user dn
          */
         java.lang.String getUserDN();
 
         /**
          * required string user_DN = 4;
+         *
+         * @return user dn byte string
          */
         com.google.protobuf.ByteString getUserDNBytes();
 
         // required string query = 5;
         /**
          * required string query = 5;
+         *
+         * @return if it has a query
          */
         boolean hasQuery();
 
         /**
          * required string query = 5;
+         *
+         * @return the query
          */
         java.lang.String getQuery();
 
         /**
          * required string query = 5;
+         *
+         * @return query byte string
          */
         com.google.protobuf.ByteString getQueryBytes();
 
         // required string query_visibility = 6;
         /**
          * required string query_visibility = 6;
+         *
+         * @return if it has query visibility
          */
         boolean hasQueryVisibility();
 
         /**
          * required string query_visibility = 6;
+         *
+         * @return the query visibility
          */
         java.lang.String getQueryVisibility();
 
         /**
          * required string query_visibility = 6;
+         *
+         * @return query visibility byte string
          */
         com.google.protobuf.ByteString getQueryVisibilityBytes();
 
         // optional int64 begin_date = 7;
         /**
          * optional int64 begin_date = 7;
+         *
+         * @return if it has a begin date
          */
         boolean hasBeginDate();
 
         /**
          * optional int64 begin_date = 7;
+         *
+         * @return begin date
          */
         long getBeginDate();
 
         // optional int64 end_date = 8;
         /**
          * optional int64 end_date = 8;
+         *
+         * @return if it has end date
          */
         boolean hasEndDate();
 
         /**
          * optional int64 end_date = 8;
+         *
+         * @return end date
          */
         long getEndDate();
 
         // required string query_authorizations = 9;
         /**
          * required string query_authorizations = 9;
+         *
+         * @return if it has query authorizations
          */
         boolean hasQueryAuthorizations();
 
         /**
          * required string query_authorizations = 9;
+         *
+         * @return query authorizations
          */
         java.lang.String getQueryAuthorizations();
 
         /**
          * required string query_authorizations = 9;
+         *
+         * @return query authorization
          */
         com.google.protobuf.ByteString getQueryAuthorizationsBytes();
 
         // required int64 expiration_date = 10;
         /**
          * required int64 expiration_date = 10;
+         *
+         * @return if it has an expiration date
          */
         boolean hasExpirationDate();
 
         /**
          * required int64 expiration_date = 10;
+         *
+         * @return the expiration date
          */
         long getExpirationDate();
 
         // required uint32 pagesize = 11 [default = 0];
         /**
          * required uint32 pagesize = 11 [default = 0];
+         *
+         * @return if it has page size
          */
         boolean hasPagesize();
 
         /**
          * required uint32 pagesize = 11 [default = 0];
+         *
+         * @return the page size
          */
         int getPagesize();
 
         // repeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
         /**
          * repeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
+         *
+         * @return the parameters list
          */
         java.util.List getParametersList();
 
         /**
          * repeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the parameter at the index provided
          */
         datawave.webservice.query.QueryMessages.QueryImpl.Parameter getParameters(int index);
 
         /**
          * repeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
+         *
+         * @return the number of parameters
          */
         int getParametersCount();
     }
@@ -317,32 +384,44 @@ public interface ParameterOrBuilder extends com.google.protobuf.MessageLiteOrBui
             // required string parameter_name = 1;
             /**
              * required string parameter_name = 1;
+             *
+             * @return if it has parameter name
              */
             boolean hasParameterName();
 
             /**
              * required string parameter_name = 1;
+             *
+             * @return parameter name
              */
             java.lang.String getParameterName();
 
             /**
              * required string parameter_name = 1;
+             *
+             * @return parameter name byte string
              */
             com.google.protobuf.ByteString getParameterNameBytes();
 
             // required string parameter_value = 2;
             /**
              * required string parameter_value = 2;
+             *
+             * @return if it has parameter value
              */
             boolean hasParameterValue();
 
             /**
              * required string parameter_value = 2;
+             *
+             * @return parameter value
              */
             java.lang.String getParameterValue();
 
             /**
              * required string parameter_value = 2;
+             *
+             * @return parameter value byte string
              */
             com.google.protobuf.ByteString getParameterValueBytes();
         }
@@ -427,6 +506,8 @@ public com.google.protobuf.Parser getParserForType() {
 
             /**
              * required string parameter_name = 1;
+             *
+             * @return if it has parameter name
              */
             public boolean hasParameterName() {
                 return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -434,6 +515,8 @@ public boolean hasParameterName() {
 
             /**
              * required string parameter_name = 1;
+             *
+             * @return parameter name
              */
             public java.lang.String getParameterName() {
                 java.lang.Object ref = parameterName_;
@@ -451,6 +534,8 @@ public java.lang.String getParameterName() {
 
             /**
              * required string parameter_name = 1;
+             *
+             * @return parameter name byte string
              */
             public com.google.protobuf.ByteString getParameterNameBytes() {
                 java.lang.Object ref = parameterName_;
@@ -469,6 +554,8 @@ public com.google.protobuf.ByteString getParameterNameBytes() {
 
             /**
              * required string parameter_value = 2;
+             *
+             * @return if it has parameter value
              */
             public boolean hasParameterValue() {
                 return ((bitField0_ & 0x00000002) == 0x00000002);
@@ -476,6 +563,8 @@ public boolean hasParameterValue() {
 
             /**
              * required string parameter_value = 2;
+             *
+             * @return parameter value
              */
             public java.lang.String getParameterValue() {
                 java.lang.Object ref = parameterValue_;
@@ -493,6 +582,8 @@ public java.lang.String getParameterValue() {
 
             /**
              * required string parameter_value = 2;
+             *
+             * @return parameter value byte string
              */
             public com.google.protobuf.ByteString getParameterValueBytes() {
                 java.lang.Object ref = parameterValue_;
@@ -737,6 +828,8 @@ public Builder mergeFrom(com.google.protobuf.CodedInputStream input, com.google.
 
                 /**
                  * required string parameter_name = 1;
+                 *
+                 * @return if it has parameter name
                  */
                 public boolean hasParameterName() {
                     return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -744,6 +837,8 @@ public boolean hasParameterName() {
 
                 /**
                  * required string parameter_name = 1;
+                 *
+                 * @return parameter name
                  */
                 public java.lang.String getParameterName() {
                     java.lang.Object ref = parameterName_;
@@ -758,6 +853,8 @@ public java.lang.String getParameterName() {
 
                 /**
                  * required string parameter_name = 1;
+                 *
+                 * @return parameter name byte string
                  */
                 public com.google.protobuf.ByteString getParameterNameBytes() {
                     java.lang.Object ref = parameterName_;
@@ -772,6 +869,11 @@ public com.google.protobuf.ByteString getParameterNameBytes() {
 
                 /**
                  * required string parameter_name = 1;
+                 *
+                 * @param value
+                 *            new parameter name
+                 *
+                 * @return builder to set parameter name to passed in value
                  */
                 public Builder setParameterName(java.lang.String value) {
                     if (value == null) {
@@ -785,6 +887,8 @@ public Builder setParameterName(java.lang.String value) {
 
                 /**
                  * required string parameter_name = 1;
+                 *
+                 * @return builder to clear the parameter name
                  */
                 public Builder clearParameterName() {
                     bitField0_ = (bitField0_ & ~0x00000001);
@@ -795,6 +899,11 @@ public Builder clearParameterName() {
 
                 /**
                  * required string parameter_name = 1;
+                 *
+                 * @param value
+                 *            new parameter name
+                 *
+                 * @return builder to set parameter name
                  */
                 public Builder setParameterNameBytes(com.google.protobuf.ByteString value) {
                     if (value == null) {
@@ -811,6 +920,8 @@ public Builder setParameterNameBytes(com.google.protobuf.ByteString value) {
 
                 /**
                  * required string parameter_value = 2;
+                 *
+                 * @return if it has parameter value
                  */
                 public boolean hasParameterValue() {
                     return ((bitField0_ & 0x00000002) == 0x00000002);
@@ -818,6 +929,8 @@ public boolean hasParameterValue() {
 
                 /**
                  * required string parameter_value = 2;
+                 *
+                 * @return parameter value
                  */
                 public java.lang.String getParameterValue() {
                     java.lang.Object ref = parameterValue_;
@@ -832,6 +945,8 @@ public java.lang.String getParameterValue() {
 
                 /**
                  * required string parameter_value = 2;
+                 *
+                 * @return parameter value byte string
                  */
                 public com.google.protobuf.ByteString getParameterValueBytes() {
                     java.lang.Object ref = parameterValue_;
@@ -846,6 +961,11 @@ public com.google.protobuf.ByteString getParameterValueBytes() {
 
                 /**
                  * required string parameter_value = 2;
+                 *
+                 * @param value
+                 *            the new parameter value
+                 *
+                 * @return builder to set parameter value
                  */
                 public Builder setParameterValue(java.lang.String value) {
                     if (value == null) {
@@ -859,6 +979,8 @@ public Builder setParameterValue(java.lang.String value) {
 
                 /**
                  * required string parameter_value = 2;
+                 *
+                 * @return builder to clear parameter value
                  */
                 public Builder clearParameterValue() {
                     bitField0_ = (bitField0_ & ~0x00000002);
@@ -869,6 +991,11 @@ public Builder clearParameterValue() {
 
                 /**
                  * required string parameter_value = 2;
+                 *
+                 * @param value
+                 *            new value
+                 *
+                 * @return builder to set parameter value
                  */
                 public Builder setParameterValueBytes(com.google.protobuf.ByteString value) {
                     if (value == null) {
@@ -898,6 +1025,8 @@ public Builder setParameterValueBytes(com.google.protobuf.ByteString value) {
 
         /**
          * required string query_logic_name = 1;
+         *
+         * @return if it has query logic name
          */
         public boolean hasQueryLogicName() {
             return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -905,6 +1034,8 @@ public boolean hasQueryLogicName() {
 
         /**
          * required string query_logic_name = 1;
+         *
+         * @return query logic name
          */
         public java.lang.String getQueryLogicName() {
             java.lang.Object ref = queryLogicName_;
@@ -922,6 +1053,8 @@ public java.lang.String getQueryLogicName() {
 
         /**
          * required string query_logic_name = 1;
+         *
+         * @return query logic name byte string
          */
         public com.google.protobuf.ByteString getQueryLogicNameBytes() {
             java.lang.Object ref = queryLogicName_;
@@ -940,6 +1073,8 @@ public com.google.protobuf.ByteString getQueryLogicNameBytes() {
 
         /**
          * required string id = 2;
+         *
+         * @return if it has id
          */
         public boolean hasId() {
             return ((bitField0_ & 0x00000002) == 0x00000002);
@@ -947,6 +1082,8 @@ public boolean hasId() {
 
         /**
          * required string id = 2;
+         *
+         * @return the id
          */
         public java.lang.String getId() {
             java.lang.Object ref = id_;
@@ -964,6 +1101,8 @@ public java.lang.String getId() {
 
         /**
          * required string id = 2;
+         *
+         * @return the id byte string
          */
         public com.google.protobuf.ByteString getIdBytes() {
             java.lang.Object ref = id_;
@@ -982,6 +1121,8 @@ public com.google.protobuf.ByteString getIdBytes() {
 
         /**
          * optional string query_name = 3;
+         *
+         * @return if it has query name
          */
         public boolean hasQueryName() {
             return ((bitField0_ & 0x00000004) == 0x00000004);
@@ -989,6 +1130,8 @@ public boolean hasQueryName() {
 
         /**
          * optional string query_name = 3;
+         *
+         * @return the query name
          */
         public java.lang.String getQueryName() {
             java.lang.Object ref = queryName_;
@@ -1006,6 +1149,8 @@ public java.lang.String getQueryName() {
 
         /**
          * optional string query_name = 3;
+         *
+         * @return the query name byte string
          */
         public com.google.protobuf.ByteString getQueryNameBytes() {
             java.lang.Object ref = queryName_;
@@ -1024,6 +1169,8 @@ public com.google.protobuf.ByteString getQueryNameBytes() {
 
         /**
          * required string user_DN = 4;
+         *
+         * @return if it has user dn
          */
         public boolean hasUserDN() {
             return ((bitField0_ & 0x00000008) == 0x00000008);
@@ -1031,6 +1178,8 @@ public boolean hasUserDN() {
 
         /**
          * required string user_DN = 4;
+         *
+         * @return the user dn
          */
         public java.lang.String getUserDN() {
             java.lang.Object ref = userDN_;
@@ -1048,6 +1197,8 @@ public java.lang.String getUserDN() {
 
         /**
          * required string user_DN = 4;
+         *
+         * @return the user dn byte string
          */
         public com.google.protobuf.ByteString getUserDNBytes() {
             java.lang.Object ref = userDN_;
@@ -1066,6 +1217,8 @@ public com.google.protobuf.ByteString getUserDNBytes() {
 
         /**
          * required string query = 5;
+         *
+         * @return if it has query
          */
         public boolean hasQuery() {
             return ((bitField0_ & 0x00000010) == 0x00000010);
@@ -1073,6 +1226,8 @@ public boolean hasQuery() {
 
         /**
          * required string query = 5;
+         *
+         * @return the query
          */
         public java.lang.String getQuery() {
             java.lang.Object ref = query_;
@@ -1090,6 +1245,8 @@ public java.lang.String getQuery() {
 
         /**
          * required string query = 5;
+         *
+         * @return the query byte string
          */
         public com.google.protobuf.ByteString getQueryBytes() {
             java.lang.Object ref = query_;
@@ -1108,6 +1265,8 @@ public com.google.protobuf.ByteString getQueryBytes() {
 
         /**
          * required string query_visibility = 6;
+         *
+         * @return if it has query visibility
          */
         public boolean hasQueryVisibility() {
             return ((bitField0_ & 0x00000020) == 0x00000020);
@@ -1115,6 +1274,8 @@ public boolean hasQueryVisibility() {
 
         /**
          * required string query_visibility = 6;
+         *
+         * @return the query visibility
          */
         public java.lang.String getQueryVisibility() {
             java.lang.Object ref = queryVisibility_;
@@ -1132,6 +1293,8 @@ public java.lang.String getQueryVisibility() {
 
         /**
          * required string query_visibility = 6;
+         *
+         * @return the query visibility byte string
          */
         public com.google.protobuf.ByteString getQueryVisibilityBytes() {
             java.lang.Object ref = queryVisibility_;
@@ -1150,6 +1313,8 @@ public com.google.protobuf.ByteString getQueryVisibilityBytes() {
 
         /**
          * optional int64 begin_date = 7;
+         *
+         * @return if it has begin date
          */
         public boolean hasBeginDate() {
             return ((bitField0_ & 0x00000040) == 0x00000040);
@@ -1157,6 +1322,8 @@ public boolean hasBeginDate() {
 
         /**
          * optional int64 begin_date = 7;
+         *
+         * @return the begin date
          */
         public long getBeginDate() {
             return beginDate_;
@@ -1168,6 +1335,8 @@ public long getBeginDate() {
 
         /**
          * optional int64 end_date = 8;
+         *
+         * @return if it has end date
          */
         public boolean hasEndDate() {
             return ((bitField0_ & 0x00000080) == 0x00000080);
@@ -1175,6 +1344,8 @@ public boolean hasEndDate() {
 
         /**
          * optional int64 end_date = 8;
+         *
+         * @return the end date
          */
         public long getEndDate() {
             return endDate_;
@@ -1186,6 +1357,8 @@ public long getEndDate() {
 
         /**
          * required string query_authorizations = 9;
+         *
+         * @return if it has query authorizations
          */
         public boolean hasQueryAuthorizations() {
             return ((bitField0_ & 0x00000100) == 0x00000100);
@@ -1193,6 +1366,8 @@ public boolean hasQueryAuthorizations() {
 
         /**
          * required string query_authorizations = 9;
+         *
+         * @return the query authorizations
          */
         public java.lang.String getQueryAuthorizations() {
             java.lang.Object ref = queryAuthorizations_;
@@ -1210,6 +1385,8 @@ public java.lang.String getQueryAuthorizations() {
 
         /**
          * required string query_authorizations = 9;
+         *
+         * @return the query authorizations byte string
          */
         public com.google.protobuf.ByteString getQueryAuthorizationsBytes() {
             java.lang.Object ref = queryAuthorizations_;
@@ -1228,6 +1405,8 @@ public com.google.protobuf.ByteString getQueryAuthorizationsBytes() {
 
         /**
          * required int64 expiration_date = 10;
+         *
+         * @return if it has expiration date
          */
         public boolean hasExpirationDate() {
             return ((bitField0_ & 0x00000200) == 0x00000200);
@@ -1235,6 +1414,8 @@ public boolean hasExpirationDate() {
 
         /**
          * required int64 expiration_date = 10;
+         *
+         * @return the expiration date
          */
         public long getExpirationDate() {
             return expirationDate_;
@@ -1246,6 +1427,8 @@ public long getExpirationDate() {
 
         /**
          * required uint32 pagesize = 11 [default = 0];
+         *
+         * @return if it has page size
          */
         public boolean hasPagesize() {
             return ((bitField0_ & 0x00000400) == 0x00000400);
@@ -1253,6 +1436,8 @@ public boolean hasPagesize() {
 
         /**
          * required uint32 pagesize = 11 [default = 0];
+         *
+         * @return the page size
          */
         public int getPagesize() {
             return pagesize_;
@@ -1264,6 +1449,8 @@ public int getPagesize() {
 
         /**
          * repeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
+         *
+         * @return parameters list
          */
         public java.util.List getParametersList() {
             return parameters_;
@@ -1271,6 +1458,8 @@ public java.util.Listrepeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
+         *
+         * @return parameters list
          */
         public java.util.List getParametersOrBuilderList() {
             return parameters_;
@@ -1278,6 +1467,8 @@ public java.util.Listrepeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
+         *
+         * @return the size of the parameters list
          */
         public int getParametersCount() {
             return parameters_.size();
@@ -1285,6 +1476,11 @@ public int getParametersCount() {
 
         /**
          * repeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the parameter at the given index
          */
         public datawave.webservice.query.QueryMessages.QueryImpl.Parameter getParameters(int index) {
             return parameters_.get(index);
@@ -1292,6 +1488,11 @@ public datawave.webservice.query.QueryMessages.QueryImpl.Parameter getParameters
 
         /**
          * repeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the parameter at the index
          */
         public datawave.webservice.query.QueryMessages.QueryImpl.ParameterOrBuilder getParametersOrBuilder(int index) {
             return parameters_.get(index);
@@ -1764,6 +1965,8 @@ public Builder mergeFrom(com.google.protobuf.CodedInputStream input, com.google.
 
             /**
              * required string query_logic_name = 1;
+             *
+             * @return if it has query logic name
              */
             public boolean hasQueryLogicName() {
                 return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -1771,6 +1974,8 @@ public boolean hasQueryLogicName() {
 
             /**
              * required string query_logic_name = 1;
+             *
+             * @return the query logic name
              */
             public java.lang.String getQueryLogicName() {
                 java.lang.Object ref = queryLogicName_;
@@ -1785,6 +1990,8 @@ public java.lang.String getQueryLogicName() {
 
             /**
              * required string query_logic_name = 1;
+             *
+             * @return the query logic name byte string
              */
             public com.google.protobuf.ByteString getQueryLogicNameBytes() {
                 java.lang.Object ref = queryLogicName_;
@@ -1799,6 +2006,11 @@ public com.google.protobuf.ByteString getQueryLogicNameBytes() {
 
             /**
              * required string query_logic_name = 1;
+             *
+             * @param value
+             *            the new query logic name
+             *
+             * @return builder to set the query logic name to a new value
              */
             public Builder setQueryLogicName(java.lang.String value) {
                 if (value == null) {
@@ -1812,6 +2024,8 @@ public Builder setQueryLogicName(java.lang.String value) {
 
             /**
              * required string query_logic_name = 1;
+             *
+             * @return builder to clear the query logic name
              */
             public Builder clearQueryLogicName() {
                 bitField0_ = (bitField0_ & ~0x00000001);
@@ -1822,6 +2036,11 @@ public Builder clearQueryLogicName() {
 
             /**
              * required string query_logic_name = 1;
+             *
+             * @param value
+             *            the value
+             *
+             * @return builder to set the query logic name byte string
              */
             public Builder setQueryLogicNameBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -1838,6 +2057,8 @@ public Builder setQueryLogicNameBytes(com.google.protobuf.ByteString value) {
 
             /**
              * required string id = 2;
+             *
+             * @return if it has id
              */
             public boolean hasId() {
                 return ((bitField0_ & 0x00000002) == 0x00000002);
@@ -1845,6 +2066,8 @@ public boolean hasId() {
 
             /**
              * required string id = 2;
+             *
+             * @return the id
              */
             public java.lang.String getId() {
                 java.lang.Object ref = id_;
@@ -1859,6 +2082,8 @@ public java.lang.String getId() {
 
             /**
              * required string id = 2;
+             *
+             * @return the id byte string
              */
             public com.google.protobuf.ByteString getIdBytes() {
                 java.lang.Object ref = id_;
@@ -1873,6 +2098,11 @@ public com.google.protobuf.ByteString getIdBytes() {
 
             /**
              * required string id = 2;
+             *
+             * @param value
+             *            the new id value
+             *
+             * @return builder to set the new id
              */
             public Builder setId(java.lang.String value) {
                 if (value == null) {
@@ -1886,6 +2116,8 @@ public Builder setId(java.lang.String value) {
 
             /**
              * required string id = 2;
+             *
+             * @return builder to clear the id
              */
             public Builder clearId() {
                 bitField0_ = (bitField0_ & ~0x00000002);
@@ -1896,6 +2128,11 @@ public Builder clearId() {
 
             /**
              * required string id = 2;
+             *
+             * @param value
+             *            byte string for the new id
+             *
+             * @return builder to set the id
              */
             public Builder setIdBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -1912,6 +2149,8 @@ public Builder setIdBytes(com.google.protobuf.ByteString value) {
 
             /**
              * optional string query_name = 3;
+             *
+             * @return if it has query name
              */
             public boolean hasQueryName() {
                 return ((bitField0_ & 0x00000004) == 0x00000004);
@@ -1919,6 +2158,8 @@ public boolean hasQueryName() {
 
             /**
              * optional string query_name = 3;
+             *
+             * @return the query name
              */
             public java.lang.String getQueryName() {
                 java.lang.Object ref = queryName_;
@@ -1933,6 +2174,8 @@ public java.lang.String getQueryName() {
 
             /**
              * optional string query_name = 3;
+             *
+             * @return the query name byte string
              */
             public com.google.protobuf.ByteString getQueryNameBytes() {
                 java.lang.Object ref = queryName_;
@@ -1947,6 +2190,11 @@ public com.google.protobuf.ByteString getQueryNameBytes() {
 
             /**
              * optional string query_name = 3;
+             *
+             * @param value
+             *            the new query name
+             *
+             * @return builder to set the query name to the new value
              */
             public Builder setQueryName(java.lang.String value) {
                 if (value == null) {
@@ -1960,6 +2208,8 @@ public Builder setQueryName(java.lang.String value) {
 
             /**
              * optional string query_name = 3;
+             *
+             * @return builder to clear the query name
              */
             public Builder clearQueryName() {
                 bitField0_ = (bitField0_ & ~0x00000004);
@@ -1970,6 +2220,11 @@ public Builder clearQueryName() {
 
             /**
              * optional string query_name = 3;
+             *
+             * @param value
+             *            new query name as byte string
+             *
+             * @return builder to set the query name to a new value
              */
             public Builder setQueryNameBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -1986,6 +2241,8 @@ public Builder setQueryNameBytes(com.google.protobuf.ByteString value) {
 
             /**
              * required string user_DN = 4;
+             *
+             * @return if it has user dn
              */
             public boolean hasUserDN() {
                 return ((bitField0_ & 0x00000008) == 0x00000008);
@@ -1993,6 +2250,8 @@ public boolean hasUserDN() {
 
             /**
              * required string user_DN = 4;
+             *
+             * @return the user dn
              */
             public java.lang.String getUserDN() {
                 java.lang.Object ref = userDN_;
@@ -2007,6 +2266,8 @@ public java.lang.String getUserDN() {
 
             /**
              * required string user_DN = 4;
+             *
+             * @return the user dn byte string
              */
             public com.google.protobuf.ByteString getUserDNBytes() {
                 java.lang.Object ref = userDN_;
@@ -2021,6 +2282,11 @@ public com.google.protobuf.ByteString getUserDNBytes() {
 
             /**
              * required string user_DN = 4;
+             *
+             * @param value
+             *            the new user dn
+             *
+             * @return builder to set the user dn to a new value
              */
             public Builder setUserDN(java.lang.String value) {
                 if (value == null) {
@@ -2034,6 +2300,8 @@ public Builder setUserDN(java.lang.String value) {
 
             /**
              * required string user_DN = 4;
+             *
+             * @return builder to clear the user dn
              */
             public Builder clearUserDN() {
                 bitField0_ = (bitField0_ & ~0x00000008);
@@ -2044,6 +2312,11 @@ public Builder clearUserDN() {
 
             /**
              * required string user_DN = 4;
+             *
+             * @param value
+             *            user dn byte string
+             *
+             * @return builder ot set the user dn to the new byte string
              */
             public Builder setUserDNBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -2060,6 +2333,8 @@ public Builder setUserDNBytes(com.google.protobuf.ByteString value) {
 
             /**
              * required string query = 5;
+             *
+             * @return if it has query
              */
             public boolean hasQuery() {
                 return ((bitField0_ & 0x00000010) == 0x00000010);
@@ -2067,6 +2342,8 @@ public boolean hasQuery() {
 
             /**
              * required string query = 5;
+             *
+             * @return the query
              */
             public java.lang.String getQuery() {
                 java.lang.Object ref = query_;
@@ -2081,6 +2358,8 @@ public java.lang.String getQuery() {
 
             /**
              * required string query = 5;
+             *
+             * @return the query byte string
              */
             public com.google.protobuf.ByteString getQueryBytes() {
                 java.lang.Object ref = query_;
@@ -2095,6 +2374,11 @@ public com.google.protobuf.ByteString getQueryBytes() {
 
             /**
              * required string query = 5;
+             *
+             * @param value
+             *            the new query value
+             *
+             * @return builder to set the query to a new value
              */
             public Builder setQuery(java.lang.String value) {
                 if (value == null) {
@@ -2108,6 +2392,8 @@ public Builder setQuery(java.lang.String value) {
 
             /**
              * required string query = 5;
+             *
+             * @return builder to clear the query
              */
             public Builder clearQuery() {
                 bitField0_ = (bitField0_ & ~0x00000010);
@@ -2118,6 +2404,11 @@ public Builder clearQuery() {
 
             /**
              * required string query = 5;
+             *
+             * @param value
+             *            byte string for the new query value
+             *
+             * @return builder to set the query to a new value
              */
             public Builder setQueryBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -2134,6 +2425,8 @@ public Builder setQueryBytes(com.google.protobuf.ByteString value) {
 
             /**
              * required string query_visibility = 6;
+             *
+             * @return if it has query visibility
              */
             public boolean hasQueryVisibility() {
                 return ((bitField0_ & 0x00000020) == 0x00000020);
@@ -2141,6 +2434,8 @@ public boolean hasQueryVisibility() {
 
             /**
              * required string query_visibility = 6;
+             *
+             * @return the query visibility
              */
             public java.lang.String getQueryVisibility() {
                 java.lang.Object ref = queryVisibility_;
@@ -2155,6 +2450,8 @@ public java.lang.String getQueryVisibility() {
 
             /**
              * required string query_visibility = 6;
+             *
+             * @return query visibility byte string
              */
             public com.google.protobuf.ByteString getQueryVisibilityBytes() {
                 java.lang.Object ref = queryVisibility_;
@@ -2169,6 +2466,11 @@ public com.google.protobuf.ByteString getQueryVisibilityBytes() {
 
             /**
              * required string query_visibility = 6;
+             *
+             * @param value
+             *            the new query visibility
+             *
+             * @return builder to set query visibility
              */
             public Builder setQueryVisibility(java.lang.String value) {
                 if (value == null) {
@@ -2182,6 +2484,8 @@ public Builder setQueryVisibility(java.lang.String value) {
 
             /**
              * required string query_visibility = 6;
+             *
+             * @return builder to clear query visibility
              */
             public Builder clearQueryVisibility() {
                 bitField0_ = (bitField0_ & ~0x00000020);
@@ -2192,6 +2496,11 @@ public Builder clearQueryVisibility() {
 
             /**
              * required string query_visibility = 6;
+             *
+             * @param value
+             *            query visibility byte string
+             *
+             * @return builder to set query visibility to new byte string
              */
             public Builder setQueryVisibilityBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -2208,6 +2517,8 @@ public Builder setQueryVisibilityBytes(com.google.protobuf.ByteString value) {
 
             /**
              * optional int64 begin_date = 7;
+             *
+             * @return if it has begin date
              */
             public boolean hasBeginDate() {
                 return ((bitField0_ & 0x00000040) == 0x00000040);
@@ -2215,6 +2526,8 @@ public boolean hasBeginDate() {
 
             /**
              * optional int64 begin_date = 7;
+             *
+             * @return begin date
              */
             public long getBeginDate() {
                 return beginDate_;
@@ -2222,6 +2535,11 @@ public long getBeginDate() {
 
             /**
              * optional int64 begin_date = 7;
+             *
+             * @param value
+             *            the new begin date
+             *
+             * @return builder to set the begin date to a new value
              */
             public Builder setBeginDate(long value) {
                 bitField0_ |= 0x00000040;
@@ -2232,6 +2550,8 @@ public Builder setBeginDate(long value) {
 
             /**
              * optional int64 begin_date = 7;
+             *
+             * @return builder to clear the begin date
              */
             public Builder clearBeginDate() {
                 bitField0_ = (bitField0_ & ~0x00000040);
@@ -2245,6 +2565,8 @@ public Builder clearBeginDate() {
 
             /**
              * optional int64 end_date = 8;
+             *
+             * @return if it has end date
              */
             public boolean hasEndDate() {
                 return ((bitField0_ & 0x00000080) == 0x00000080);
@@ -2252,6 +2574,8 @@ public boolean hasEndDate() {
 
             /**
              * optional int64 end_date = 8;
+             *
+             * @return the end date
              */
             public long getEndDate() {
                 return endDate_;
@@ -2259,6 +2583,11 @@ public long getEndDate() {
 
             /**
              * optional int64 end_date = 8;
+             *
+             * @param value
+             *            the new end date
+             *
+             * @return builder to set end date to new value
              */
             public Builder setEndDate(long value) {
                 bitField0_ |= 0x00000080;
@@ -2269,6 +2598,8 @@ public Builder setEndDate(long value) {
 
             /**
              * optional int64 end_date = 8;
+             *
+             * @return builder to clear the end date
              */
             public Builder clearEndDate() {
                 bitField0_ = (bitField0_ & ~0x00000080);
@@ -2282,6 +2613,8 @@ public Builder clearEndDate() {
 
             /**
              * required string query_authorizations = 9;
+             *
+             * @return if it has query authorizations
              */
             public boolean hasQueryAuthorizations() {
                 return ((bitField0_ & 0x00000100) == 0x00000100);
@@ -2289,6 +2622,8 @@ public boolean hasQueryAuthorizations() {
 
             /**
              * required string query_authorizations = 9;
+             *
+             * @return query authorizations
              */
             public java.lang.String getQueryAuthorizations() {
                 java.lang.Object ref = queryAuthorizations_;
@@ -2303,6 +2638,8 @@ public java.lang.String getQueryAuthorizations() {
 
             /**
              * required string query_authorizations = 9;
+             *
+             * @return query authorizations byte string
              */
             public com.google.protobuf.ByteString getQueryAuthorizationsBytes() {
                 java.lang.Object ref = queryAuthorizations_;
@@ -2317,6 +2654,11 @@ public com.google.protobuf.ByteString getQueryAuthorizationsBytes() {
 
             /**
              * required string query_authorizations = 9;
+             *
+             * @param value
+             *            the new query authorizations
+             *
+             * @return builder to set query authorizations to provided value
              */
             public Builder setQueryAuthorizations(java.lang.String value) {
                 if (value == null) {
@@ -2330,6 +2672,8 @@ public Builder setQueryAuthorizations(java.lang.String value) {
 
             /**
              * required string query_authorizations = 9;
+             *
+             * @return builder to clear query authorizations
              */
             public Builder clearQueryAuthorizations() {
                 bitField0_ = (bitField0_ & ~0x00000100);
@@ -2340,6 +2684,11 @@ public Builder clearQueryAuthorizations() {
 
             /**
              * required string query_authorizations = 9;
+             *
+             * @param value
+             *            query authorization byte string
+             *
+             * @return builder to set query authorization to new byte string
              */
             public Builder setQueryAuthorizationsBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -2356,6 +2705,8 @@ public Builder setQueryAuthorizationsBytes(com.google.protobuf.ByteString value)
 
             /**
              * required int64 expiration_date = 10;
+             *
+             * @return if it has expiration date
              */
             public boolean hasExpirationDate() {
                 return ((bitField0_ & 0x00000200) == 0x00000200);
@@ -2363,6 +2714,8 @@ public boolean hasExpirationDate() {
 
             /**
              * required int64 expiration_date = 10;
+             *
+             * @return the expiration date
              */
             public long getExpirationDate() {
                 return expirationDate_;
@@ -2370,6 +2723,11 @@ public long getExpirationDate() {
 
             /**
              * required int64 expiration_date = 10;
+             *
+             * @param value
+             *            the expiration date
+             *
+             * @return builder to set expiration date to new value
              */
             public Builder setExpirationDate(long value) {
                 bitField0_ |= 0x00000200;
@@ -2380,6 +2738,8 @@ public Builder setExpirationDate(long value) {
 
             /**
              * required int64 expiration_date = 10;
+             *
+             * @return builder to clear expiration date
              */
             public Builder clearExpirationDate() {
                 bitField0_ = (bitField0_ & ~0x00000200);
@@ -2393,6 +2753,8 @@ public Builder clearExpirationDate() {
 
             /**
              * required uint32 pagesize = 11 [default = 0];
+             *
+             * @return if it has page size
              */
             public boolean hasPagesize() {
                 return ((bitField0_ & 0x00000400) == 0x00000400);
@@ -2400,6 +2762,8 @@ public boolean hasPagesize() {
 
             /**
              * required uint32 pagesize = 11 [default = 0];
+             *
+             * @return page size
              */
             public int getPagesize() {
                 return pagesize_;
@@ -2407,6 +2771,11 @@ public int getPagesize() {
 
             /**
              * required uint32 pagesize = 11 [default = 0];
+             *
+             * @param value
+             *            new page size
+             *
+             * @return builder to set page size to new value
              */
             public Builder setPagesize(int value) {
                 bitField0_ |= 0x00000400;
@@ -2417,6 +2786,8 @@ public Builder setPagesize(int value) {
 
             /**
              * required uint32 pagesize = 11 [default = 0];
+             *
+             * @return builder to clear page size
              */
             public Builder clearPagesize() {
                 bitField0_ = (bitField0_ & ~0x00000400);
@@ -2437,6 +2808,8 @@ private void ensureParametersIsMutable() {
 
             /**
              * repeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
+             *
+             * @return list of parameters
              */
             public java.util.List getParametersList() {
                 return java.util.Collections.unmodifiableList(parameters_);
@@ -2444,6 +2817,8 @@ public java.util.Listrepeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
+             *
+             * @return the number of parameters in the list
              */
             public int getParametersCount() {
                 return parameters_.size();
@@ -2451,6 +2826,11 @@ public int getParametersCount() {
 
             /**
              * repeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
+             *
+             * @param index
+             *            the index
+             *
+             * @return the parameter at the provided index
              */
             public datawave.webservice.query.QueryMessages.QueryImpl.Parameter getParameters(int index) {
                 return parameters_.get(index);
@@ -2458,6 +2838,13 @@ public datawave.webservice.query.QueryMessages.QueryImpl.Parameter getParameters
 
             /**
              * repeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the new value to be set at the provided index
+             *
+             * @return builder to set parameter at provided index
              */
             public Builder setParameters(int index, datawave.webservice.query.QueryMessages.QueryImpl.Parameter value) {
                 if (value == null) {
@@ -2471,6 +2858,13 @@ public Builder setParameters(int index, datawave.webservice.query.QueryMessages.
 
             /**
              * repeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
+             *
+             * @param index
+             *            the index
+             * @param builderForValue
+             *            paramater value to be set at provided index
+             *
+             * @return builder to set parameter at provided index
              */
             public Builder setParameters(int index, datawave.webservice.query.QueryMessages.QueryImpl.Parameter.Builder builderForValue) {
                 ensureParametersIsMutable();
@@ -2481,6 +2875,11 @@ public Builder setParameters(int index, datawave.webservice.query.QueryMessages.
 
             /**
              * repeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
+             *
+             * @param value
+             *            the new parameter to add to the list
+             *
+             * @return builder to add parameter
              */
             public Builder addParameters(datawave.webservice.query.QueryMessages.QueryImpl.Parameter value) {
                 if (value == null) {
@@ -2494,6 +2893,13 @@ public Builder addParameters(datawave.webservice.query.QueryMessages.QueryImpl.P
 
             /**
              * repeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the new parameter to add at the index provided
+             *
+             * @return builder to add parameter
              */
             public Builder addParameters(int index, datawave.webservice.query.QueryMessages.QueryImpl.Parameter value) {
                 if (value == null) {
@@ -2507,6 +2913,11 @@ public Builder addParameters(int index, datawave.webservice.query.QueryMessages.
 
             /**
              * repeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
+             *
+             * @param builderForValue
+             *            parameter to add to the list of parameters
+             *
+             * @return builder to add parameters
              */
             public Builder addParameters(datawave.webservice.query.QueryMessages.QueryImpl.Parameter.Builder builderForValue) {
                 ensureParametersIsMutable();
@@ -2517,6 +2928,13 @@ public Builder addParameters(datawave.webservice.query.QueryMessages.QueryImpl.P
 
             /**
              * repeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
+             *
+             * @param index
+             *            the index
+             * @param builderForValue
+             *            builder to creat the value to add at the provided index
+             *
+             * @return builder to add parameter at provided index
              */
             public Builder addParameters(int index, datawave.webservice.query.QueryMessages.QueryImpl.Parameter.Builder builderForValue) {
                 ensureParametersIsMutable();
@@ -2527,6 +2945,11 @@ public Builder addParameters(int index, datawave.webservice.query.QueryMessages.
 
             /**
              * repeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
+             *
+             * @param values
+             *            iterator of values to add to the parameter list
+             *
+             * @return builder to add parameters
              */
             public Builder addAllParameters(java.lang.Iterable values) {
                 ensureParametersIsMutable();
@@ -2537,6 +2960,8 @@ public Builder addAllParameters(java.lang.Iterablerepeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
+             *
+             * @return builder to clear the parameters list
              */
             public Builder clearParameters() {
                 parameters_ = java.util.Collections.emptyList();
@@ -2547,6 +2972,11 @@ public Builder clearParameters() {
 
             /**
              * repeated .datawave.webservice.query.QueryImpl.Parameter parameters = 12;
+             *
+             * @param index
+             *            the index
+             *
+             * @return builder to remove parameters at the provided index
              */
             public Builder removeParameters(int index) {
                 ensureParametersIsMutable();
diff --git a/web-services/client/src/main/java/datawave/webservice/query/QueryParametersImpl.java b/web-services/client/src/main/java/datawave/webservice/query/QueryParametersImpl.java
index ea08f4cc381..3895aa81e0d 100644
--- a/web-services/client/src/main/java/datawave/webservice/query/QueryParametersImpl.java
+++ b/web-services/client/src/main/java/datawave/webservice/query/QueryParametersImpl.java
@@ -68,6 +68,7 @@ public QueryParametersImpl() {
      * @param parameters
      *            - a Map of QueryParameters
      * @throws IllegalArgumentException
+     *             when a bad argument is encountered
      */
     public void validate(Map> parameters) throws IllegalArgumentException {
         for (String param : KNOWN_PARAMS) {
@@ -281,7 +282,7 @@ public static synchronized Date parseDate(String s, String defaultTime, String d
      * @param queryName
      *            - client-supplied name of query
      * @param queryVisibility
-     *            -
+     *            - query
      * @param beginDate
      *            - start date
      * @param endDate
@@ -289,18 +290,22 @@ public static synchronized Date parseDate(String s, String defaultTime, String d
      * @param queryAuthorizations
      *            - what auths the query should run with
      * @param expirationDate
-     *            -
+     *            - expiration date
      * @param pagesize
-     *            -
+     *            - page size
      * @param pageTimeout
+     *            - page timeout
      * @param maxResultsOverride
+     *            - max results override
      * @param persistenceMode
-     *            -
+     *            - persistence mode
+     * @param systemFrom
+     *            - system from
      * @param parameters
      *            - additional parameters passed in as map
      * @param trace
-     *            -
-     * @return
+     *            - trace flag
+     * @return parameter map
      * @throws ParseException
      *             on date parse/format error
      */
diff --git a/web-services/client/src/main/java/datawave/webservice/query/exception/ExceptionMessages.java b/web-services/client/src/main/java/datawave/webservice/query/exception/ExceptionMessages.java
index 93e1639123a..9f5d0a788b7 100644
--- a/web-services/client/src/main/java/datawave/webservice/query/exception/ExceptionMessages.java
+++ b/web-services/client/src/main/java/datawave/webservice/query/exception/ExceptionMessages.java
@@ -13,32 +13,44 @@ public interface QueryExceptionTypeOrBuilder extends com.google.protobuf.Message
         // optional string message = 1;
         /**
          * optional string message = 1;
+         *
+         * @return if it has message
          */
         boolean hasMessage();
 
         /**
          * optional string message = 1;
+         *
+         * @return the message
          */
         java.lang.String getMessage();
 
         /**
          * optional string message = 1;
+         *
+         * @return the message byte string
          */
         com.google.protobuf.ByteString getMessageBytes();
 
         // optional string cause = 2;
         /**
          * optional string cause = 2;
+         *
+         * @return if it has cause
          */
         boolean hasCause();
 
         /**
          * optional string cause = 2;
+         *
+         * @return the cause
          */
         java.lang.String getCause();
 
         /**
          * optional string cause = 2;
+         *
+         * @return the cause byte string
          */
         com.google.protobuf.ByteString getCauseBytes();
     }
@@ -123,6 +135,8 @@ public com.google.protobuf.Parser getParserForType() {
 
         /**
          * optional string message = 1;
+         *
+         * @return if it has a message
          */
         public boolean hasMessage() {
             return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -130,6 +144,8 @@ public boolean hasMessage() {
 
         /**
          * optional string message = 1;
+         *
+         * @return the message
          */
         public java.lang.String getMessage() {
             java.lang.Object ref = message_;
@@ -147,6 +163,8 @@ public java.lang.String getMessage() {
 
         /**
          * optional string message = 1;
+         *
+         * @return the message byte string
          */
         public com.google.protobuf.ByteString getMessageBytes() {
             java.lang.Object ref = message_;
@@ -165,6 +183,8 @@ public com.google.protobuf.ByteString getMessageBytes() {
 
         /**
          * optional string cause = 2;
+         *
+         * @return if it has a cause
          */
         public boolean hasCause() {
             return ((bitField0_ & 0x00000002) == 0x00000002);
@@ -172,6 +192,8 @@ public boolean hasCause() {
 
         /**
          * optional string cause = 2;
+         *
+         * @return the cause
          */
         public java.lang.String getCause() {
             java.lang.Object ref = cause_;
@@ -189,6 +211,8 @@ public java.lang.String getCause() {
 
         /**
          * optional string cause = 2;
+         *
+         * @return the cause byte string
          */
         public com.google.protobuf.ByteString getCauseBytes() {
             java.lang.Object ref = cause_;
@@ -419,6 +443,8 @@ public Builder mergeFrom(com.google.protobuf.CodedInputStream input, com.google.
 
             /**
              * optional string message = 1;
+             *
+             * @return if it has a message
              */
             public boolean hasMessage() {
                 return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -426,6 +452,8 @@ public boolean hasMessage() {
 
             /**
              * optional string message = 1;
+             *
+             * @return the message
              */
             public java.lang.String getMessage() {
                 java.lang.Object ref = message_;
@@ -440,6 +468,8 @@ public java.lang.String getMessage() {
 
             /**
              * optional string message = 1;
+             *
+             * @return the message byte string
              */
             public com.google.protobuf.ByteString getMessageBytes() {
                 java.lang.Object ref = message_;
@@ -454,6 +484,11 @@ public com.google.protobuf.ByteString getMessageBytes() {
 
             /**
              * optional string message = 1;
+             *
+             * @param value
+             *            the message
+             *
+             * @return builder to set the message
              */
             public Builder setMessage(java.lang.String value) {
                 if (value == null) {
@@ -467,6 +502,8 @@ public Builder setMessage(java.lang.String value) {
 
             /**
              * optional string message = 1;
+             *
+             * @return builder to clear the message
              */
             public Builder clearMessage() {
                 bitField0_ = (bitField0_ & ~0x00000001);
@@ -477,6 +514,11 @@ public Builder clearMessage() {
 
             /**
              * optional string message = 1;
+             *
+             * @param value
+             *            the value byte string
+             *
+             * @return builder to set the message to provided byte string
              */
             public Builder setMessageBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -493,6 +535,8 @@ public Builder setMessageBytes(com.google.protobuf.ByteString value) {
 
             /**
              * optional string cause = 2;
+             *
+             * @return if it has a cause
              */
             public boolean hasCause() {
                 return ((bitField0_ & 0x00000002) == 0x00000002);
@@ -500,6 +544,8 @@ public boolean hasCause() {
 
             /**
              * optional string cause = 2;
+             *
+             * @return the cause
              */
             public java.lang.String getCause() {
                 java.lang.Object ref = cause_;
@@ -514,6 +560,8 @@ public java.lang.String getCause() {
 
             /**
              * optional string cause = 2;
+             *
+             * @return the cause byte string
              */
             public com.google.protobuf.ByteString getCauseBytes() {
                 java.lang.Object ref = cause_;
@@ -528,6 +576,11 @@ public com.google.protobuf.ByteString getCauseBytes() {
 
             /**
              * optional string cause = 2;
+             *
+             * @param value
+             *            the cause
+             *
+             * @return builder to set the cause to the provided value
              */
             public Builder setCause(java.lang.String value) {
                 if (value == null) {
@@ -541,6 +594,8 @@ public Builder setCause(java.lang.String value) {
 
             /**
              * optional string cause = 2;
+             *
+             * @return builder to clear the cause
              */
             public Builder clearCause() {
                 bitField0_ = (bitField0_ & ~0x00000002);
@@ -551,6 +606,11 @@ public Builder clearCause() {
 
             /**
              * optional string cause = 2;
+             *
+             * @param value
+             *            new cause value as byte string
+             *
+             * @return builder to set cause to byte string
              */
             public Builder setCauseBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/EventBase.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/EventBase.java
index 00bb63dcb87..5314c8f1b0a 100644
--- a/web-services/client/src/main/java/datawave/webservice/query/result/event/EventBase.java
+++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/EventBase.java
@@ -40,6 +40,8 @@ public abstract class EventBase> implements HasMarkings
     /**
      * Get the approximate size of this event in bytes. Used by the ObjectSizeOf mechanism in the webservice. Throws an exception if the local size was not set
      * to allow the ObjectSizeOf mechanism to do its thang.
+     *
+     * @return the size in bytes
      */
     public abstract long sizeInBytes();
 
diff --git a/web-services/client/src/main/java/datawave/webservice/query/result/event/ResponseObjectFactory.java b/web-services/client/src/main/java/datawave/webservice/query/result/event/ResponseObjectFactory.java
index f70105354fb..6facccda7de 100644
--- a/web-services/client/src/main/java/datawave/webservice/query/result/event/ResponseObjectFactory.java
+++ b/web-services/client/src/main/java/datawave/webservice/query/result/event/ResponseObjectFactory.java
@@ -38,7 +38,7 @@ public abstract class ResponseObjectFactory {
      * provided here, then a javax.ws.rs.ext.Provider must be created which implements {@code ContextResolver}. Therein a resolver for a
      * LookupResponse needs to include the provided implementation within a jaxb context to ensure appropriate serialization.
      *
-     * @return
+     * @return the keybase
      */
     public abstract KeyBase getKey();
 
diff --git a/web-services/client/src/main/java/datawave/webservice/result/ResponseMessages.java b/web-services/client/src/main/java/datawave/webservice/result/ResponseMessages.java
index 00c2a2ba0fe..9036b931e6c 100644
--- a/web-services/client/src/main/java/datawave/webservice/result/ResponseMessages.java
+++ b/web-services/client/src/main/java/datawave/webservice/result/ResponseMessages.java
@@ -13,48 +13,75 @@ public interface VoidResponseOrBuilder extends com.google.protobuf.MessageLiteOr
         // required uint64 operation_time_ms = 1 [default = 0];
         /**
          * required uint64 operation_time_ms = 1 [default = 0];
+         *
+         * @return if it has operation time in ms
          */
         boolean hasOperationTimeMs();
 
         /**
          * required uint64 operation_time_ms = 1 [default = 0];
+         *
+         * @return the operation time in ms
          */
         long getOperationTimeMs();
 
         // repeated string messages = 2;
         /**
          * repeated string messages = 2;
+         *
+         * @return list of messages
          */
         java.util.List getMessagesList();
 
         /**
          * repeated string messages = 2;
+         *
+         * @return the number of messages in the list
          */
         int getMessagesCount();
 
         /**
          * repeated string messages = 2;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the message at the provided index
          */
         java.lang.String getMessages(int index);
 
         /**
          * repeated string messages = 2;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the message byte string at the provided index
          */
         com.google.protobuf.ByteString getMessagesBytes(int index);
 
         // repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
         /**
          * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @return list of exceptions
          */
         java.util.List getExceptionsList();
 
         /**
          * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the exception at the provided index
          */
         datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType getExceptions(int index);
 
         /**
          * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @return the number of exceptions
          */
         int getExceptionsCount();
     }
@@ -157,6 +184,8 @@ public com.google.protobuf.Parser getParserForType() {
 
         /**
          * required uint64 operation_time_ms = 1 [default = 0];
+         *
+         * @return if it has an operation time
          */
         public boolean hasOperationTimeMs() {
             return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -164,6 +193,8 @@ public boolean hasOperationTimeMs() {
 
         /**
          * required uint64 operation_time_ms = 1 [default = 0];
+         *
+         * @return the operation time in ms
          */
         public long getOperationTimeMs() {
             return operationTimeMs_;
@@ -175,6 +206,8 @@ public long getOperationTimeMs() {
 
         /**
          * repeated string messages = 2;
+         *
+         * @return the list of messages
          */
         public java.util.List getMessagesList() {
             return messages_;
@@ -182,6 +215,8 @@ public java.util.List getMessagesList() {
 
         /**
          * repeated string messages = 2;
+         *
+         * @return the number of messages
          */
         public int getMessagesCount() {
             return messages_.size();
@@ -189,6 +224,11 @@ public int getMessagesCount() {
 
         /**
          * repeated string messages = 2;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the message at the index provided
          */
         public java.lang.String getMessages(int index) {
             return messages_.get(index);
@@ -196,6 +236,11 @@ public java.lang.String getMessages(int index) {
 
         /**
          * repeated string messages = 2;
+         *
+         * @param index
+         *            the index
+         *
+         * @return message byte string at the provided index
          */
         public com.google.protobuf.ByteString getMessagesBytes(int index) {
             return messages_.getByteString(index);
@@ -207,6 +252,8 @@ public com.google.protobuf.ByteString getMessagesBytes(int index) {
 
         /**
          * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @return list of exceptions
          */
         public java.util.List getExceptionsList() {
             return exceptions_;
@@ -214,6 +261,8 @@ public java.util.Listrepeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @return list of exceptions
          */
         public java.util.List getExceptionsOrBuilderList() {
             return exceptions_;
@@ -221,6 +270,8 @@ public java.util.Listrepeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @return the number of exceptions in the list
          */
         public int getExceptionsCount() {
             return exceptions_.size();
@@ -228,6 +279,11 @@ public int getExceptionsCount() {
 
         /**
          * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the exception at the index
          */
         public datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType getExceptions(int index) {
             return exceptions_.get(index);
@@ -235,6 +291,11 @@ public datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType
 
         /**
          * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the exception at the index
          */
         public datawave.webservice.query.exception.ExceptionMessages.QueryExceptionTypeOrBuilder getExceptionsOrBuilder(int index) {
             return exceptions_.get(index);
@@ -497,6 +558,8 @@ public Builder mergeFrom(com.google.protobuf.CodedInputStream input, com.google.
 
             /**
              * required uint64 operation_time_ms = 1 [default = 0];
+             *
+             * @return if it has operation time in ms
              */
             public boolean hasOperationTimeMs() {
                 return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -504,6 +567,8 @@ public boolean hasOperationTimeMs() {
 
             /**
              * required uint64 operation_time_ms = 1 [default = 0];
+             *
+             * @return operation time in ms
              */
             public long getOperationTimeMs() {
                 return operationTimeMs_;
@@ -511,6 +576,11 @@ public long getOperationTimeMs() {
 
             /**
              * required uint64 operation_time_ms = 1 [default = 0];
+             *
+             * @param value
+             *            the new operation time in ms
+             *
+             * @return builder to set operation time in ms
              */
             public Builder setOperationTimeMs(long value) {
                 bitField0_ |= 0x00000001;
@@ -521,6 +591,8 @@ public Builder setOperationTimeMs(long value) {
 
             /**
              * required uint64 operation_time_ms = 1 [default = 0];
+             *
+             * @return builder to clear operation time
              */
             public Builder clearOperationTimeMs() {
                 bitField0_ = (bitField0_ & ~0x00000001);
@@ -541,6 +613,8 @@ private void ensureMessagesIsMutable() {
 
             /**
              * repeated string messages = 2;
+             *
+             * @return the message list
              */
             public java.util.List getMessagesList() {
                 return java.util.Collections.unmodifiableList(messages_);
@@ -548,6 +622,8 @@ public java.util.List getMessagesList() {
 
             /**
              * repeated string messages = 2;
+             *
+             * @return the number of messages in the list
              */
             public int getMessagesCount() {
                 return messages_.size();
@@ -555,6 +631,11 @@ public int getMessagesCount() {
 
             /**
              * repeated string messages = 2;
+             *
+             * @param index
+             *            the index
+             *
+             * @return the message at the provided index
              */
             public java.lang.String getMessages(int index) {
                 return messages_.get(index);
@@ -562,6 +643,11 @@ public java.lang.String getMessages(int index) {
 
             /**
              * repeated string messages = 2;
+             *
+             * @param index
+             *            the index
+             *
+             * @return the message byte string at the provided index
              */
             public com.google.protobuf.ByteString getMessagesBytes(int index) {
                 return messages_.getByteString(index);
@@ -569,6 +655,13 @@ public com.google.protobuf.ByteString getMessagesBytes(int index) {
 
             /**
              * repeated string messages = 2;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the new messagew
+             *
+             * @return builder to set the message value at the provided index
              */
             public Builder setMessages(int index, java.lang.String value) {
                 if (value == null) {
@@ -582,6 +675,11 @@ public Builder setMessages(int index, java.lang.String value) {
 
             /**
              * repeated string messages = 2;
+             *
+             * @param value
+             *            a message to add to the lsit
+             *
+             * @return builder to add the provided message
              */
             public Builder addMessages(java.lang.String value) {
                 if (value == null) {
@@ -595,6 +693,11 @@ public Builder addMessages(java.lang.String value) {
 
             /**
              * repeated string messages = 2;
+             *
+             * @param values
+             *            iterable of messages to add
+             *
+             * @return builder to add all messages provided
              */
             public Builder addAllMessages(java.lang.Iterable values) {
                 ensureMessagesIsMutable();
@@ -605,6 +708,8 @@ public Builder addAllMessages(java.lang.Iterable values) {
 
             /**
              * repeated string messages = 2;
+             *
+             * @return builder to clear the message list
              */
             public Builder clearMessages() {
                 messages_ = com.google.protobuf.LazyStringArrayList.EMPTY;
@@ -615,6 +720,11 @@ public Builder clearMessages() {
 
             /**
              * repeated string messages = 2;
+             *
+             * @param value
+             *            new message byte string
+             *
+             * @return builder to add the provided message as byte string
              */
             public Builder addMessagesBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -638,6 +748,8 @@ private void ensureExceptionsIsMutable() {
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @return list of exceptions
              */
             public java.util.List getExceptionsList() {
                 return java.util.Collections.unmodifiableList(exceptions_);
@@ -645,6 +757,8 @@ public java.util.Listrepeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @return the number of exceptions in the list
              */
             public int getExceptionsCount() {
                 return exceptions_.size();
@@ -652,6 +766,11 @@ public int getExceptionsCount() {
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param index
+             *            the index
+             *
+             * @return the exception at the provided index
              */
             public datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType getExceptions(int index) {
                 return exceptions_.get(index);
@@ -659,6 +778,13 @@ public datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the new exception type
+             *
+             * @return builder to set exception at provided index to the value
              */
             public Builder setExceptions(int index, datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType value) {
                 if (value == null) {
@@ -672,6 +798,13 @@ public Builder setExceptions(int index, datawave.webservice.query.exception.Exce
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param index
+             *            the index
+             * @param builderForValue
+             *            query exception type builder
+             *
+             * @return builder to set the exception at the index to the provided value
              */
             public Builder setExceptions(int index, datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType.Builder builderForValue) {
                 ensureExceptionsIsMutable();
@@ -682,6 +815,11 @@ public Builder setExceptions(int index, datawave.webservice.query.exception.Exce
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param value
+             *            the exception type to add
+             *
+             * @return builder to add the provided value
              */
             public Builder addExceptions(datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType value) {
                 if (value == null) {
@@ -695,6 +833,13 @@ public Builder addExceptions(datawave.webservice.query.exception.ExceptionMessag
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the exception to add
+             *
+             * @return builder to add the provided exception to the provided index
              */
             public Builder addExceptions(int index, datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType value) {
                 if (value == null) {
@@ -708,6 +853,11 @@ public Builder addExceptions(int index, datawave.webservice.query.exception.Exce
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param builderForValue
+             *            the builder with new value
+             *
+             * @return builder to add provided exception
              */
             public Builder addExceptions(datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType.Builder builderForValue) {
                 ensureExceptionsIsMutable();
@@ -718,6 +868,13 @@ public Builder addExceptions(datawave.webservice.query.exception.ExceptionMessag
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param index
+             *            the index
+             * @param builderForValue
+             *            the builder to add a value
+             *
+             * @return builder to add exception at provided index
              */
             public Builder addExceptions(int index, datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType.Builder builderForValue) {
                 ensureExceptionsIsMutable();
@@ -728,6 +885,11 @@ public Builder addExceptions(int index, datawave.webservice.query.exception.Exce
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param values
+             *            iterable of values to add to exceptions list
+             *
+             * @return builder to add provided values
              */
             public Builder addAllExceptions(java.lang.Iterable values) {
                 ensureExceptionsIsMutable();
@@ -738,6 +900,8 @@ public Builder addAllExceptions(java.lang.Iterablerepeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @return builder to clear exceptions
              */
             public Builder clearExceptions() {
                 exceptions_ = java.util.Collections.emptyList();
@@ -748,6 +912,11 @@ public Builder clearExceptions() {
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param index
+             *            the index
+             *
+             * @return builder to remove exceptions at the provided index
              */
             public Builder removeExceptions(int index) {
                 ensureExceptionsIsMutable();
@@ -772,75 +941,115 @@ public interface QueryImplListResponseOrBuilder extends com.google.protobuf.Mess
         // required uint64 operation_time_ms = 1 [default = 0];
         /**
          * required uint64 operation_time_ms = 1 [default = 0];
+         *
+         * @return if it has operation time
          */
         boolean hasOperationTimeMs();
 
         /**
          * required uint64 operation_time_ms = 1 [default = 0];
+         *
+         * @return the operation time in ms
          */
         long getOperationTimeMs();
 
         // repeated string messages = 2;
         /**
          * repeated string messages = 2;
+         *
+         * @return the messages list
          */
         java.util.List getMessagesList();
 
         /**
          * repeated string messages = 2;
+         *
+         * @return the number of messages in the list
          */
         int getMessagesCount();
 
         /**
          * repeated string messages = 2;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the message at the provided index
          */
         java.lang.String getMessages(int index);
 
         /**
          * repeated string messages = 2;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the message byte string at the provided index
          */
         com.google.protobuf.ByteString getMessagesBytes(int index);
 
         // repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
         /**
          * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @return list of exceptions
          */
         java.util.List getExceptionsList();
 
         /**
          * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the exception at the provided index
          */
         datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType getExceptions(int index);
 
         /**
          * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @return the number of exceptions in the list
          */
         int getExceptionsCount();
 
         // repeated .datawave.webservice.query.QueryImpl query = 4;
         /**
          * repeated .datawave.webservice.query.QueryImpl query = 4;
+         *
+         * @return the query list
          */
         java.util.List getQueryList();
 
         /**
          * repeated .datawave.webservice.query.QueryImpl query = 4;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the query message at the provided index
          */
         datawave.webservice.query.QueryMessages.QueryImpl getQuery(int index);
 
         /**
          * repeated .datawave.webservice.query.QueryImpl query = 4;
+         *
+         * @return the number of queries in the list
          */
         int getQueryCount();
 
         // optional uint32 numResults = 5 [default = 0];
         /**
          * optional uint32 numResults = 5 [default = 0];
+         *
+         * @return if it has a number of results
          */
         boolean hasNumResults();
 
         /**
          * optional uint32 numResults = 5 [default = 0];
+         *
+         * @return the number of results
          */
         int getNumResults();
     }
@@ -959,6 +1168,8 @@ public com.google.protobuf.Parser getParserForType() {
 
         /**
          * required uint64 operation_time_ms = 1 [default = 0];
+         *
+         * @return if it has operation time
          */
         public boolean hasOperationTimeMs() {
             return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -966,6 +1177,8 @@ public boolean hasOperationTimeMs() {
 
         /**
          * required uint64 operation_time_ms = 1 [default = 0];
+         *
+         * @return the operation time in ms
          */
         public long getOperationTimeMs() {
             return operationTimeMs_;
@@ -977,6 +1190,8 @@ public long getOperationTimeMs() {
 
         /**
          * repeated string messages = 2;
+         *
+         * @return the messages list
          */
         public java.util.List getMessagesList() {
             return messages_;
@@ -984,6 +1199,8 @@ public java.util.List getMessagesList() {
 
         /**
          * repeated string messages = 2;
+         *
+         * @return the number of messages in the list
          */
         public int getMessagesCount() {
             return messages_.size();
@@ -991,6 +1208,11 @@ public int getMessagesCount() {
 
         /**
          * repeated string messages = 2;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the message at the provided index
          */
         public java.lang.String getMessages(int index) {
             return messages_.get(index);
@@ -998,6 +1220,11 @@ public java.lang.String getMessages(int index) {
 
         /**
          * repeated string messages = 2;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the message byte string at the provided index
          */
         public com.google.protobuf.ByteString getMessagesBytes(int index) {
             return messages_.getByteString(index);
@@ -1009,6 +1236,8 @@ public com.google.protobuf.ByteString getMessagesBytes(int index) {
 
         /**
          * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @return the exception list
          */
         public java.util.List getExceptionsList() {
             return exceptions_;
@@ -1016,6 +1245,8 @@ public java.util.Listrepeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @return the exception list
          */
         public java.util.List getExceptionsOrBuilderList() {
             return exceptions_;
@@ -1023,6 +1254,8 @@ public java.util.Listrepeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @return the number of exceptions in the list
          */
         public int getExceptionsCount() {
             return exceptions_.size();
@@ -1030,6 +1263,11 @@ public int getExceptionsCount() {
 
         /**
          * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the exception at the provided index
          */
         public datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType getExceptions(int index) {
             return exceptions_.get(index);
@@ -1037,6 +1275,11 @@ public datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType
 
         /**
          * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @param index
+         *            the index
+         *
+         * @return exception at the provided index
          */
         public datawave.webservice.query.exception.ExceptionMessages.QueryExceptionTypeOrBuilder getExceptionsOrBuilder(int index) {
             return exceptions_.get(index);
@@ -1048,6 +1291,8 @@ public datawave.webservice.query.exception.ExceptionMessages.QueryExceptionTypeO
 
         /**
          * repeated .datawave.webservice.query.QueryImpl query = 4;
+         *
+         * @return the query list
          */
         public java.util.List getQueryList() {
             return query_;
@@ -1055,6 +1300,8 @@ public java.util.List getQuer
 
         /**
          * repeated .datawave.webservice.query.QueryImpl query = 4;
+         *
+         * @return the query list
          */
         public java.util.List getQueryOrBuilderList() {
             return query_;
@@ -1062,6 +1309,8 @@ public java.util.Listrepeated .datawave.webservice.query.QueryImpl query = 4;
+         *
+         * @return the number of queries in the list
          */
         public int getQueryCount() {
             return query_.size();
@@ -1069,6 +1318,11 @@ public int getQueryCount() {
 
         /**
          * repeated .datawave.webservice.query.QueryImpl query = 4;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the query at the provided index
          */
         public datawave.webservice.query.QueryMessages.QueryImpl getQuery(int index) {
             return query_.get(index);
@@ -1076,6 +1330,11 @@ public datawave.webservice.query.QueryMessages.QueryImpl getQuery(int index) {
 
         /**
          * repeated .datawave.webservice.query.QueryImpl query = 4;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the query at the provided index
          */
         public datawave.webservice.query.QueryMessages.QueryImplOrBuilder getQueryOrBuilder(int index) {
             return query_.get(index);
@@ -1087,6 +1346,8 @@ public datawave.webservice.query.QueryMessages.QueryImplOrBuilder getQueryOrBuil
 
         /**
          * optional uint32 numResults = 5 [default = 0];
+         *
+         * @return if there are a number of results
          */
         public boolean hasNumResults() {
             return ((bitField0_ & 0x00000002) == 0x00000002);
@@ -1094,6 +1355,8 @@ public boolean hasNumResults() {
 
         /**
          * optional uint32 numResults = 5 [default = 0];
+         *
+         * @return the number of results
          */
         public int getNumResults() {
             return numResults_;
@@ -1410,6 +1673,8 @@ public Builder mergeFrom(com.google.protobuf.CodedInputStream input, com.google.
 
             /**
              * required uint64 operation_time_ms = 1 [default = 0];
+             *
+             * @return if it has operation time
              */
             public boolean hasOperationTimeMs() {
                 return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -1417,6 +1682,8 @@ public boolean hasOperationTimeMs() {
 
             /**
              * required uint64 operation_time_ms = 1 [default = 0];
+             *
+             * @return the operation time in ms
              */
             public long getOperationTimeMs() {
                 return operationTimeMs_;
@@ -1424,6 +1691,11 @@ public long getOperationTimeMs() {
 
             /**
              * required uint64 operation_time_ms = 1 [default = 0];
+             *
+             * @param value
+             *            new operation time in ms
+             *
+             * @return builder to set operation time to provided value
              */
             public Builder setOperationTimeMs(long value) {
                 bitField0_ |= 0x00000001;
@@ -1434,6 +1706,8 @@ public Builder setOperationTimeMs(long value) {
 
             /**
              * required uint64 operation_time_ms = 1 [default = 0];
+             *
+             * @return builder to clear operation time
              */
             public Builder clearOperationTimeMs() {
                 bitField0_ = (bitField0_ & ~0x00000001);
@@ -1454,6 +1728,8 @@ private void ensureMessagesIsMutable() {
 
             /**
              * repeated string messages = 2;
+             *
+             * @return the messages list
              */
             public java.util.List getMessagesList() {
                 return java.util.Collections.unmodifiableList(messages_);
@@ -1461,6 +1737,8 @@ public java.util.List getMessagesList() {
 
             /**
              * repeated string messages = 2;
+             *
+             * @return the number of messages in the list
              */
             public int getMessagesCount() {
                 return messages_.size();
@@ -1468,6 +1746,11 @@ public int getMessagesCount() {
 
             /**
              * repeated string messages = 2;
+             *
+             * @param index
+             *            the index
+             *
+             * @return the message at the provided index
              */
             public java.lang.String getMessages(int index) {
                 return messages_.get(index);
@@ -1475,6 +1758,11 @@ public java.lang.String getMessages(int index) {
 
             /**
              * repeated string messages = 2;
+             *
+             * @param index
+             *            the index
+             *
+             * @return the message byte string at the provided index
              */
             public com.google.protobuf.ByteString getMessagesBytes(int index) {
                 return messages_.getByteString(index);
@@ -1482,6 +1770,13 @@ public com.google.protobuf.ByteString getMessagesBytes(int index) {
 
             /**
              * repeated string messages = 2;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the new message
+             *
+             * @return builder to set the message at the provided index to the provided value
              */
             public Builder setMessages(int index, java.lang.String value) {
                 if (value == null) {
@@ -1495,6 +1790,11 @@ public Builder setMessages(int index, java.lang.String value) {
 
             /**
              * repeated string messages = 2;
+             *
+             * @param value
+             *            the new message
+             *
+             * @return builder to add the message
              */
             public Builder addMessages(java.lang.String value) {
                 if (value == null) {
@@ -1508,6 +1808,11 @@ public Builder addMessages(java.lang.String value) {
 
             /**
              * repeated string messages = 2;
+             *
+             * @param values
+             *            iterable of messages to add to the list
+             *
+             * @return builder to add all messages
              */
             public Builder addAllMessages(java.lang.Iterable values) {
                 ensureMessagesIsMutable();
@@ -1518,6 +1823,8 @@ public Builder addAllMessages(java.lang.Iterable values) {
 
             /**
              * repeated string messages = 2;
+             *
+             * @return builder to clear messages
              */
             public Builder clearMessages() {
                 messages_ = com.google.protobuf.LazyStringArrayList.EMPTY;
@@ -1528,6 +1835,11 @@ public Builder clearMessages() {
 
             /**
              * repeated string messages = 2;
+             *
+             * @param value
+             *            message byte string
+             *
+             * @return builder to add message byte string
              */
             public Builder addMessagesBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -1551,6 +1863,8 @@ private void ensureExceptionsIsMutable() {
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @return list of exceptions
              */
             public java.util.List getExceptionsList() {
                 return java.util.Collections.unmodifiableList(exceptions_);
@@ -1558,6 +1872,8 @@ public java.util.Listrepeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @return the size of the exceptions list
              */
             public int getExceptionsCount() {
                 return exceptions_.size();
@@ -1565,6 +1881,11 @@ public int getExceptionsCount() {
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param index
+             *            the index
+             *
+             * @return the exception at the provided index
              */
             public datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType getExceptions(int index) {
                 return exceptions_.get(index);
@@ -1572,6 +1893,13 @@ public datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the exception to add
+             *
+             * @return builder to set the exception at the provided index
              */
             public Builder setExceptions(int index, datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType value) {
                 if (value == null) {
@@ -1585,6 +1913,13 @@ public Builder setExceptions(int index, datawave.webservice.query.exception.Exce
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param index
+             *            the index
+             * @param builderForValue
+             *            builder with the value for the exception to add
+             *
+             * @return builder to set exception to provided builder
              */
             public Builder setExceptions(int index, datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType.Builder builderForValue) {
                 ensureExceptionsIsMutable();
@@ -1595,6 +1930,11 @@ public Builder setExceptions(int index, datawave.webservice.query.exception.Exce
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param value
+             *            exception to add
+             *
+             * @return builder to add exceptions provided
              */
             public Builder addExceptions(datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType value) {
                 if (value == null) {
@@ -1608,6 +1948,13 @@ public Builder addExceptions(datawave.webservice.query.exception.ExceptionMessag
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the exception to add
+             *
+             * @return builder to add exception to provided index
              */
             public Builder addExceptions(int index, datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType value) {
                 if (value == null) {
@@ -1621,6 +1968,11 @@ public Builder addExceptions(int index, datawave.webservice.query.exception.Exce
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param builderForValue
+             *            builder for the exception
+             *
+             * @return builder to add exception provided
              */
             public Builder addExceptions(datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType.Builder builderForValue) {
                 ensureExceptionsIsMutable();
@@ -1631,6 +1983,13 @@ public Builder addExceptions(datawave.webservice.query.exception.ExceptionMessag
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param index
+             *            the index
+             * @param builderForValue
+             *            builder to create exception
+             *
+             * @return builder to add exception from builder to provided index
              */
             public Builder addExceptions(int index, datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType.Builder builderForValue) {
                 ensureExceptionsIsMutable();
@@ -1641,6 +2000,11 @@ public Builder addExceptions(int index, datawave.webservice.query.exception.Exce
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param values
+             *            iterable of exceptions to add
+             *
+             * @return builder to add provided exceptions
              */
             public Builder addAllExceptions(java.lang.Iterable values) {
                 ensureExceptionsIsMutable();
@@ -1651,6 +2015,8 @@ public Builder addAllExceptions(java.lang.Iterablerepeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @return builder to clear exceptions
              */
             public Builder clearExceptions() {
                 exceptions_ = java.util.Collections.emptyList();
@@ -1661,6 +2027,11 @@ public Builder clearExceptions() {
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param index
+             *            the index
+             *
+             * @return builder to remove the exception at the provided index
              */
             public Builder removeExceptions(int index) {
                 ensureExceptionsIsMutable();
@@ -1681,6 +2052,8 @@ private void ensureQueryIsMutable() {
 
             /**
              * repeated .datawave.webservice.query.QueryImpl query = 4;
+             *
+             * @return the query list
              */
             public java.util.List getQueryList() {
                 return java.util.Collections.unmodifiableList(query_);
@@ -1688,6 +2061,8 @@ public java.util.List getQuer
 
             /**
              * repeated .datawave.webservice.query.QueryImpl query = 4;
+             *
+             * @return the number of queries in the list
              */
             public int getQueryCount() {
                 return query_.size();
@@ -1695,6 +2070,11 @@ public int getQueryCount() {
 
             /**
              * repeated .datawave.webservice.query.QueryImpl query = 4;
+             *
+             * @param index
+             *            the index
+             *
+             * @return return the query at the provided index
              */
             public datawave.webservice.query.QueryMessages.QueryImpl getQuery(int index) {
                 return query_.get(index);
@@ -1702,6 +2082,13 @@ public datawave.webservice.query.QueryMessages.QueryImpl getQuery(int index) {
 
             /**
              * repeated .datawave.webservice.query.QueryImpl query = 4;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the query to set
+             *
+             * @return builder to set the query at the provided index to the provided value
              */
             public Builder setQuery(int index, datawave.webservice.query.QueryMessages.QueryImpl value) {
                 if (value == null) {
@@ -1715,6 +2102,13 @@ public Builder setQuery(int index, datawave.webservice.query.QueryMessages.Query
 
             /**
              * repeated .datawave.webservice.query.QueryImpl query = 4;
+             *
+             * @param index
+             *            the index
+             * @param builderForValue
+             *            the builder for the value
+             *
+             * @return builder to set query at the provided index
              */
             public Builder setQuery(int index, datawave.webservice.query.QueryMessages.QueryImpl.Builder builderForValue) {
                 ensureQueryIsMutable();
@@ -1725,6 +2119,11 @@ public Builder setQuery(int index, datawave.webservice.query.QueryMessages.Query
 
             /**
              * repeated .datawave.webservice.query.QueryImpl query = 4;
+             *
+             * @param value
+             *            the query to add
+             *
+             * @return builder to add the query value
              */
             public Builder addQuery(datawave.webservice.query.QueryMessages.QueryImpl value) {
                 if (value == null) {
@@ -1738,6 +2137,13 @@ public Builder addQuery(datawave.webservice.query.QueryMessages.QueryImpl value)
 
             /**
              * repeated .datawave.webservice.query.QueryImpl query = 4;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the query to add
+             *
+             * @return builder to add query to provided index
              */
             public Builder addQuery(int index, datawave.webservice.query.QueryMessages.QueryImpl value) {
                 if (value == null) {
@@ -1751,6 +2157,11 @@ public Builder addQuery(int index, datawave.webservice.query.QueryMessages.Query
 
             /**
              * repeated .datawave.webservice.query.QueryImpl query = 4;
+             *
+             * @param builderForValue
+             *            builder with the query to add
+             *
+             * @return builder to add query
              */
             public Builder addQuery(datawave.webservice.query.QueryMessages.QueryImpl.Builder builderForValue) {
                 ensureQueryIsMutable();
@@ -1761,6 +2172,13 @@ public Builder addQuery(datawave.webservice.query.QueryMessages.QueryImpl.Builde
 
             /**
              * repeated .datawave.webservice.query.QueryImpl query = 4;
+             *
+             * @param index
+             *            the index
+             * @param builderForValue
+             *            query to add
+             *
+             * @return builder to add query to provided index
              */
             public Builder addQuery(int index, datawave.webservice.query.QueryMessages.QueryImpl.Builder builderForValue) {
                 ensureQueryIsMutable();
@@ -1771,6 +2189,11 @@ public Builder addQuery(int index, datawave.webservice.query.QueryMessages.Query
 
             /**
              * repeated .datawave.webservice.query.QueryImpl query = 4;
+             *
+             * @param values
+             *            iterable of queries to add
+             *
+             * @return builder to add all provided queries
              */
             public Builder addAllQuery(java.lang.Iterable values) {
                 ensureQueryIsMutable();
@@ -1781,6 +2204,8 @@ public Builder addAllQuery(java.lang.Iterablerepeated .datawave.webservice.query.QueryImpl query = 4;
+             *
+             * @return builder to clear the query list
              */
             public Builder clearQuery() {
                 query_ = java.util.Collections.emptyList();
@@ -1791,6 +2216,11 @@ public Builder clearQuery() {
 
             /**
              * repeated .datawave.webservice.query.QueryImpl query = 4;
+             *
+             * @param index
+             *            the index
+             *
+             * @return builder to remove the query at the provided index
              */
             public Builder removeQuery(int index) {
                 ensureQueryIsMutable();
@@ -1804,6 +2234,8 @@ public Builder removeQuery(int index) {
 
             /**
              * optional uint32 numResults = 5 [default = 0];
+             *
+             * @return if it has a number of results
              */
             public boolean hasNumResults() {
                 return ((bitField0_ & 0x00000010) == 0x00000010);
@@ -1811,6 +2243,8 @@ public boolean hasNumResults() {
 
             /**
              * optional uint32 numResults = 5 [default = 0];
+             *
+             * @return the number of results
              */
             public int getNumResults() {
                 return numResults_;
@@ -1818,6 +2252,11 @@ public int getNumResults() {
 
             /**
              * optional uint32 numResults = 5 [default = 0];
+             *
+             * @param value
+             *            new number of results
+             *
+             * @return builder to set the number of results
              */
             public Builder setNumResults(int value) {
                 bitField0_ |= 0x00000010;
@@ -1828,6 +2267,8 @@ public Builder setNumResults(int value) {
 
             /**
              * optional uint32 numResults = 5 [default = 0];
+             *
+             * @return builder to clear the number of results
              */
             public Builder clearNumResults() {
                 bitField0_ = (bitField0_ & ~0x00000010);
@@ -1852,157 +2293,225 @@ public interface GenericResponseOrBuilder extends com.google.protobuf.MessageLit
         // required uint64 operation_time_ms = 1 [default = 0];
         /**
          * required uint64 operation_time_ms = 1 [default = 0];
+         *
+         * @return if it has operation time
          */
         boolean hasOperationTimeMs();
 
         /**
          * required uint64 operation_time_ms = 1 [default = 0];
+         *
+         * @return the operation time in ms
          */
         long getOperationTimeMs();
 
         // repeated string messages = 2;
+
         /**
          * repeated string messages = 2;
+         *
+         * @return list of messages
          */
         java.util.List getMessagesList();
 
         /**
          * repeated string messages = 2;
+         *
+         * @return the number of messages
          */
         int getMessagesCount();
 
         /**
          * repeated string messages = 2;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the message at the provided index
          */
         java.lang.String getMessages(int index);
 
         /**
          * repeated string messages = 2;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the byte string for the message at the provided index
          */
         com.google.protobuf.ByteString getMessagesBytes(int index);
 
         // repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
         /**
          * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @return the list of exceptions
          */
         java.util.List getExceptionsList();
 
         /**
          * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the exception at the provided index
          */
         datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType getExceptions(int index);
 
         /**
          * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @return the number of exceptions
          */
         int getExceptionsCount();
 
         // required string result_class_name = 4;
         /**
          * required string result_class_name = 4;
+         *
+         * @return if it has a result class name
          */
         boolean hasResultClassName();
 
         /**
          * required string result_class_name = 4;
+         *
+         * @return the result class name
          */
         java.lang.String getResultClassName();
 
         /**
          * required string result_class_name = 4;
+         *
+         * @return the result class name byte string
          */
         com.google.protobuf.ByteString getResultClassNameBytes();
 
         // optional string result_as_string = 5;
         /**
          * optional string result_as_string = 5;
+         *
+         * @return if it has result as a string
          */
         boolean hasResultAsString();
 
         /**
          * optional string result_as_string = 5;
+         *
+         * @return the results as a string
          */
         java.lang.String getResultAsString();
 
         /**
          * optional string result_as_string = 5;
+         *
+         * @return the result as a byte string
          */
         com.google.protobuf.ByteString getResultAsStringBytes();
 
         // optional bool result_as_boolean = 6;
         /**
          * optional bool result_as_boolean = 6;
+         *
+         * @return if it has the result as boolean
          */
         boolean hasResultAsBoolean();
 
         /**
          * optional bool result_as_boolean = 6;
+         *
+         * @return the result as a boolean
          */
         boolean getResultAsBoolean();
 
         // optional sint32 result_as_int = 7;
         /**
          * optional sint32 result_as_int = 7;
+         *
+         * @return if it has result as integer
          */
         boolean hasResultAsInt();
 
         /**
          * optional sint32 result_as_int = 7;
+         *
+         * @return the result as an integer
          */
         int getResultAsInt();
 
         // optional sint64 result_as_long = 8;
         /**
          * optional sint64 result_as_long = 8;
+         *
+         * @return if it has result as a long
          */
         boolean hasResultAsLong();
 
         /**
          * optional sint64 result_as_long = 8;
+         *
+         * @return the result as a long
          */
         long getResultAsLong();
 
         // optional float result_as_float = 9;
         /**
          * optional float result_as_float = 9;
+         *
+         * @return if it has the result as a float
          */
         boolean hasResultAsFloat();
 
         /**
          * optional float result_as_float = 9;
+         *
+         * @return the result as a float
          */
         float getResultAsFloat();
 
         // optional double result_as_double = 10;
         /**
          * optional double result_as_double = 10;
+         *
+         * @return if it has the result as a double
          */
         boolean hasResultAsDouble();
 
         /**
          * optional double result_as_double = 10;
+         *
+         * @return the result as a double
          */
         double getResultAsDouble();
 
         // optional bytes result_as_bytes = 11;
         /**
          * optional bytes result_as_bytes = 11;
+         *
+         * @return if it has result as bytes
          */
         boolean hasResultAsBytes();
 
         /**
          * optional bytes result_as_bytes = 11;
+         *
+         * @return the result as bytes
          */
         com.google.protobuf.ByteString getResultAsBytes();
 
         // optional .datawave.webservice.results.cached.result.Description result_as_description = 12;
         /**
          * optional .datawave.webservice.results.cached.result.Description result_as_description = 12;
+         *
+         * @return if the result is a description
          */
         boolean hasResultAsDescription();
 
         /**
          * optional .datawave.webservice.results.cached.result.Description result_as_description = 12;
+         *
+         * @return the result as a description
          */
         datawave.webservice.results.cached.result.CachedresultMessages.Description getResultAsDescription();
     }
@@ -2159,6 +2668,8 @@ public com.google.protobuf.Parser getParserForType() {
 
         /**
          * required uint64 operation_time_ms = 1 [default = 0];
+         *
+         * @return if it has an operation time
          */
         public boolean hasOperationTimeMs() {
             return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -2166,6 +2677,8 @@ public boolean hasOperationTimeMs() {
 
         /**
          * required uint64 operation_time_ms = 1 [default = 0];
+         *
+         * @return the operation time in ms
          */
         public long getOperationTimeMs() {
             return operationTimeMs_;
@@ -2177,6 +2690,8 @@ public long getOperationTimeMs() {
 
         /**
          * repeated string messages = 2;
+         *
+         * @return the message list
          */
         public java.util.List getMessagesList() {
             return messages_;
@@ -2184,6 +2699,8 @@ public java.util.List getMessagesList() {
 
         /**
          * repeated string messages = 2;
+         *
+         * @return the number of messages in the lsit
          */
         public int getMessagesCount() {
             return messages_.size();
@@ -2191,6 +2708,11 @@ public int getMessagesCount() {
 
         /**
          * repeated string messages = 2;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the message at the provided index
          */
         public java.lang.String getMessages(int index) {
             return messages_.get(index);
@@ -2198,6 +2720,11 @@ public java.lang.String getMessages(int index) {
 
         /**
          * repeated string messages = 2;
+         *
+         * @param index
+         *            the index
+         *
+         * @return message as byte string at the provided index
          */
         public com.google.protobuf.ByteString getMessagesBytes(int index) {
             return messages_.getByteString(index);
@@ -2209,6 +2736,8 @@ public com.google.protobuf.ByteString getMessagesBytes(int index) {
 
         /**
          * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @return the exceptions list
          */
         public java.util.List getExceptionsList() {
             return exceptions_;
@@ -2216,6 +2745,8 @@ public java.util.Listrepeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @return the exceptions list
          */
         public java.util.List getExceptionsOrBuilderList() {
             return exceptions_;
@@ -2223,6 +2754,8 @@ public java.util.Listrepeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @return the number of exceptions in the list
          */
         public int getExceptionsCount() {
             return exceptions_.size();
@@ -2230,6 +2763,11 @@ public int getExceptionsCount() {
 
         /**
          * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the exception at the provided index
          */
         public datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType getExceptions(int index) {
             return exceptions_.get(index);
@@ -2237,6 +2775,11 @@ public datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType
 
         /**
          * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the exception at the index provided
          */
         public datawave.webservice.query.exception.ExceptionMessages.QueryExceptionTypeOrBuilder getExceptionsOrBuilder(int index) {
             return exceptions_.get(index);
@@ -2248,6 +2791,8 @@ public datawave.webservice.query.exception.ExceptionMessages.QueryExceptionTypeO
 
         /**
          * required string result_class_name = 4;
+         *
+         * @return if it has a result class name
          */
         public boolean hasResultClassName() {
             return ((bitField0_ & 0x00000002) == 0x00000002);
@@ -2255,6 +2800,8 @@ public boolean hasResultClassName() {
 
         /**
          * required string result_class_name = 4;
+         *
+         * @return the result class name
          */
         public java.lang.String getResultClassName() {
             java.lang.Object ref = resultClassName_;
@@ -2272,6 +2819,8 @@ public java.lang.String getResultClassName() {
 
         /**
          * required string result_class_name = 4;
+         *
+         * @return the result class name byte string
          */
         public com.google.protobuf.ByteString getResultClassNameBytes() {
             java.lang.Object ref = resultClassName_;
@@ -2290,6 +2839,8 @@ public com.google.protobuf.ByteString getResultClassNameBytes() {
 
         /**
          * optional string result_as_string = 5;
+         *
+         * @return if it has results as a string
          */
         public boolean hasResultAsString() {
             return ((bitField0_ & 0x00000004) == 0x00000004);
@@ -2297,6 +2848,8 @@ public boolean hasResultAsString() {
 
         /**
          * optional string result_as_string = 5;
+         *
+         * @return the results as a string
          */
         public java.lang.String getResultAsString() {
             java.lang.Object ref = resultAsString_;
@@ -2314,6 +2867,8 @@ public java.lang.String getResultAsString() {
 
         /**
          * optional string result_as_string = 5;
+         *
+         * @return the result as a byte string
          */
         public com.google.protobuf.ByteString getResultAsStringBytes() {
             java.lang.Object ref = resultAsString_;
@@ -2332,6 +2887,8 @@ public com.google.protobuf.ByteString getResultAsStringBytes() {
 
         /**
          * optional bool result_as_boolean = 6;
+         *
+         * @return if it has result as a boolean
          */
         public boolean hasResultAsBoolean() {
             return ((bitField0_ & 0x00000008) == 0x00000008);
@@ -2339,6 +2896,8 @@ public boolean hasResultAsBoolean() {
 
         /**
          * optional bool result_as_boolean = 6;
+         *
+         * @return the result as a boolean
          */
         public boolean getResultAsBoolean() {
             return resultAsBoolean_;
@@ -2350,6 +2909,8 @@ public boolean getResultAsBoolean() {
 
         /**
          * optional sint32 result_as_int = 7;
+         *
+         * @return if it has result as an integer
          */
         public boolean hasResultAsInt() {
             return ((bitField0_ & 0x00000010) == 0x00000010);
@@ -2357,6 +2918,8 @@ public boolean hasResultAsInt() {
 
         /**
          * optional sint32 result_as_int = 7;
+         *
+         * @return result as an integer
          */
         public int getResultAsInt() {
             return resultAsInt_;
@@ -2368,6 +2931,8 @@ public int getResultAsInt() {
 
         /**
          * optional sint64 result_as_long = 8;
+         *
+         * @return if it has result as a long
          */
         public boolean hasResultAsLong() {
             return ((bitField0_ & 0x00000020) == 0x00000020);
@@ -2375,6 +2940,8 @@ public boolean hasResultAsLong() {
 
         /**
          * optional sint64 result_as_long = 8;
+         *
+         * @return the result as a long
          */
         public long getResultAsLong() {
             return resultAsLong_;
@@ -2386,6 +2953,8 @@ public long getResultAsLong() {
 
         /**
          * optional float result_as_float = 9;
+         *
+         * @return if it has result as a float
          */
         public boolean hasResultAsFloat() {
             return ((bitField0_ & 0x00000040) == 0x00000040);
@@ -2393,6 +2962,8 @@ public boolean hasResultAsFloat() {
 
         /**
          * optional float result_as_float = 9;
+         *
+         * @return the result as a float
          */
         public float getResultAsFloat() {
             return resultAsFloat_;
@@ -2404,6 +2975,8 @@ public float getResultAsFloat() {
 
         /**
          * optional double result_as_double = 10;
+         *
+         * @return if it has a result as a double
          */
         public boolean hasResultAsDouble() {
             return ((bitField0_ & 0x00000080) == 0x00000080);
@@ -2411,6 +2984,8 @@ public boolean hasResultAsDouble() {
 
         /**
          * optional double result_as_double = 10;
+         *
+         * @return the result as a double
          */
         public double getResultAsDouble() {
             return resultAsDouble_;
@@ -2422,6 +2997,8 @@ public double getResultAsDouble() {
 
         /**
          * optional bytes result_as_bytes = 11;
+         *
+         * @return if it has result as a byte string
          */
         public boolean hasResultAsBytes() {
             return ((bitField0_ & 0x00000100) == 0x00000100);
@@ -2429,6 +3006,8 @@ public boolean hasResultAsBytes() {
 
         /**
          * optional bytes result_as_bytes = 11;
+         *
+         * @return the result as a byte string
          */
         public com.google.protobuf.ByteString getResultAsBytes() {
             return resultAsBytes_;
@@ -2440,6 +3019,8 @@ public com.google.protobuf.ByteString getResultAsBytes() {
 
         /**
          * optional .datawave.webservice.results.cached.result.Description result_as_description = 12;
+         *
+         * @return if iut has the result as a description
          */
         public boolean hasResultAsDescription() {
             return ((bitField0_ & 0x00000200) == 0x00000200);
@@ -2447,6 +3028,8 @@ public boolean hasResultAsDescription() {
 
         /**
          * optional .datawave.webservice.results.cached.result.Description result_as_description = 12;
+         *
+         * @return the result as a description
          */
         public datawave.webservice.results.cached.result.CachedresultMessages.Description getResultAsDescription() {
             return resultAsDescription_;
@@ -2877,6 +3460,8 @@ public Builder mergeFrom(com.google.protobuf.CodedInputStream input, com.google.
 
             /**
              * required uint64 operation_time_ms = 1 [default = 0];
+             *
+             * @return if it has operation time
              */
             public boolean hasOperationTimeMs() {
                 return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -2884,6 +3469,8 @@ public boolean hasOperationTimeMs() {
 
             /**
              * required uint64 operation_time_ms = 1 [default = 0];
+             *
+             * @return the operation time in ms
              */
             public long getOperationTimeMs() {
                 return operationTimeMs_;
@@ -2891,6 +3478,11 @@ public long getOperationTimeMs() {
 
             /**
              * required uint64 operation_time_ms = 1 [default = 0];
+             *
+             * @param value
+             *            the new operation time in ms
+             *
+             * @return builder to set operation time in ms
              */
             public Builder setOperationTimeMs(long value) {
                 bitField0_ |= 0x00000001;
@@ -2901,6 +3493,8 @@ public Builder setOperationTimeMs(long value) {
 
             /**
              * required uint64 operation_time_ms = 1 [default = 0];
+             *
+             * @return builder to clear operation time
              */
             public Builder clearOperationTimeMs() {
                 bitField0_ = (bitField0_ & ~0x00000001);
@@ -2921,6 +3515,8 @@ private void ensureMessagesIsMutable() {
 
             /**
              * repeated string messages = 2;
+             *
+             * @return the message list
              */
             public java.util.List getMessagesList() {
                 return java.util.Collections.unmodifiableList(messages_);
@@ -2928,6 +3524,8 @@ public java.util.List getMessagesList() {
 
             /**
              * repeated string messages = 2;
+             *
+             * @return the messages count
              */
             public int getMessagesCount() {
                 return messages_.size();
@@ -2935,6 +3533,11 @@ public int getMessagesCount() {
 
             /**
              * repeated string messages = 2;
+             *
+             * @param index
+             *            the index
+             *
+             * @return the message at the provided index
              */
             public java.lang.String getMessages(int index) {
                 return messages_.get(index);
@@ -2942,6 +3545,11 @@ public java.lang.String getMessages(int index) {
 
             /**
              * repeated string messages = 2;
+             *
+             * @param index
+             *            the index
+             *
+             * @return the message byte string for the provided index
              */
             public com.google.protobuf.ByteString getMessagesBytes(int index) {
                 return messages_.getByteString(index);
@@ -2949,6 +3557,13 @@ public com.google.protobuf.ByteString getMessagesBytes(int index) {
 
             /**
              * repeated string messages = 2;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the message to add
+             *
+             * @return builder to set the message at the provided index
              */
             public Builder setMessages(int index, java.lang.String value) {
                 if (value == null) {
@@ -2962,6 +3577,11 @@ public Builder setMessages(int index, java.lang.String value) {
 
             /**
              * repeated string messages = 2;
+             *
+             * @param value
+             *            the new message
+             *
+             * @return builder to add the provided value
              */
             public Builder addMessages(java.lang.String value) {
                 if (value == null) {
@@ -2975,6 +3595,11 @@ public Builder addMessages(java.lang.String value) {
 
             /**
              * repeated string messages = 2;
+             *
+             * @param values
+             *            iterable of messages to add
+             *
+             * @return builder to add the values provided
              */
             public Builder addAllMessages(java.lang.Iterable values) {
                 ensureMessagesIsMutable();
@@ -2985,6 +3610,8 @@ public Builder addAllMessages(java.lang.Iterable values) {
 
             /**
              * repeated string messages = 2;
+             *
+             * @return builder to clear messages
              */
             public Builder clearMessages() {
                 messages_ = com.google.protobuf.LazyStringArrayList.EMPTY;
@@ -2995,6 +3622,11 @@ public Builder clearMessages() {
 
             /**
              * repeated string messages = 2;
+             *
+             * @param value
+             *            message byte string
+             *
+             * @return builder to add message byte string
              */
             public Builder addMessagesBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -3018,6 +3650,8 @@ private void ensureExceptionsIsMutable() {
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @return the list of exceptions
              */
             public java.util.List getExceptionsList() {
                 return java.util.Collections.unmodifiableList(exceptions_);
@@ -3025,6 +3659,8 @@ public java.util.Listrepeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @return the number of exceptions
              */
             public int getExceptionsCount() {
                 return exceptions_.size();
@@ -3032,6 +3668,11 @@ public int getExceptionsCount() {
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param index
+             *            the index
+             *
+             * @return the exception at the provided index
              */
             public datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType getExceptions(int index) {
                 return exceptions_.get(index);
@@ -3039,6 +3680,13 @@ public datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the new exception to add
+             *
+             * @return builder to set exception at index to provided value
              */
             public Builder setExceptions(int index, datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType value) {
                 if (value == null) {
@@ -3052,6 +3700,13 @@ public Builder setExceptions(int index, datawave.webservice.query.exception.Exce
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param index
+             *            the index
+             * @param builderForValue
+             *            builder for the value
+             *
+             * @return builder to set exception at provided index
              */
             public Builder setExceptions(int index, datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType.Builder builderForValue) {
                 ensureExceptionsIsMutable();
@@ -3062,6 +3717,11 @@ public Builder setExceptions(int index, datawave.webservice.query.exception.Exce
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param value
+             *            the exception to add
+             *
+             * @return builder to add exception
              */
             public Builder addExceptions(datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType value) {
                 if (value == null) {
@@ -3075,6 +3735,13 @@ public Builder addExceptions(datawave.webservice.query.exception.ExceptionMessag
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the exception to add
+             *
+             * @return builder to add exception to the list
              */
             public Builder addExceptions(int index, datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType value) {
                 if (value == null) {
@@ -3088,6 +3755,11 @@ public Builder addExceptions(int index, datawave.webservice.query.exception.Exce
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param builderForValue
+             *            builder to create exception to add
+             *
+             * @return builder to add exception
              */
             public Builder addExceptions(datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType.Builder builderForValue) {
                 ensureExceptionsIsMutable();
@@ -3098,6 +3770,13 @@ public Builder addExceptions(datawave.webservice.query.exception.ExceptionMessag
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param index
+             *            the index
+             * @param builderForValue
+             *            builder to create exception
+             *
+             * @return builder to add exception at provided index
              */
             public Builder addExceptions(int index, datawave.webservice.query.exception.ExceptionMessages.QueryExceptionType.Builder builderForValue) {
                 ensureExceptionsIsMutable();
@@ -3108,6 +3787,11 @@ public Builder addExceptions(int index, datawave.webservice.query.exception.Exce
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param values
+             *            iterable of exceptions to add
+             *
+             * @return builder to add exceptions
              */
             public Builder addAllExceptions(java.lang.Iterable values) {
                 ensureExceptionsIsMutable();
@@ -3118,6 +3802,8 @@ public Builder addAllExceptions(java.lang.Iterablerepeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @return builder to clear exceptions
              */
             public Builder clearExceptions() {
                 exceptions_ = java.util.Collections.emptyList();
@@ -3128,6 +3814,11 @@ public Builder clearExceptions() {
 
             /**
              * repeated .datawave.webservice.query.exception.QueryExceptionType exceptions = 3;
+             *
+             * @param index
+             *            the index
+             *
+             * @return builder to remove exception at given index
              */
             public Builder removeExceptions(int index) {
                 ensureExceptionsIsMutable();
@@ -3141,6 +3832,8 @@ public Builder removeExceptions(int index) {
 
             /**
              * required string result_class_name = 4;
+             *
+             * @return if it has result class name
              */
             public boolean hasResultClassName() {
                 return ((bitField0_ & 0x00000008) == 0x00000008);
@@ -3148,6 +3841,8 @@ public boolean hasResultClassName() {
 
             /**
              * required string result_class_name = 4;
+             *
+             * @return the result class name
              */
             public java.lang.String getResultClassName() {
                 java.lang.Object ref = resultClassName_;
@@ -3162,6 +3857,8 @@ public java.lang.String getResultClassName() {
 
             /**
              * required string result_class_name = 4;
+             *
+             * @return result class name byte string
              */
             public com.google.protobuf.ByteString getResultClassNameBytes() {
                 java.lang.Object ref = resultClassName_;
@@ -3176,6 +3873,11 @@ public com.google.protobuf.ByteString getResultClassNameBytes() {
 
             /**
              * required string result_class_name = 4;
+             *
+             * @param value
+             *            the new result class name
+             *
+             * @return builder to set result class name
              */
             public Builder setResultClassName(java.lang.String value) {
                 if (value == null) {
@@ -3189,6 +3891,8 @@ public Builder setResultClassName(java.lang.String value) {
 
             /**
              * required string result_class_name = 4;
+             *
+             * @return builder to clear the result class name
              */
             public Builder clearResultClassName() {
                 bitField0_ = (bitField0_ & ~0x00000008);
@@ -3199,6 +3903,11 @@ public Builder clearResultClassName() {
 
             /**
              * required string result_class_name = 4;
+             *
+             * @param value
+             *            result class name byte string
+             *
+             * @return builder to set result class name
              */
             public Builder setResultClassNameBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -3215,6 +3924,8 @@ public Builder setResultClassNameBytes(com.google.protobuf.ByteString value) {
 
             /**
              * optional string result_as_string = 5;
+             *
+             * return if it has result as a string
              */
             public boolean hasResultAsString() {
                 return ((bitField0_ & 0x00000010) == 0x00000010);
@@ -3222,6 +3933,8 @@ public boolean hasResultAsString() {
 
             /**
              * optional string result_as_string = 5;
+             *
+             * @return the result as a string
              */
             public java.lang.String getResultAsString() {
                 java.lang.Object ref = resultAsString_;
@@ -3236,6 +3949,8 @@ public java.lang.String getResultAsString() {
 
             /**
              * optional string result_as_string = 5;
+             *
+             * @return the result as a string
              */
             public com.google.protobuf.ByteString getResultAsStringBytes() {
                 java.lang.Object ref = resultAsString_;
@@ -3250,6 +3965,11 @@ public com.google.protobuf.ByteString getResultAsStringBytes() {
 
             /**
              * optional string result_as_string = 5;
+             *
+             * @param value
+             *            new result string
+             *
+             * @return builder to set result string
              */
             public Builder setResultAsString(java.lang.String value) {
                 if (value == null) {
@@ -3263,6 +3983,8 @@ public Builder setResultAsString(java.lang.String value) {
 
             /**
              * optional string result_as_string = 5;
+             *
+             * @return builder to clear string result
              */
             public Builder clearResultAsString() {
                 bitField0_ = (bitField0_ & ~0x00000010);
@@ -3273,6 +3995,11 @@ public Builder clearResultAsString() {
 
             /**
              * optional string result_as_string = 5;
+             *
+             * @param value
+             *            byte string
+             *
+             * @return builder to set result to byte string
              */
             public Builder setResultAsStringBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -3289,6 +4016,8 @@ public Builder setResultAsStringBytes(com.google.protobuf.ByteString value) {
 
             /**
              * optional bool result_as_boolean = 6;
+             *
+             * @return if the result is a boolean
              */
             public boolean hasResultAsBoolean() {
                 return ((bitField0_ & 0x00000020) == 0x00000020);
@@ -3296,6 +4025,8 @@ public boolean hasResultAsBoolean() {
 
             /**
              * optional bool result_as_boolean = 6;
+             *
+             * @return the result as a boolean
              */
             public boolean getResultAsBoolean() {
                 return resultAsBoolean_;
@@ -3303,6 +4034,11 @@ public boolean getResultAsBoolean() {
 
             /**
              * optional bool result_as_boolean = 6;
+             *
+             * @param value
+             *            the new boolean value
+             *
+             * @return the result as a boolean
              */
             public Builder setResultAsBoolean(boolean value) {
                 bitField0_ |= 0x00000020;
@@ -3313,6 +4049,8 @@ public Builder setResultAsBoolean(boolean value) {
 
             /**
              * optional bool result_as_boolean = 6;
+             *
+             * @return builder to clear result as a boolean
              */
             public Builder clearResultAsBoolean() {
                 bitField0_ = (bitField0_ & ~0x00000020);
@@ -3326,6 +4064,8 @@ public Builder clearResultAsBoolean() {
 
             /**
              * optional sint32 result_as_int = 7;
+             *
+             * @return if the result has an integer
              */
             public boolean hasResultAsInt() {
                 return ((bitField0_ & 0x00000040) == 0x00000040);
@@ -3333,6 +4073,8 @@ public boolean hasResultAsInt() {
 
             /**
              * optional sint32 result_as_int = 7;
+             *
+             * @return the result as an integer
              */
             public int getResultAsInt() {
                 return resultAsInt_;
@@ -3340,6 +4082,11 @@ public int getResultAsInt() {
 
             /**
              * optional sint32 result_as_int = 7;
+             *
+             * @param value
+             *            the new int value
+             *
+             * @return builder to set result to provided value
              */
             public Builder setResultAsInt(int value) {
                 bitField0_ |= 0x00000040;
@@ -3350,6 +4097,8 @@ public Builder setResultAsInt(int value) {
 
             /**
              * optional sint32 result_as_int = 7;
+             *
+             * @return builder to clear results as int
              */
             public Builder clearResultAsInt() {
                 bitField0_ = (bitField0_ & ~0x00000040);
@@ -3363,6 +4112,8 @@ public Builder clearResultAsInt() {
 
             /**
              * optional sint64 result_as_long = 8;
+             *
+             * @return if it has result as a long
              */
             public boolean hasResultAsLong() {
                 return ((bitField0_ & 0x00000080) == 0x00000080);
@@ -3370,6 +4121,8 @@ public boolean hasResultAsLong() {
 
             /**
              * optional sint64 result_as_long = 8;
+             *
+             * @return the result as a long
              */
             public long getResultAsLong() {
                 return resultAsLong_;
@@ -3377,6 +4130,11 @@ public long getResultAsLong() {
 
             /**
              * optional sint64 result_as_long = 8;
+             *
+             * @param value
+             *            the new long value
+             *
+             * @return builder to set result to long value provided
              */
             public Builder setResultAsLong(long value) {
                 bitField0_ |= 0x00000080;
@@ -3387,6 +4145,8 @@ public Builder setResultAsLong(long value) {
 
             /**
              * optional sint64 result_as_long = 8;
+             *
+             * @return builder to clear result as long
              */
             public Builder clearResultAsLong() {
                 bitField0_ = (bitField0_ & ~0x00000080);
@@ -3400,6 +4160,8 @@ public Builder clearResultAsLong() {
 
             /**
              * optional float result_as_float = 9;
+             *
+             * @return if the result is a float
              */
             public boolean hasResultAsFloat() {
                 return ((bitField0_ & 0x00000100) == 0x00000100);
@@ -3407,6 +4169,8 @@ public boolean hasResultAsFloat() {
 
             /**
              * optional float result_as_float = 9;
+             *
+             * @return the result as a float
              */
             public float getResultAsFloat() {
                 return resultAsFloat_;
@@ -3414,6 +4178,11 @@ public float getResultAsFloat() {
 
             /**
              * optional float result_as_float = 9;
+             *
+             * @param value
+             *            the new float value
+             *
+             * @return builder to set result as float provided
              */
             public Builder setResultAsFloat(float value) {
                 bitField0_ |= 0x00000100;
@@ -3424,6 +4193,8 @@ public Builder setResultAsFloat(float value) {
 
             /**
              * optional float result_as_float = 9;
+             *
+             * @return builder to clear results as float
              */
             public Builder clearResultAsFloat() {
                 bitField0_ = (bitField0_ & ~0x00000100);
@@ -3437,6 +4208,8 @@ public Builder clearResultAsFloat() {
 
             /**
              * optional double result_as_double = 10;
+             *
+             * @return if it has result as a double
              */
             public boolean hasResultAsDouble() {
                 return ((bitField0_ & 0x00000200) == 0x00000200);
@@ -3444,6 +4217,8 @@ public boolean hasResultAsDouble() {
 
             /**
              * optional double result_as_double = 10;
+             *
+             * @return results as a double
              */
             public double getResultAsDouble() {
                 return resultAsDouble_;
@@ -3451,6 +4226,11 @@ public double getResultAsDouble() {
 
             /**
              * optional double result_as_double = 10;
+             *
+             * @param value
+             *            the new double value
+             *
+             * @return builder to set result as a double
              */
             public Builder setResultAsDouble(double value) {
                 bitField0_ |= 0x00000200;
@@ -3461,6 +4241,8 @@ public Builder setResultAsDouble(double value) {
 
             /**
              * optional double result_as_double = 10;
+             *
+             * @return builder to clear result as double
              */
             public Builder clearResultAsDouble() {
                 bitField0_ = (bitField0_ & ~0x00000200);
@@ -3474,6 +4256,8 @@ public Builder clearResultAsDouble() {
 
             /**
              * optional bytes result_as_bytes = 11;
+             *
+             * @return if it has result as byte string
              */
             public boolean hasResultAsBytes() {
                 return ((bitField0_ & 0x00000400) == 0x00000400);
@@ -3481,6 +4265,8 @@ public boolean hasResultAsBytes() {
 
             /**
              * optional bytes result_as_bytes = 11;
+             *
+             * @return the result as byte string
              */
             public com.google.protobuf.ByteString getResultAsBytes() {
                 return resultAsBytes_;
@@ -3488,6 +4274,11 @@ public com.google.protobuf.ByteString getResultAsBytes() {
 
             /**
              * optional bytes result_as_bytes = 11;
+             *
+             * @param value
+             *            new value as byte string
+             *
+             * @return builder to set result as byte string
              */
             public Builder setResultAsBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -3501,6 +4292,8 @@ public Builder setResultAsBytes(com.google.protobuf.ByteString value) {
 
             /**
              * optional bytes result_as_bytes = 11;
+             *
+             * @return builder to clear rseult as byte string
              */
             public Builder clearResultAsBytes() {
                 bitField0_ = (bitField0_ & ~0x00000400);
@@ -3515,6 +4308,8 @@ public Builder clearResultAsBytes() {
 
             /**
              * optional .datawave.webservice.results.cached.result.Description result_as_description = 12;
+             *
+             * @return if it has result as a description
              */
             public boolean hasResultAsDescription() {
                 return ((bitField0_ & 0x00000800) == 0x00000800);
@@ -3522,6 +4317,8 @@ public boolean hasResultAsDescription() {
 
             /**
              * optional .datawave.webservice.results.cached.result.Description result_as_description = 12;
+             *
+             * @return the result as a description
              */
             public datawave.webservice.results.cached.result.CachedresultMessages.Description getResultAsDescription() {
                 return resultAsDescription_;
@@ -3529,6 +4326,11 @@ public datawave.webservice.results.cached.result.CachedresultMessages.Descriptio
 
             /**
              * optional .datawave.webservice.results.cached.result.Description result_as_description = 12;
+             *
+             * @param value
+             *            the new description value
+             *
+             * @return builder to set result as description provided
              */
             public Builder setResultAsDescription(datawave.webservice.results.cached.result.CachedresultMessages.Description value) {
                 if (value == null) {
@@ -3542,6 +4344,11 @@ public Builder setResultAsDescription(datawave.webservice.results.cached.result.
 
             /**
              * optional .datawave.webservice.results.cached.result.Description result_as_description = 12;
+             *
+             * @param builderForValue
+             *            builder to create description
+             *
+             * @return builder to set result to provided description
              */
             public Builder setResultAsDescription(datawave.webservice.results.cached.result.CachedresultMessages.Description.Builder builderForValue) {
                 resultAsDescription_ = builderForValue.build();
@@ -3552,6 +4359,11 @@ public Builder setResultAsDescription(datawave.webservice.results.cached.result.
 
             /**
              * optional .datawave.webservice.results.cached.result.Description result_as_description = 12;
+             *
+             * @param value
+             *            the new description value
+             *
+             * @return builder to merge result as description provided
              */
             public Builder mergeResultAsDescription(datawave.webservice.results.cached.result.CachedresultMessages.Description value) {
                 if (((bitField0_ & 0x00000800) == 0x00000800)
@@ -3568,6 +4380,8 @@ public Builder mergeResultAsDescription(datawave.webservice.results.cached.resul
 
             /**
              * optional .datawave.webservice.results.cached.result.Description result_as_description = 12;
+             *
+             * @return builder to clear the result as description
              */
             public Builder clearResultAsDescription() {
                 resultAsDescription_ = datawave.webservice.results.cached.result.CachedresultMessages.Description.getDefaultInstance();
diff --git a/web-services/client/src/main/java/datawave/webservice/results/cached/result/CachedresultMessages.java b/web-services/client/src/main/java/datawave/webservice/results/cached/result/CachedresultMessages.java
index de2d63157fb..c9d3e2aa47a 100644
--- a/web-services/client/src/main/java/datawave/webservice/results/cached/result/CachedresultMessages.java
+++ b/web-services/client/src/main/java/datawave/webservice/results/cached/result/CachedresultMessages.java
@@ -13,16 +13,22 @@ public interface ColumnVisibilityOrBuilder extends com.google.protobuf.MessageLi
         // required string visibility = 1;
         /**
          * required string visibility = 1;
+         *
+         * @return if it has visibility
          */
         boolean hasVisibility();
 
         /**
          * required string visibility = 1;
+         *
+         * @return the visibility
          */
         java.lang.String getVisibility();
 
         /**
          * required string visibility = 1;
+         *
+         * @return the visibility as a byte string
          */
         com.google.protobuf.ByteString getVisibilityBytes();
     }
@@ -102,6 +108,8 @@ public com.google.protobuf.Parser getParserForType() {
 
         /**
          * required string visibility = 1;
+         *
+         * @return if it has visibility
          */
         public boolean hasVisibility() {
             return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -109,6 +117,8 @@ public boolean hasVisibility() {
 
         /**
          * required string visibility = 1;
+         *
+         * @return the visibility
          */
         public java.lang.String getVisibility() {
             java.lang.Object ref = visibility_;
@@ -126,6 +136,8 @@ public java.lang.String getVisibility() {
 
         /**
          * required string visibility = 1;
+         *
+         * @return the visibility as a byte string
          */
         public com.google.protobuf.ByteString getVisibilityBytes() {
             java.lang.Object ref = visibility_;
@@ -347,6 +359,8 @@ public Builder mergeFrom(com.google.protobuf.CodedInputStream input, com.google.
 
             /**
              * required string visibility = 1;
+             *
+             * @return if it has visibility
              */
             public boolean hasVisibility() {
                 return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -354,6 +368,8 @@ public boolean hasVisibility() {
 
             /**
              * required string visibility = 1;
+             *
+             * @return the visibility
              */
             public java.lang.String getVisibility() {
                 java.lang.Object ref = visibility_;
@@ -368,6 +384,8 @@ public java.lang.String getVisibility() {
 
             /**
              * required string visibility = 1;
+             *
+             * @return the visibility byte string
              */
             public com.google.protobuf.ByteString getVisibilityBytes() {
                 java.lang.Object ref = visibility_;
@@ -382,6 +400,11 @@ public com.google.protobuf.ByteString getVisibilityBytes() {
 
             /**
              * required string visibility = 1;
+             *
+             * @param value
+             *            the new visibility value
+             *
+             * @return builder to set visibility to the provided value
              */
             public Builder setVisibility(java.lang.String value) {
                 if (value == null) {
@@ -395,6 +418,8 @@ public Builder setVisibility(java.lang.String value) {
 
             /**
              * required string visibility = 1;
+             *
+             * @return builder to clear visibility
              */
             public Builder clearVisibility() {
                 bitField0_ = (bitField0_ & ~0x00000001);
@@ -405,6 +430,11 @@ public Builder clearVisibility() {
 
             /**
              * required string visibility = 1;
+             *
+             * @param value
+             *            visibility byte string
+             *
+             * @return builder to set the visibility bytes
              */
             public Builder setVisibilityBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -432,16 +462,22 @@ public interface AttributeOrBuilder extends com.google.protobuf.MessageLiteOrBui
         // required string name = 1;
         /**
          * required string name = 1;
+         *
+         * @return if it has a name
          */
         boolean hasName();
 
         /**
          * required string name = 1;
+         *
+         * @return the name
          */
         java.lang.String getName();
 
         /**
          * required string name = 1;
+         *
+         * @return the name as a byte string
          */
         com.google.protobuf.ByteString getNameBytes();
     }
@@ -521,6 +557,8 @@ public com.google.protobuf.Parser getParserForType() {
 
         /**
          * required string name = 1;
+         *
+         * @return if it has a name
          */
         public boolean hasName() {
             return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -528,6 +566,8 @@ public boolean hasName() {
 
         /**
          * required string name = 1;
+         *
+         * @return the name
          */
         public java.lang.String getName() {
             java.lang.Object ref = name_;
@@ -545,6 +585,8 @@ public java.lang.String getName() {
 
         /**
          * required string name = 1;
+         *
+         * @return the name as a byte string
          */
         public com.google.protobuf.ByteString getNameBytes() {
             java.lang.Object ref = name_;
@@ -765,6 +807,8 @@ public Builder mergeFrom(com.google.protobuf.CodedInputStream input, com.google.
 
             /**
              * required string name = 1;
+             *
+             * @return if it has a name
              */
             public boolean hasName() {
                 return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -772,6 +816,8 @@ public boolean hasName() {
 
             /**
              * required string name = 1;
+             *
+             * @return the name
              */
             public java.lang.String getName() {
                 java.lang.Object ref = name_;
@@ -786,6 +832,8 @@ public java.lang.String getName() {
 
             /**
              * required string name = 1;
+             *
+             * @return the name as a byte string
              */
             public com.google.protobuf.ByteString getNameBytes() {
                 java.lang.Object ref = name_;
@@ -800,6 +848,11 @@ public com.google.protobuf.ByteString getNameBytes() {
 
             /**
              * required string name = 1;
+             *
+             * @param value
+             *            the new name value
+             *
+             * @return builder to set the name
              */
             public Builder setName(java.lang.String value) {
                 if (value == null) {
@@ -813,6 +866,8 @@ public Builder setName(java.lang.String value) {
 
             /**
              * required string name = 1;
+             *
+             * @return builder to clear the name
              */
             public Builder clearName() {
                 bitField0_ = (bitField0_ & ~0x00000001);
@@ -823,6 +878,11 @@ public Builder clearName() {
 
             /**
              * required string name = 1;
+             *
+             * @param value
+             *            the name byte string
+             *
+             * @return builder to set the name to the provided byte string
              */
             public Builder setNameBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -850,48 +910,73 @@ public interface DescriptionOrBuilder extends com.google.protobuf.MessageLiteOrB
         // required string view = 1;
         /**
          * required string view = 1;
+         *
+         * @return if it has a view
          */
         boolean hasView();
 
         /**
          * required string view = 1;
+         *
+         * @return the view
          */
         java.lang.String getView();
 
         /**
          * required string view = 1;
+         *
+         * @return the view as a byte string
+         *
          */
         com.google.protobuf.ByteString getViewBytes();
 
         // repeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
         /**
          * repeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
+         *
+         * @return list of attributes
          */
         java.util.List getAttributesList();
 
         /**
          * repeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the attribute at the provided index
          */
         datawave.webservice.results.cached.result.CachedresultMessages.Attribute getAttributes(int index);
 
         /**
          * repeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
+         *
+         * @return the number of attributes
          */
         int getAttributesCount();
 
         // repeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
         /**
          * repeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
+         *
+         * @return the visibility list
          */
         java.util.List getVisibilitiesList();
 
         /**
          * repeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the visibility at the provided index
          */
         datawave.webservice.results.cached.result.CachedresultMessages.ColumnVisibility getVisibilities(int index);
 
         /**
          * repeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
+         *
+         * @return the number of visibility
          */
         int getVisibilitiesCount();
     }
@@ -995,6 +1080,8 @@ public com.google.protobuf.Parser getParserForType() {
 
         /**
          * required string view = 1;
+         *
+         * @return if there is a view
          */
         public boolean hasView() {
             return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -1002,6 +1089,8 @@ public boolean hasView() {
 
         /**
          * required string view = 1;
+         *
+         * @return the view
          */
         public java.lang.String getView() {
             java.lang.Object ref = view_;
@@ -1019,6 +1108,8 @@ public java.lang.String getView() {
 
         /**
          * required string view = 1;
+         *
+         * @return the view as a byte string
          */
         public com.google.protobuf.ByteString getViewBytes() {
             java.lang.Object ref = view_;
@@ -1037,6 +1128,8 @@ public com.google.protobuf.ByteString getViewBytes() {
 
         /**
          * repeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
+         *
+         * @return the attributes list
          */
         public java.util.List getAttributesList() {
             return attributes_;
@@ -1044,6 +1137,8 @@ public java.util.Listrepeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
+         *
+         * @return list of attributes
          */
         public java.util.List getAttributesOrBuilderList() {
             return attributes_;
@@ -1051,6 +1146,8 @@ public java.util.Listrepeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
+         *
+         * @return the number of attributes in the list
          */
         public int getAttributesCount() {
             return attributes_.size();
@@ -1058,6 +1155,11 @@ public int getAttributesCount() {
 
         /**
          * repeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the attribute at the provided index
          */
         public datawave.webservice.results.cached.result.CachedresultMessages.Attribute getAttributes(int index) {
             return attributes_.get(index);
@@ -1065,6 +1167,11 @@ public datawave.webservice.results.cached.result.CachedresultMessages.Attribute
 
         /**
          * repeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the attribute at the provided index
          */
         public datawave.webservice.results.cached.result.CachedresultMessages.AttributeOrBuilder getAttributesOrBuilder(int index) {
             return attributes_.get(index);
@@ -1076,6 +1183,8 @@ public datawave.webservice.results.cached.result.CachedresultMessages.AttributeO
 
         /**
          * repeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
+         *
+         * @return the visibility list
          */
         public java.util.List getVisibilitiesList() {
             return visibilities_;
@@ -1083,6 +1192,8 @@ public java.util.Listrepeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
+         *
+         * @return the visibility list
          */
         public java.util.List getVisibilitiesOrBuilderList() {
             return visibilities_;
@@ -1090,6 +1201,8 @@ public java.util.Listrepeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
+         *
+         * @return the number of visibilities in the list
          */
         public int getVisibilitiesCount() {
             return visibilities_.size();
@@ -1097,6 +1210,11 @@ public int getVisibilitiesCount() {
 
         /**
          * repeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
+         *
+         * @param index
+         *            the index
+         *
+         * @return the visibility at the provided index
          */
         public datawave.webservice.results.cached.result.CachedresultMessages.ColumnVisibility getVisibilities(int index) {
             return visibilities_.get(index);
@@ -1104,6 +1222,11 @@ public datawave.webservice.results.cached.result.CachedresultMessages.ColumnVisi
 
         /**
          * repeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
+         *
+         * @param index
+         *            the index
+         *
+         * @return visibility at the provided index
          */
         public datawave.webservice.results.cached.result.CachedresultMessages.ColumnVisibilityOrBuilder getVisibilitiesOrBuilder(int index) {
             return visibilities_.get(index);
@@ -1390,6 +1513,8 @@ public Builder mergeFrom(com.google.protobuf.CodedInputStream input, com.google.
 
             /**
              * required string view = 1;
+             *
+             * @return if it has a view
              */
             public boolean hasView() {
                 return ((bitField0_ & 0x00000001) == 0x00000001);
@@ -1397,6 +1522,8 @@ public boolean hasView() {
 
             /**
              * required string view = 1;
+             *
+             * @return the view
              */
             public java.lang.String getView() {
                 java.lang.Object ref = view_;
@@ -1411,6 +1538,8 @@ public java.lang.String getView() {
 
             /**
              * required string view = 1;
+             *
+             * @return the view as a byte string
              */
             public com.google.protobuf.ByteString getViewBytes() {
                 java.lang.Object ref = view_;
@@ -1425,6 +1554,11 @@ public com.google.protobuf.ByteString getViewBytes() {
 
             /**
              * required string view = 1;
+             *
+             * @param value
+             *            the new view
+             *
+             * @return builder to set the view to the provided value
              */
             public Builder setView(java.lang.String value) {
                 if (value == null) {
@@ -1438,6 +1572,8 @@ public Builder setView(java.lang.String value) {
 
             /**
              * required string view = 1;
+             *
+             * @return builder to clear the view
              */
             public Builder clearView() {
                 bitField0_ = (bitField0_ & ~0x00000001);
@@ -1448,6 +1584,11 @@ public Builder clearView() {
 
             /**
              * required string view = 1;
+             *
+             * @param value
+             *            the new view value
+             *
+             * @return builder to set view bytes to provided value
              */
             public Builder setViewBytes(com.google.protobuf.ByteString value) {
                 if (value == null) {
@@ -1471,6 +1612,8 @@ private void ensureAttributesIsMutable() {
 
             /**
              * repeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
+             *
+             * @return attributes list
              */
             public java.util.List getAttributesList() {
                 return java.util.Collections.unmodifiableList(attributes_);
@@ -1478,6 +1621,8 @@ public java.util.Listrepeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
+             *
+             * @return the number of attributes in the list
              */
             public int getAttributesCount() {
                 return attributes_.size();
@@ -1485,6 +1630,11 @@ public int getAttributesCount() {
 
             /**
              * repeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
+             *
+             * @param index
+             *            the index
+             *
+             * @return the attribute at the provided index
              */
             public datawave.webservice.results.cached.result.CachedresultMessages.Attribute getAttributes(int index) {
                 return attributes_.get(index);
@@ -1492,6 +1642,13 @@ public datawave.webservice.results.cached.result.CachedresultMessages.Attribute
 
             /**
              * repeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the new attribute
+             *
+             * @return builder to set the attribute at the provided index to the provided value
              */
             public Builder setAttributes(int index, datawave.webservice.results.cached.result.CachedresultMessages.Attribute value) {
                 if (value == null) {
@@ -1505,6 +1662,13 @@ public Builder setAttributes(int index, datawave.webservice.results.cached.resul
 
             /**
              * repeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
+             *
+             * @param index
+             *            the index
+             * @param builderForValue
+             *            the new attribute builder
+             *
+             * @return builder to set the attribute at the provided index to the provided value
              */
             public Builder setAttributes(int index, datawave.webservice.results.cached.result.CachedresultMessages.Attribute.Builder builderForValue) {
                 ensureAttributesIsMutable();
@@ -1515,6 +1679,11 @@ public Builder setAttributes(int index, datawave.webservice.results.cached.resul
 
             /**
              * repeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
+             *
+             * @param value
+             *            the new attribute
+             *
+             * @return builder to add the provided attribute
              */
             public Builder addAttributes(datawave.webservice.results.cached.result.CachedresultMessages.Attribute value) {
                 if (value == null) {
@@ -1528,6 +1697,13 @@ public Builder addAttributes(datawave.webservice.results.cached.result.Cachedres
 
             /**
              * repeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the attribute to add
+             *
+             * @return builder to add the provided attribute to the provided index
              */
             public Builder addAttributes(int index, datawave.webservice.results.cached.result.CachedresultMessages.Attribute value) {
                 if (value == null) {
@@ -1541,6 +1717,11 @@ public Builder addAttributes(int index, datawave.webservice.results.cached.resul
 
             /**
              * repeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
+             *
+             * @param builderForValue
+             *            builder to create attribute
+             *
+             * @return builder to add attribute provided
              */
             public Builder addAttributes(datawave.webservice.results.cached.result.CachedresultMessages.Attribute.Builder builderForValue) {
                 ensureAttributesIsMutable();
@@ -1551,6 +1732,13 @@ public Builder addAttributes(datawave.webservice.results.cached.result.Cachedres
 
             /**
              * repeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
+             *
+             * @param index
+             *            the index
+             * @param builderForValue
+             *            builder to create attribute
+             *
+             * @return builder to add attribute at provided index
              */
             public Builder addAttributes(int index, datawave.webservice.results.cached.result.CachedresultMessages.Attribute.Builder builderForValue) {
                 ensureAttributesIsMutable();
@@ -1561,6 +1749,11 @@ public Builder addAttributes(int index, datawave.webservice.results.cached.resul
 
             /**
              * repeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
+             *
+             * @param values
+             *            iterable of values to add
+             *
+             * @return builder to add iterable of attributes
              */
             public Builder addAllAttributes(java.lang.Iterable values) {
                 ensureAttributesIsMutable();
@@ -1571,6 +1764,8 @@ public Builder addAllAttributes(java.lang.Iterablerepeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
+             *
+             * @return builder to clear attributes
              */
             public Builder clearAttributes() {
                 attributes_ = java.util.Collections.emptyList();
@@ -1581,6 +1776,11 @@ public Builder clearAttributes() {
 
             /**
              * repeated .datawave.webservice.results.cached.result.Attribute attributes = 2;
+             *
+             * @param index
+             *            the index
+             *
+             * @return builder to remove attributes
              */
             public Builder removeAttributes(int index) {
                 ensureAttributesIsMutable();
@@ -1602,6 +1802,8 @@ private void ensureVisibilitiesIsMutable() {
 
             /**
              * repeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
+             *
+             * @return list of visibilities
              */
             public java.util.List getVisibilitiesList() {
                 return java.util.Collections.unmodifiableList(visibilities_);
@@ -1609,6 +1811,8 @@ public java.util.Listrepeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
+             *
+             * @return the number of visibilities in the list
              */
             public int getVisibilitiesCount() {
                 return visibilities_.size();
@@ -1616,6 +1820,11 @@ public int getVisibilitiesCount() {
 
             /**
              * repeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
+             *
+             * @param index
+             *            the index
+             *
+             * @return the visibility at the provided index
              */
             public datawave.webservice.results.cached.result.CachedresultMessages.ColumnVisibility getVisibilities(int index) {
                 return visibilities_.get(index);
@@ -1623,6 +1832,13 @@ public datawave.webservice.results.cached.result.CachedresultMessages.ColumnVisi
 
             /**
              * repeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the visibility to add
+             *
+             * @return builder to set the visibility at the provided index to the provided value
              */
             public Builder setVisibilities(int index, datawave.webservice.results.cached.result.CachedresultMessages.ColumnVisibility value) {
                 if (value == null) {
@@ -1636,6 +1852,13 @@ public Builder setVisibilities(int index, datawave.webservice.results.cached.res
 
             /**
              * repeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
+             *
+             * @param index
+             *            the index
+             * @param builderForValue
+             *            visibility builder
+             *
+             * @return builder to set the visibility at the provided index to the provided value
              */
             public Builder setVisibilities(int index, datawave.webservice.results.cached.result.CachedresultMessages.ColumnVisibility.Builder builderForValue) {
                 ensureVisibilitiesIsMutable();
@@ -1646,6 +1869,11 @@ public Builder setVisibilities(int index, datawave.webservice.results.cached.res
 
             /**
              * repeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
+             *
+             * @param value
+             *            visibility to add
+             *
+             * @return builder to add visibility
              */
             public Builder addVisibilities(datawave.webservice.results.cached.result.CachedresultMessages.ColumnVisibility value) {
                 if (value == null) {
@@ -1659,6 +1887,13 @@ public Builder addVisibilities(datawave.webservice.results.cached.result.Cachedr
 
             /**
              * repeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
+             *
+             * @param index
+             *            the index
+             * @param value
+             *            the visibility to add
+             *
+             * @return builder to add visibility
              */
             public Builder addVisibilities(int index, datawave.webservice.results.cached.result.CachedresultMessages.ColumnVisibility value) {
                 if (value == null) {
@@ -1672,6 +1907,11 @@ public Builder addVisibilities(int index, datawave.webservice.results.cached.res
 
             /**
              * repeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
+             *
+             * @param builderForValue
+             *            visibility builder
+             *
+             * @return builder to add visibility
              */
             public Builder addVisibilities(datawave.webservice.results.cached.result.CachedresultMessages.ColumnVisibility.Builder builderForValue) {
                 ensureVisibilitiesIsMutable();
@@ -1682,6 +1922,13 @@ public Builder addVisibilities(datawave.webservice.results.cached.result.Cachedr
 
             /**
              * repeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
+             *
+             * @param index
+             *            the index
+             * @param builderForValue
+             *            visibility builder
+             *
+             * @return builder to add visibility at provided index
              */
             public Builder addVisibilities(int index, datawave.webservice.results.cached.result.CachedresultMessages.ColumnVisibility.Builder builderForValue) {
                 ensureVisibilitiesIsMutable();
@@ -1692,6 +1939,11 @@ public Builder addVisibilities(int index, datawave.webservice.results.cached.res
 
             /**
              * repeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
+             *
+             * @param values
+             *            iterable of visilibities
+             *
+             * @return builder to add all visibilities provided
              */
             public Builder addAllVisibilities(
                             java.lang.Iterable values) {
@@ -1703,6 +1955,8 @@ public Builder addAllVisibilities(
 
             /**
              * repeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
+             *
+             * @return builder to clear visibilities
              */
             public Builder clearVisibilities() {
                 visibilities_ = java.util.Collections.emptyList();
@@ -1713,6 +1967,11 @@ public Builder clearVisibilities() {
 
             /**
              * repeated .datawave.webservice.results.cached.result.ColumnVisibility visibilities = 3;
+             *
+             * @param index
+             *            the index
+             *
+             * @return builder to remove the visibility at the proivided index
              */
             public Builder removeVisibilities(int index) {
                 ensureVisibilitiesIsMutable();
diff --git a/web-services/common-util/src/main/java/datawave/security/util/WSAuthorizationsUtil.java b/web-services/common-util/src/main/java/datawave/security/util/WSAuthorizationsUtil.java
index 2d2123bb149..de62b9bb600 100644
--- a/web-services/common-util/src/main/java/datawave/security/util/WSAuthorizationsUtil.java
+++ b/web-services/common-util/src/main/java/datawave/security/util/WSAuthorizationsUtil.java
@@ -17,6 +17,7 @@ public class WSAuthorizationsUtil extends AuthorizationsUtil {
      * Merge principals. This can be used to create a composite view of a principal when including remote systems
      *
      * @param principals
+     *            the principal users
      * @return The merge principal
      */
     public static DatawavePrincipal mergePrincipals(DatawavePrincipal... principals) {
diff --git a/web-services/map-reduce/src/main/java/datawave/webservice/mr/state/MapReduceStatePersisterBean.java b/web-services/map-reduce/src/main/java/datawave/webservice/mr/state/MapReduceStatePersisterBean.java
index 76928e2ff0d..8e46b6653ad 100644
--- a/web-services/map-reduce/src/main/java/datawave/webservice/mr/state/MapReduceStatePersisterBean.java
+++ b/web-services/map-reduce/src/main/java/datawave/webservice/mr/state/MapReduceStatePersisterBean.java
@@ -64,7 +64,7 @@
  *
  * MapReduce Table
  * 
    DateIndex
    Schema TypeUse
    
- * 
+ * 
  * 
  * 
  * 
@@ -117,7 +117,7 @@
  *
  * MapReduce Index
  * 
    MapReduceTable
    RowColF
    
- * 
+ * 
  * 
  * 
  * 
diff --git a/web-services/query/src/main/java/datawave/webservice/query/cache/CreatedQueryLogicCacheBean.java b/web-services/query/src/main/java/datawave/webservice/query/cache/CreatedQueryLogicCacheBean.java
index 8f9f16c022d..391e5646131 100644
--- a/web-services/query/src/main/java/datawave/webservice/query/cache/CreatedQueryLogicCacheBean.java
+++ b/web-services/query/src/main/java/datawave/webservice/query/cache/CreatedQueryLogicCacheBean.java
@@ -99,6 +99,7 @@ public boolean equals(Object o) {
      * @param logic
      *            the query logic
      * @param client
+     *            accumulo client
      * @return true if there was no previous mapping for the given queryId in the cache.
      */
     public boolean add(String queryId, String userId, QueryLogic logic, AccumuloClient client) {
diff --git a/web-services/query/src/main/java/datawave/webservice/query/logic/QueryLogic.java b/web-services/query/src/main/java/datawave/webservice/query/logic/QueryLogic.java
index 872fbf2e519..f7c5cccae8b 100644
--- a/web-services/query/src/main/java/datawave/webservice/query/logic/QueryLogic.java
+++ b/web-services/query/src/main/java/datawave/webservice/query/logic/QueryLogic.java
@@ -439,7 +439,9 @@ default long getResultLimit(Query settings) {
      * it to filter this call as well. Most query logics will not implement this.
      *
      * @param settings
+     *            query settings
      * @param userAuthorizations
+     *            a set of user authorizations
      */
     default void preInitialize(Query settings, Set userAuthorizations) {
         // noop
diff --git a/web-services/query/src/main/java/datawave/webservice/query/metric/QueryMetricsWriter.java b/web-services/query/src/main/java/datawave/webservice/query/metric/QueryMetricsWriter.java
index e011bb4a589..a77b9f7f7fc 100644
--- a/web-services/query/src/main/java/datawave/webservice/query/metric/QueryMetricsWriter.java
+++ b/web-services/query/src/main/java/datawave/webservice/query/metric/QueryMetricsWriter.java
@@ -183,8 +183,10 @@ public void addMetricToQueue(QueryMetricHolder queryMetricHolder) {
      * Poll the blocking queue for new query metric updates until the number of updates reached batchSize or maxLatency is reached.
      *
      * @param batchSize
+     *            the size of the batch
      * @param maxLatency
-     * @return
+     *            maximum latency
+     * @return list of query metric holders
      */
     private List getMetricsFromQueue(int batchSize, long maxLatency) {
         List metricHolderList = new ArrayList<>();
@@ -322,6 +324,7 @@ public void run() {
          * Process query metric updates using either the RemoteQueryMetricService or the ShardTableQueryMetricHandler
          *
          * @param metricHolderList
+         *            query metric updates
          */
         private void processQueryMetrics(List metricHolderList) {
             if (!metricHolderList.isEmpty()) {
@@ -337,6 +340,7 @@ private void processQueryMetrics(List metricHolderList) {
          * Process query metric updates using the RemoteQueryMetricService
          *
          * @param metricHolderList
+         *            query metric updates
          */
         private void processQueryMetricsWithRemoteService(List metricHolderList) {
             List metricList = metricHolderList.stream().map(QueryMetricHolder::getQueryMetric).collect(Collectors.toList());
@@ -358,6 +362,7 @@ private void processQueryMetricsWithRemoteService(List metric
          * Process query metric updates using the ShardTableQueryMetricHandler
          *
          * @param metricHolderList
+         *            query metric updates
          */
         private void processQueryMetricsWithHandler(List metricHolderList) {
             List currentFailures = new ArrayList<>();
@@ -392,7 +397,8 @@ private void processQueryMetricsWithHandler(List metricHolder
          * Attempt to send metrics that previously failed to send
          *
          * @param failedMetrics
-         * @return
+         *            list of metrics that failed
+         * @return if successfully wrote failed metrics
          */
         private boolean writeFailedMetrics(List failedMetrics) {
             Iterator itr = failedMetrics.iterator();
@@ -426,6 +432,7 @@ private boolean writeFailedMetrics(List failedMetrics) {
          * Determine if we should discard failed query metric updates or keep retrying
          *
          * @param anySuccessful
+         *            the any successful
          */
         private void processFailedMetricList(boolean anySuccessful) {
             long discardForFailureCount = 0;
@@ -467,7 +474,9 @@ private void processFailedMetricList(boolean anySuccessful) {
          * Wraps the sending of query metric updates to the RemoteQueryMetricService Failure is indicated by throwing an Exception
          *
          * @param updatedMetrics
+         *            updated metrics
          * @throws Exception
+         *             when probelms arise
          */
         private void writeMetricsToRemoteService(List updatedMetrics) throws Exception {
             if (!updatedMetrics.isEmpty()) {
@@ -483,8 +492,10 @@ private void writeMetricsToRemoteService(List updatedMetrics) t
          * Wraps the sending of query metric updates to the ShardTableQueryMetricHandler Failure is indicated by returning a list of failed query metric updates
          *
          * @param queryMetricHandler
+         *            metric handler
          * @param metricQueue
-         * @return
+         *            list of metrics to process
+         * @return list of failed query metrics
          */
         private List writeMetricsToHandler(QueryMetricHandler queryMetricHandler, List metricQueue) {
             List failedMetrics = new ArrayList<>();
@@ -511,6 +522,7 @@ private List writeMetricsToHandler(QueryMetricHandler queryMe
          * service handles this for the path that uses the RemoteQueryMetricService
          *
          * @param queryMetric
+         *            query metric
          */
         private synchronized void sendMetricsToTimely(BaseQueryMetric queryMetric) {
 
diff --git a/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java b/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java
index 07d21f100cf..47df0e00ff7 100644
--- a/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java
+++ b/web-services/query/src/main/java/datawave/webservice/query/runner/QueryExecutorBean.java
@@ -1165,7 +1165,6 @@ private RunningQuery adminGetQueryById(String id) throws Exception {
      *            the ID of the query to reload/reset
      * @return an empty response
      *
-     * @return datawave.webservice.result.VoidResponse
      * @RequestHeader X-ProxiedEntitiesChain use when proxying request for user, by specifying a chain of DNs of the identities to proxy
      * @RequestHeader X-ProxiedIssuersChain required when using X-ProxiedEntitiesChain, specify one issuer DN per subject DN listed in X-ProxiedEntitiesChain
      * @ResponseHeader query-session-id this header and value will be in the Set-Cookie header, subsequent calls for this session will need to supply the
diff --git a/web-services/security/src/main/java/datawave/security/authorization/simple/DatabaseUserService.java b/web-services/security/src/main/java/datawave/security/authorization/simple/DatabaseUserService.java
index 26fe5cf6807..d84a07f37a8 100644
--- a/web-services/security/src/main/java/datawave/security/authorization/simple/DatabaseUserService.java
+++ b/web-services/security/src/main/java/datawave/security/authorization/simple/DatabaseUserService.java
@@ -34,7 +34,7 @@
  * contain two tables: users (name is customizable by setting the dw.databaseUsersService.usersTableName property) and roleToAuthMapping (name is customizable
  * by setting the dw.databaseUsersService.mappingTableName property). The expected structure of the users table is:
  * 
    MapReduceIndex
    RowColF
    
- * 
+ * 
  * 
  * 
  * 
@@ -70,7 +70,7 @@
  * The roleToAuthMapping table contains the mappings of roles seen in the roles column of the users table into Accumulo auths that appear in the auths column of
  * the users table. The expected structure of this table is:
  * 
    User data table
    Column NameColumn Type
    
- * 
+ * 
  * 
  * 
  * 

From 4a92fa8cc36664fea7447dbe4b613df1ed3ee795 Mon Sep 17 00:00:00 2001
From: Ivan Bella 
Date: Wed, 14 Feb 2024 16:16:54 +0000
Subject: [PATCH 15/50] puller version into sync with the rest of datawave

---
 microservices/configcheck/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/microservices/configcheck/pom.xml b/microservices/configcheck/pom.xml
index 903efa54929..83cb77aca80 100644
--- a/microservices/configcheck/pom.xml
+++ b/microservices/configcheck/pom.xml
@@ -3,7 +3,7 @@
     4.0.0
     gov.nsa.datawave.microservice
     datawave-microservice-configcheck
-    1.0.0-SNAPSHOT
+    6.7.0-SNAPSHOT
     ${project.artifactId}
     
         

From 6fbd22292b6446f9afa6c448180917fcb9eae6f6 Mon Sep 17 00:00:00 2001
From: Christopher Gross 
Date: Wed, 14 Feb 2024 11:55:36 -0500
Subject: [PATCH 16/50] 2170 - update dependencies (#2176)

* 2170 - update dependencies
---------
Co-authored-by: Christopher Gross cogross 
---
 pom.xml                         | 17 ++++++++++-------
 warehouse/pom.xml               |  6 +++---
 web-services/client/pom.xml     |  2 +-
 web-services/map-reduce/pom.xml |  2 +-
 web-services/pom.xml            |  2 +-
 5 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/pom.xml b/pom.xml
index 66b51533c93..351566a582c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -42,7 +42,7 @@
         1.4.1.Final
         1.0.0.Final
         3.20.2
-        1.8.2
+        1.11.3
         3.1.0
         1.9.4
         1.4
@@ -50,7 +50,7 @@
         3.2.2
         4.3
         1.10
-        2.7
+        2.8.0
         2.6
         3.3
         2.6
@@ -61,7 +61,7 @@
         1.9.0
         4.0.2
         2.15.0
-        2.22.1
+        2.22.5
         28.1
         1.2.0
         31.1-jre
@@ -74,6 +74,7 @@
         3.0.1
         9.4.21.Final
         2.10.0.pr1
+        1.9.13
         3.24.0-GA
         3.1.0
         2.0.1.Final
@@ -81,10 +82,12 @@
         6.1.26
         4.0.19.Final
         0.11.2
+        20231013
         1.19.0
         4.13.2
         5.5.2
         5.5.2
+        2.7.2b2
         2.20
         2.20
         2.17.2
@@ -104,12 +107,12 @@
         2.0.1
         1.2
         2.23.0
-        8.0.16
+        8.0.28
         4.1.42.Final
         2.1
         5.0.3.Final
         2.0.9
-        3.7.1
+        3.16.3
         1.6.2
         1.7.36
         3.1.5
@@ -123,10 +126,10 @@
         
         2.3.5.Final
         17.0.1.Final
-        5.0.3
+        5.4.0
         3.1.4
         2.12.2
-        3.8.1
+        3.8.3
         
     
     
diff --git a/warehouse/pom.xml b/warehouse/pom.xml
index f08999e6bf3..abf4fff5a30 100644
--- a/warehouse/pom.xml
+++ b/warehouse/pom.xml
@@ -338,7 +338,7 @@
             
                 org.codehaus.jackson
                 jackson-mapper-asl
-                1.9.13
+                ${version.jackson-mapper-asl}
             
             
                 org.fusesource.jdbm
@@ -358,7 +358,7 @@
             
                 org.json
                 json
-                20080701
+                ${version.json}
             
             
                 org.mortbay.jetty
@@ -373,7 +373,7 @@
             
                 org.python
                 jython
-                2.5.2b2-callable
+                ${version.jython}
             
             
                 org.slf4j
diff --git a/web-services/client/pom.xml b/web-services/client/pom.xml
index d24e5838ec9..587d5f818f7 100644
--- a/web-services/client/pom.xml
+++ b/web-services/client/pom.xml
@@ -37,7 +37,7 @@
         
             commons-net
             commons-net
-            3.1
+            ${version.commons-net}
         
         
             gov.nsa.datawave
diff --git a/web-services/map-reduce/pom.xml b/web-services/map-reduce/pom.xml
index 5ca9823d953..e61c1bd1304 100644
--- a/web-services/map-reduce/pom.xml
+++ b/web-services/map-reduce/pom.xml
@@ -161,7 +161,7 @@
             
             org.codehaus.jackson
             jackson-mapper-asl
-            1.9.13
+            ${version.jackson-mapper-asl}
             runtime
         
         
diff --git a/web-services/pom.xml b/web-services/pom.xml
index c877836bfde..e1b5614829d 100644
--- a/web-services/pom.xml
+++ b/web-services/pom.xml
@@ -44,7 +44,7 @@
         1.21
         1.7
         3.2
-        3.6
+        3.9.0
         2.6.1
         1.1
         1.0.1

From 2f370d372c89db9ea569664d28d3c0ce8e0fe5b5 Mon Sep 17 00:00:00 2001
From: Christopher Gross cogross 
Date: Thu, 2 Nov 2023 13:08:04 +0000
Subject: [PATCH 17/50] 2013 - jexl uuid query parser research/changes

---
 .../query/language/parser/jexl/LuceneToJexlUUIDQueryParser.java  | 1 -
 .../datawave/query/language/parser/lucene/LuceneQueryParser.java | 1 -
 2 files changed, 2 deletions(-)

diff --git a/warehouse/query-core/src/main/java/datawave/query/language/parser/jexl/LuceneToJexlUUIDQueryParser.java b/warehouse/query-core/src/main/java/datawave/query/language/parser/jexl/LuceneToJexlUUIDQueryParser.java
index f741ce5c063..f10b29fc71e 100644
--- a/warehouse/query-core/src/main/java/datawave/query/language/parser/jexl/LuceneToJexlUUIDQueryParser.java
+++ b/warehouse/query-core/src/main/java/datawave/query/language/parser/jexl/LuceneToJexlUUIDQueryParser.java
@@ -12,7 +12,6 @@
 import datawave.query.search.RangeFieldedTerm;
 import datawave.query.search.WildcardFieldedTerm;
 
-@Deprecated
 public class LuceneToJexlUUIDQueryParser extends LuceneToJexlQueryParser {
     private List uuidTypes = new ArrayList<>();
     private LuceneQueryParser luceneParser = new LuceneQueryParser();
diff --git a/warehouse/query-core/src/main/java/datawave/query/language/parser/lucene/LuceneQueryParser.java b/warehouse/query-core/src/main/java/datawave/query/language/parser/lucene/LuceneQueryParser.java
index 3fd61311084..46c9d2a41d5 100644
--- a/warehouse/query-core/src/main/java/datawave/query/language/parser/lucene/LuceneQueryParser.java
+++ b/warehouse/query-core/src/main/java/datawave/query/language/parser/lucene/LuceneQueryParser.java
@@ -30,7 +30,6 @@
 import datawave.query.search.RangeFieldedTerm;
 import datawave.query.search.Term;
 
-@Deprecated
 public class LuceneQueryParser implements QueryParser {
     private static Logger log = Logger.getLogger(LuceneQueryParser.class.getName());
     private Map filters = new HashMap<>();

From 2a269b31cf4381731b77899b0d10465da8148010 Mon Sep 17 00:00:00 2001
From: Drew Farris 
Date: Wed, 14 Feb 2024 16:39:07 -0500
Subject: [PATCH 18/50] Improved SSDeep ingest and query test framework (#2230)

* Replaced custom Tuple2 implementation with ImmutablePair from commons-lang3
* Includes complete ingest testframework for SSDeep data that includes both shard data and ssdeepIndex data.
* Cleans up some sins of the past related to the faceted query logic leaking into the test framework.
* Adds protoype datawave ingest SSDeepIndexHandler, good enough for testing at this point.
* Trimmed pom for ssdeep-ingest to minimum dependency set
* Reorganized package structure to keep ssdeep code/tests together
* Moved SSDeepQueryTest to common package
* Fix parent pom version for ingest-ssdeep and static version for ssdeep-common
* Addressed comments from code review
---
 warehouse/ingest-ssdeep/pom.xml               |  61 ++++
 .../handler/ssdeep/SSDeepIndexHandler.java    | 178 ++++++++++++
 .../config/SSDeepIndexTableConfigHelper.java  |  56 ++++
 warehouse/pom.xml                             |   1 +
 warehouse/query-core/pom.xml                  |   6 +
 .../SSDeepSimilarityQueryConfiguration.java   |  21 +-
 .../SSDeepSimilarityQueryLogic.java           |   9 +-
 .../SSDeepSimilarityQueryTransformer.java     |   2 +-
 .../datawave/query/QueryTestTableHelper.java  |  63 ++---
 .../facets/FacetQueryTestTableHelper.java     |  63 +++++
 .../tables/facets/FacetedCitiesDataType.java  |   8 +-
 .../tables/facets/FacetedQueryLogicTest.java  |  16 +-
 .../tables/ssdeep/SSDeepIngestQueryTest.java  | 174 ++++++++++++
 .../{ => tables/ssdeep}/SSDeepQueryTest.java  |  16 +-
 .../ssdeep/SSDeepSimilarityQueryTest.java     | 162 +++++++++++
 .../SSDeepSimilarityQueryTransformerTest.java |   5 +-
 .../testframework/SSDeepDataManager.java      |  72 +++++
 .../ssdeep/testframework/SSDeepDataType.java  | 266 ++++++++++++++++++
 .../ssdeep/testframework/SSDeepFields.java    |  40 +++
 .../SSDeepQueryTestTableHelper.java           |  65 +++++
 .../tables/ssdeep/util/SSDeepTestUtil.java    | 158 +++++++++++
 .../query/testframework/AccumuloSetup.java    |  50 +++-
 .../query/testframework/BaseRawData.java      |  16 +-
 .../src/test/resources/input/ssdeep-data.csv  |  14 +
 .../datawave/util/ssdeep/NGramGenerator.java  |  17 +-
 .../java/datawave/util/ssdeep/SSDeepHash.java |   5 +-
 .../util/ssdeep/NGramGeneratorTest.java       |   2 +-
 .../query/SSDeepQueryLogicFactory.xml         |   2 +-
 28 files changed, 1459 insertions(+), 89 deletions(-)
 create mode 100644 warehouse/ingest-ssdeep/pom.xml
 create mode 100644 warehouse/ingest-ssdeep/src/main/java/datawave/ingest/mapreduce/handler/ssdeep/SSDeepIndexHandler.java
 create mode 100644 warehouse/ingest-ssdeep/src/main/java/datawave/ingest/table/config/SSDeepIndexTableConfigHelper.java
 rename warehouse/query-core/src/main/java/datawave/query/tables/{ => ssdeep}/SSDeepSimilarityQueryLogic.java (96%)
 rename warehouse/query-core/src/main/java/datawave/query/{transformer => tables/ssdeep}/SSDeepSimilarityQueryTransformer.java (99%)
 create mode 100644 warehouse/query-core/src/test/java/datawave/query/tables/facets/FacetQueryTestTableHelper.java
 create mode 100644 warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java
 rename warehouse/query-core/src/test/java/datawave/query/{ => tables/ssdeep}/SSDeepQueryTest.java (96%)
 create mode 100644 warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTest.java
 rename warehouse/query-core/src/test/java/datawave/query/{transformer => tables/ssdeep}/SSDeepSimilarityQueryTransformerTest.java (96%)
 create mode 100644 warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepDataManager.java
 create mode 100644 warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepDataType.java
 create mode 100644 warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepFields.java
 create mode 100644 warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepQueryTestTableHelper.java
 create mode 100644 warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/util/SSDeepTestUtil.java
 create mode 100644 warehouse/query-core/src/test/resources/input/ssdeep-data.csv

diff --git a/warehouse/ingest-ssdeep/pom.xml b/warehouse/ingest-ssdeep/pom.xml
new file mode 100644
index 00000000000..c4774015a00
--- /dev/null
+++ b/warehouse/ingest-ssdeep/pom.xml
@@ -0,0 +1,61 @@
+
+
+    4.0.0
+    
+        gov.nsa.datawave
+        datawave-warehouse-parent
+        6.7.0-SNAPSHOT
+    
+    datawave-ingest-ssdeep
+    
+        11
+        11
+        UTF-8
+    
+    
+        
+            com.google.guava
+            guava
+        
+        
+            gov.nsa.datawave
+            datawave-core
+            ${project.version}
+        
+        
+            gov.nsa.datawave
+            datawave-ingest-core
+            ${project.version}
+        
+        
+            gov.nsa.datawave
+            datawave-ssdeep-common
+            ${project.version}
+        
+        
+            gov.nsa.datawave.microservice
+            accumulo-utils
+        
+        
+            org.apache.accumulo
+            accumulo-core
+        
+        
+            org.apache.hadoop
+            hadoop-client-api
+        
+        
+            org.apache.hadoop
+            hadoop-mapreduce-client-core
+        
+        
+            org.apache.logging.log4j
+            log4j-1.2-api
+        
+        
+            junit
+            junit
+            test
+        
+    
+
diff --git a/warehouse/ingest-ssdeep/src/main/java/datawave/ingest/mapreduce/handler/ssdeep/SSDeepIndexHandler.java b/warehouse/ingest-ssdeep/src/main/java/datawave/ingest/mapreduce/handler/ssdeep/SSDeepIndexHandler.java
new file mode 100644
index 00000000000..b4d3a1aa60d
--- /dev/null
+++ b/warehouse/ingest-ssdeep/src/main/java/datawave/ingest/mapreduce/handler/ssdeep/SSDeepIndexHandler.java
@@ -0,0 +1,178 @@
+package datawave.ingest.mapreduce.handler.ssdeep;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.StatusReporter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskInputOutputContext;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
+
+import datawave.ingest.data.RawRecordContainer;
+import datawave.ingest.data.Type;
+import datawave.ingest.data.TypeRegistry;
+import datawave.ingest.data.config.ConfigurationHelper;
+import datawave.ingest.data.config.DataTypeHelper;
+import datawave.ingest.data.config.NormalizedContentInterface;
+import datawave.ingest.data.config.ingest.IngestHelperInterface;
+import datawave.ingest.mapreduce.handler.ExtendedDataTypeHandler;
+import datawave.ingest.mapreduce.job.BulkIngestKey;
+import datawave.ingest.mapreduce.job.writer.ContextWriter;
+import datawave.ingest.metadata.RawRecordMetadata;
+import datawave.marking.MarkingFunctions;
+import datawave.util.ssdeep.BucketAccumuloKeyGenerator;
+import datawave.util.ssdeep.NGramByteHashGenerator;
+import datawave.util.ssdeep.NGramGenerator;
+import datawave.util.ssdeep.NGramTuple;
+
+public class SSDeepIndexHandler implements ExtendedDataTypeHandler {
+
+    /* Global configuration properties */
+
+    public static final String SSDEEP_INDEX_TABLE_NAME = "ssdeepIndex.table.name";
+    public static final String SSDEEP_INDEX_TABLE_LOADER_PRIORITY = "ssdeepIndex.table.loader.priority";
+    public static final String SSDEEP_FIELD_SET = ".ssdeepIndex.fields";
+
+    public static final String SSDEEP_BUCKET_COUNT = ".ssdeepIndex.bucket.count";
+
+    public static final int DEFAULT_SSDEEP_BUCKET_COUNT = BucketAccumuloKeyGenerator.DEFAULT_BUCKET_COUNT;
+
+    public static final String SSDEEP_BUCKET_ENCODING_BASE = ".ssdeepIndex.bucket.encoding.base";
+
+    public static final int DEFAULT_SSDEEP_BUCKET_ENCODING_BASE = BucketAccumuloKeyGenerator.DEFAULT_BUCKET_ENCODING_BASE;
+
+    public static final String SSDEEP_BUCKET_ENCODING_LENGTH = ".ssdeepIndex.bucket.encoding.length";
+
+    public static final int DEFAULT_SSDEEP_BUCKET_ENCODING_LENGTH = BucketAccumuloKeyGenerator.DEFAULT_BUCKET_ENCODING_LENGTH;
+
+    public static final String SSDEEP_INDEX_NGRAM_SIZE = ".ssdeepIndex.ngram.size.min";
+
+    public static final int DEFAULT_SSDEEP_INDEX_NGRAM_SIZE = NGramGenerator.DEFAULT_NGRAM_SIZE;
+
+    public static final String SSDEEP_MIN_HASH_SIZE = ".ssdeepIndex.chunk.size.min";
+
+    public static final int DEFAULT_SSDEEP_MIN_HASH_SIZE = NGramGenerator.DEFAULT_MIN_HASH_SIZE;
+
+    /** The priority of 40 is arbitrary, based on setting other priorities - this controls the bulk loading process */
+    public static final int DEFAULT_SSDEEP_INDEX_TABLE_LOADER_PRIORITY = 40;
+    public static final String DEFAULT_SSDEEP_INDEX_TABLE_NAME = "ssdeepIndex";
+
+    protected Text ssdeepIndexTableName;
+
+    protected MarkingFunctions markingFunctions;
+
+    protected TaskAttemptContext taskAttemptContext;
+
+    protected Set ssdeepFieldNames;
+
+    protected NGramByteHashGenerator nGramGenerator;
+    protected BucketAccumuloKeyGenerator accumuloKeyGenerator;
+
+    @Override
+    public void setup(TaskAttemptContext context) {
+        markingFunctions = MarkingFunctions.Factory.createMarkingFunctions();
+        taskAttemptContext = context;
+
+        Configuration conf = context.getConfiguration();
+
+        final String dataType = ConfigurationHelper.isNull(conf, DataTypeHelper.Properties.DATA_NAME, String.class);
+        TypeRegistry.getInstance(conf);
+        Type type = TypeRegistry.getType(dataType);
+
+        int bucketCount = conf.getInt(type.typeName() + SSDEEP_BUCKET_COUNT, DEFAULT_SSDEEP_BUCKET_COUNT);
+        int bucketEncodingBase = conf.getInt(type.typeName() + SSDEEP_BUCKET_ENCODING_BASE, DEFAULT_SSDEEP_BUCKET_ENCODING_BASE);
+        int bucketEncodingLength = conf.getInt(type.typeName() + SSDEEP_BUCKET_ENCODING_LENGTH, DEFAULT_SSDEEP_BUCKET_ENCODING_LENGTH);
+        int ngramSize = conf.getInt(type.typeName() + SSDEEP_INDEX_NGRAM_SIZE, DEFAULT_SSDEEP_INDEX_NGRAM_SIZE);
+        int minHashSize = conf.getInt(type.typeName() + SSDEEP_MIN_HASH_SIZE, DEFAULT_SSDEEP_MIN_HASH_SIZE);
+
+        accumuloKeyGenerator = new BucketAccumuloKeyGenerator(bucketCount, bucketEncodingBase, bucketEncodingLength);
+        nGramGenerator = new NGramByteHashGenerator(ngramSize, bucketCount, minHashSize);
+
+        String[] fieldNameArray = conf.getStrings(type.typeName() + SSDEEP_FIELD_SET);
+        ssdeepFieldNames = new HashSet<>(List.of(fieldNameArray));
+        ssdeepIndexTableName = new Text(ConfigurationHelper.isNull(conf, SSDEEP_INDEX_TABLE_NAME, String.class));
+    }
+
+    @Override
+    public String[] getTableNames(Configuration conf) {
+        return new String[] {conf.get(SSDEEP_INDEX_TABLE_NAME, DEFAULT_SSDEEP_INDEX_TABLE_NAME)};
+    }
+
+    @Override
+    public int[] getTableLoaderPriorities(Configuration conf) {
+        return new int[] {conf.getInt(SSDEEP_INDEX_TABLE_LOADER_PRIORITY, DEFAULT_SSDEEP_INDEX_TABLE_LOADER_PRIORITY)};
+    }
+
+    @Override
+    public Multimap processBulk(KEYIN key, RawRecordContainer event, Multimap fields,
+                    StatusReporter reporter) {
+        throw new UnsupportedOperationException("processBulk is not supported, please use process");
+    }
+
+    @Override
+    public IngestHelperInterface getHelper(Type datatype) {
+        return datatype.getIngestHelper(this.taskAttemptContext.getConfiguration());
+    }
+
+    @Override
+    public void close(TaskAttemptContext context) {
+        /* no-op */
+    }
+
+    @Override
+    public RawRecordMetadata getMetadata() {
+        return null;
+    }
+
+    protected byte[] flatten(ColumnVisibility vis) {
+        return markingFunctions == null ? vis.flatten() : markingFunctions.flatten(vis);
+    }
+
+    @Override
+    public long process(KEYIN key, RawRecordContainer event, Multimap fields,
+                    TaskInputOutputContext context, ContextWriter contextWriter)
+                    throws IOException, InterruptedException {
+
+        final Set eventFieldSet = fields.keySet();
+        long countWritten = 0;
+        final Multimap results = ArrayListMultimap.create();
+
+        for (String ssdeepFieldName : ssdeepFieldNames) {
+            if (!eventFieldSet.contains(ssdeepFieldName))
+                continue;
+
+            for (NormalizedContentInterface ssdeepTypes : fields.get(ssdeepFieldName)) {
+                countWritten += generateSSDeepIndexEntries(ssdeepTypes.getEventFieldValue(), results);
+            }
+        }
+
+        contextWriter.write(results, context);
+        return countWritten;
+    }
+
+    public long generateSSDeepIndexEntries(String fieldValue, Multimap results) {
+        long countWritten = 0L;
+        Iterator> it = nGramGenerator.call(fieldValue);
+        while (it.hasNext()) {
+            ImmutablePair nt = it.next();
+            ImmutablePair at = accumuloKeyGenerator.call(nt);
+            BulkIngestKey indexKey = new BulkIngestKey(ssdeepIndexTableName, at.getKey());
+            results.put(indexKey, at.getValue());
+            countWritten++;
+        }
+        return countWritten;
+    }
+}
diff --git a/warehouse/ingest-ssdeep/src/main/java/datawave/ingest/table/config/SSDeepIndexTableConfigHelper.java b/warehouse/ingest-ssdeep/src/main/java/datawave/ingest/table/config/SSDeepIndexTableConfigHelper.java
new file mode 100644
index 00000000000..bb50c0ad8d9
--- /dev/null
+++ b/warehouse/ingest-ssdeep/src/main/java/datawave/ingest/table/config/SSDeepIndexTableConfigHelper.java
@@ -0,0 +1,56 @@
+package datawave.ingest.table.config;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Logger;
+
+import datawave.ingest.mapreduce.handler.ssdeep.SSDeepIndexHandler;
+
+public class SSDeepIndexTableConfigHelper extends AbstractTableConfigHelper {
+
+    protected Logger log;
+
+    public enum SSDeepTableType {
+        INDEX
+    }
+
+    protected Configuration conf;
+    protected String tableName;
+    protected String ssdeepIndexTableName;
+    protected SSDeepTableType tableType;
+
+    @Override
+    public void setup(String tableName, Configuration config, Logger log) throws IllegalArgumentException {
+        this.log = log;
+        this.conf = config;
+
+        ssdeepIndexTableName = conf.get(SSDeepIndexHandler.SSDEEP_INDEX_TABLE_NAME, SSDeepIndexHandler.DEFAULT_SSDEEP_INDEX_TABLE_NAME);
+
+        // TODO: generic balancer, markings, bloom filters, locality groups.
+
+        if (tableName.equals(ssdeepIndexTableName)) {
+            this.tableType = SSDeepTableType.INDEX;
+        } else {
+            throw new IllegalArgumentException("Invalid SSDeepIndex Table Definition For: " + tableName);
+        }
+        this.tableName = tableName;
+    }
+
+    @Override
+    public void configure(TableOperations tops) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+        if (this.tableType == SSDeepTableType.INDEX) {
+            configureSSDeepIndexTable(tops);
+            return;
+        }
+
+        throw new TableNotFoundException(null, tableName, "Table is not a SSDeep Index Type Table");
+    }
+
+    @SuppressWarnings({"unused", "RedundantThrows"})
+    protected void configureSSDeepIndexTable(TableOperations tops) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+        /* currently a no-op for test use, will potentially need to implement this for production use */
+    }
+}
diff --git a/warehouse/pom.xml b/warehouse/pom.xml
index abf4fff5a30..f383da9e34e 100644
--- a/warehouse/pom.xml
+++ b/warehouse/pom.xml
@@ -30,6 +30,7 @@
         data-dictionary-core
         edge-model-configuration-core
         ssdeep-common
+        ingest-ssdeep
     
     
         scm:git:https://fixme/git/warehouse/
diff --git a/warehouse/query-core/pom.xml b/warehouse/query-core/pom.xml
index 78fc6678468..42a8e1d2c99 100644
--- a/warehouse/query-core/pom.xml
+++ b/warehouse/query-core/pom.xml
@@ -227,6 +227,12 @@
             ${project.version}
             test
         
+        
+            gov.nsa.datawave
+            datawave-ingest-ssdeep
+            ${project.version}
+            test
+        
         
             gov.nsa.datawave.webservices
             datawave-ws-cached-results
diff --git a/warehouse/query-core/src/main/java/datawave/query/config/SSDeepSimilarityQueryConfiguration.java b/warehouse/query-core/src/main/java/datawave/query/config/SSDeepSimilarityQueryConfiguration.java
index fe8997d6919..a2b91d56c2b 100644
--- a/warehouse/query-core/src/main/java/datawave/query/config/SSDeepSimilarityQueryConfiguration.java
+++ b/warehouse/query-core/src/main/java/datawave/query/config/SSDeepSimilarityQueryConfiguration.java
@@ -9,6 +9,7 @@
 import datawave.util.ssdeep.BucketAccumuloKeyGenerator;
 import datawave.util.ssdeep.ChunkSizeEncoding;
 import datawave.util.ssdeep.IntegerEncoding;
+import datawave.util.ssdeep.NGramGenerator;
 import datawave.util.ssdeep.NGramTuple;
 import datawave.util.ssdeep.SSDeepHash;
 import datawave.webservice.query.Query;
@@ -19,8 +20,10 @@
 public class SSDeepSimilarityQueryConfiguration extends GenericQueryConfiguration {
 
     int queryThreads = 100;
-    int maxRepeatedCharacters = 3;
 
+    int ngramSize = NGramGenerator.DEFAULT_NGRAM_SIZE;
+    int maxRepeatedCharacters = SSDeepHash.DEFAULT_MAX_REPEATED_CHARACTERS;
+    int minHashSize = NGramGenerator.DEFAULT_MIN_HASH_SIZE;
     int indexBuckets = BucketAccumuloKeyGenerator.DEFAULT_BUCKET_COUNT;
     int bucketEncodingBase = BucketAccumuloKeyGenerator.DEFAULT_BUCKET_ENCODING_BASE;
     int bucketEncodingLength = BucketAccumuloKeyGenerator.DEFAULT_BUCKET_ENCODING_LENGTH;
@@ -89,6 +92,14 @@ public void setQueryThreads(int queryThreads) {
         this.queryThreads = queryThreads;
     }
 
+    public int getNGramSize() {
+        return ngramSize;
+    }
+
+    public void setNGramSize(int ngramSize) {
+        this.ngramSize = ngramSize;
+    }
+
     public int getMaxRepeatedCharacters() {
         return maxRepeatedCharacters;
     }
@@ -97,6 +108,14 @@ public void setMaxRepeatedCharacters(int maxRepeatedCharacters) {
         this.maxRepeatedCharacters = maxRepeatedCharacters;
     }
 
+    public int getMinHashSize() {
+        return minHashSize;
+    }
+
+    public void setMinHashSize(int minHashSize) {
+        this.minHashSize = minHashSize;
+    }
+
     public int getBucketEncodingBase() {
         return bucketEncodingBase;
     }
diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/SSDeepSimilarityQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryLogic.java
similarity index 96%
rename from warehouse/query-core/src/main/java/datawave/query/tables/SSDeepSimilarityQueryLogic.java
rename to warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryLogic.java
index 61736ff85fd..ce20fb0c5f2 100644
--- a/warehouse/query-core/src/main/java/datawave/query/tables/SSDeepSimilarityQueryLogic.java
+++ b/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryLogic.java
@@ -1,4 +1,4 @@
-package datawave.query.tables;
+package datawave.query.tables.ssdeep;
 
 import java.util.Arrays;
 import java.util.Collections;
@@ -21,7 +21,7 @@
 import com.google.common.collect.Multimap;
 
 import datawave.query.config.SSDeepSimilarityQueryConfiguration;
-import datawave.query.transformer.SSDeepSimilarityQueryTransformer;
+import datawave.query.tables.ScannerFactory;
 import datawave.util.ssdeep.ChunkSizeEncoding;
 import datawave.util.ssdeep.IntegerEncoding;
 import datawave.util.ssdeep.NGramGenerator;
@@ -109,9 +109,9 @@ public void setupRanges(Query settings, SSDeepSimilarityQueryConfiguration confi
             return pos > 0 ? k.substring(pos + 1) : k;
         }).map(SSDeepHash::parse).collect(Collectors.toSet());
 
-        final NGramGenerator nGramEngine = new NGramGenerator(7);
-        log.info("Pre-processing " + queries.size() + " SSDeepHash queries");
         final int maxRepeatedCharacters = config.getMaxRepeatedCharacters();
+        final NGramGenerator nGramEngine = new NGramGenerator(config.getNGramSize(), maxRepeatedCharacters, config.getMinHashSize());
+        log.info("Pre-processing " + queries.size() + " SSDeepHash queries");
         if (maxRepeatedCharacters > 0) {
             log.info("Normalizing SSDeepHashes to remove long runs of consecutive characters");
             queries = queries.stream().map(h -> h.normalize(maxRepeatedCharacters)).collect(Collectors.toSet());
@@ -170,6 +170,7 @@ public AccumuloConnectionFactory.Priority getConnectionPriority() {
         return AccumuloConnectionFactory.Priority.NORMAL;
     }
 
+    @SuppressWarnings("rawtypes")
     @Override
     public QueryLogicTransformer getTransformer(Query settings) {
         final SSDeepSimilarityQueryConfiguration config = getConfig();
diff --git a/warehouse/query-core/src/main/java/datawave/query/transformer/SSDeepSimilarityQueryTransformer.java b/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformer.java
similarity index 99%
rename from warehouse/query-core/src/main/java/datawave/query/transformer/SSDeepSimilarityQueryTransformer.java
rename to warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformer.java
index 5a0ab39ea1a..d400e46b733 100644
--- a/warehouse/query-core/src/main/java/datawave/query/transformer/SSDeepSimilarityQueryTransformer.java
+++ b/warehouse/query-core/src/main/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformer.java
@@ -1,4 +1,4 @@
-package datawave.query.transformer;
+package datawave.query.tables.ssdeep;
 
 import java.util.AbstractMap;
 import java.util.ArrayList;
diff --git a/warehouse/query-core/src/test/java/datawave/query/QueryTestTableHelper.java b/warehouse/query-core/src/test/java/datawave/query/QueryTestTableHelper.java
index dccb2fb85e1..72888245c57 100644
--- a/warehouse/query-core/src/test/java/datawave/query/QueryTestTableHelper.java
+++ b/warehouse/query-core/src/test/java/datawave/query/QueryTestTableHelper.java
@@ -23,6 +23,7 @@
 
 import datawave.accumulo.inmemory.InMemoryAccumuloClient;
 import datawave.accumulo.inmemory.InMemoryInstance;
+import datawave.helpers.PrintUtility;
 import datawave.ingest.mapreduce.handler.facet.FacetHandler;
 import datawave.ingest.mapreduce.handler.shard.ShardedDataTypeHandler;
 import datawave.ingest.table.config.FacetTableConfigHelper;
@@ -35,21 +36,16 @@
 /**
  * Creates and configures tables that are commonly needed for queries
  */
-public final class QueryTestTableHelper {
+public class QueryTestTableHelper {
 
     public static final String METADATA_TABLE_NAME = "metadata";
     public static final String SHARD_DICT_INDEX_NAME = "shardTermDictionary";
     public static final String MODEL_TABLE_NAME = "DatawaveMetadata";
 
-    // TODO: elsewhere?
-    public static final String FACET_TABLE_NAME = "facet";
-    public static final String FACET_HASH_TABLE_NAME = "facetHash";
-    public static final String FACET_METADATA_TABLE_NAME = "facetMetadata";
-
     private static final BatchWriterConfig bwCfg = new BatchWriterConfig().setMaxLatency(1, TimeUnit.SECONDS).setMaxMemory(1000L).setMaxWriteThreads(1);
 
     public final AccumuloClient client;
-    private final Logger log; // passed in for context when debugging
+    protected final Logger log; // passed in for context when debugging
 
     public QueryTestTableHelper(AccumuloClient client, Logger log)
                     throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException {
@@ -74,26 +70,15 @@ public QueryTestTableHelper(String instanceName, Logger log, RebuildingScannerTe
         createTables();
     }
 
-    public void dumpTables(Authorizations auths) {
-        try {
-            dumpTable(METADATA_TABLE_NAME, auths);
-            dumpTable(TableName.DATE_INDEX, auths);
-            dumpTable(TableName.LOAD_DATES, auths);
-            dumpTable(TableName.SHARD, auths);
-            dumpTable(TableName.SHARD_INDEX, auths);
-            dumpTable(TableName.SHARD_RINDEX, auths);
-            dumpTable(SHARD_DICT_INDEX_NAME, auths);
-            dumpTable(MODEL_TABLE_NAME, auths);
-
-            // TODO: elsewhere?
-            dumpTable(FACET_TABLE_NAME, auths);
-            dumpTable(FACET_HASH_TABLE_NAME, auths);
-            dumpTable(FACET_METADATA_TABLE_NAME, auths);
-
-        } catch (TableNotFoundException e) {
-            // should not happen
-            throw new IllegalArgumentException(e);
-        }
+    public void dumpTables(Authorizations auths) throws TableNotFoundException {
+        dumpTable(METADATA_TABLE_NAME, auths);
+        dumpTable(TableName.DATE_INDEX, auths);
+        dumpTable(TableName.LOAD_DATES, auths);
+        dumpTable(TableName.SHARD, auths);
+        dumpTable(TableName.SHARD_INDEX, auths);
+        dumpTable(TableName.SHARD_RINDEX, auths);
+        dumpTable(SHARD_DICT_INDEX_NAME, auths);
+        dumpTable(MODEL_TABLE_NAME, auths);
     }
 
     public void dumpTable(String table, Authorizations auths) throws TableNotFoundException {
@@ -108,7 +93,14 @@ public void dumpTable(String table, Authorizations auths) throws TableNotFoundEx
         scanner.close();
     }
 
-    private void createTables() throws AccumuloSecurityException, AccumuloException, TableNotFoundException, TableExistsException {
+    public void printTables(Authorizations auths) throws TableNotFoundException {
+        PrintUtility.printTable(client, auths, QueryTestTableHelper.METADATA_TABLE_NAME);
+        PrintUtility.printTable(client, auths, TableName.SHARD);
+        PrintUtility.printTable(client, auths, TableName.SHARD_INDEX);
+        PrintUtility.printTable(client, auths, TableName.SHARD_RINDEX);
+    }
+
+    protected void createTables() throws AccumuloSecurityException, AccumuloException, TableNotFoundException, TableExistsException {
         TableOperations tops = client.tableOperations();
         deleteAndCreateTable(tops, METADATA_TABLE_NAME);
         deleteAndCreateTable(tops, TableName.DATE_INDEX);
@@ -118,15 +110,9 @@ private void createTables() throws AccumuloSecurityException, AccumuloException,
         deleteAndCreateTable(tops, TableName.SHARD_RINDEX);
         deleteAndCreateTable(tops, SHARD_DICT_INDEX_NAME);
         deleteAndCreateTable(tops, MODEL_TABLE_NAME);
-
-        // TODO: move these elsewhere?
-        deleteAndCreateTable(tops, FACET_TABLE_NAME);
-        deleteAndCreateTable(tops, FACET_HASH_TABLE_NAME);
-        deleteAndCreateTable(tops, FACET_METADATA_TABLE_NAME);
-
     }
 
-    private void deleteAndCreateTable(TableOperations tops, String tableName)
+    protected void deleteAndCreateTable(TableOperations tops, String tableName)
                     throws AccumuloException, AccumuloSecurityException, TableNotFoundException, TableExistsException {
         if (tops.exists(tableName)) {
             tops.delete(tableName);
@@ -153,15 +139,10 @@ public void configureTables(MockAccumuloRecordWriter writer) throws AccumuloSecu
         configureAShardRelatedTable(writer, new ShardTableConfigHelper(), ShardedDataTypeHandler.SHARD_GRIDX_TNAME, TableName.SHARD_RINDEX);
         configureAShardRelatedTable(writer, new ShardTableConfigHelper(), ShardedDataTypeHandler.SHARD_DINDX_NAME, SHARD_DICT_INDEX_NAME);
 
-        // TODO: move this elsewhere?
-        configureAShardRelatedTable(writer, new FacetTableConfigHelper(), FacetHandler.FACET_TABLE_NAME, FACET_TABLE_NAME);
-        configureAShardRelatedTable(writer, new FacetTableConfigHelper(), FacetHandler.FACET_METADATA_TABLE_NAME, FACET_METADATA_TABLE_NAME);
-        configureAShardRelatedTable(writer, new FacetTableConfigHelper(), FacetHandler.FACET_HASH_TABLE_NAME, FACET_HASH_TABLE_NAME);
-
         // todo - configure the other tables...
     }
 
-    private void configureAShardRelatedTable(MockAccumuloRecordWriter writer, TableConfigHelper helper, String keyForTableName, String tableName)
+    protected void configureAShardRelatedTable(MockAccumuloRecordWriter writer, TableConfigHelper helper, String keyForTableName, String tableName)
                     throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
         log.debug("---------------  configure table (" + keyForTableName + ")  ---------------");
         Configuration tableConfig = new Configuration();
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/facets/FacetQueryTestTableHelper.java b/warehouse/query-core/src/test/java/datawave/query/tables/facets/FacetQueryTestTableHelper.java
new file mode 100644
index 00000000000..c359f8dbb04
--- /dev/null
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/facets/FacetQueryTestTableHelper.java
@@ -0,0 +1,63 @@
+package datawave.query.tables.facets;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.log4j.Logger;
+
+import datawave.helpers.PrintUtility;
+import datawave.ingest.mapreduce.handler.facet.FacetHandler;
+import datawave.ingest.table.config.FacetTableConfigHelper;
+import datawave.query.MockAccumuloRecordWriter;
+import datawave.query.QueryTestTableHelper;
+import datawave.query.RebuildingScannerTestHelper;
+
+public class FacetQueryTestTableHelper extends QueryTestTableHelper {
+    public static final String FACET_TABLE_NAME = "facet";
+    public static final String FACET_HASH_TABLE_NAME = "facetHash";
+    public static final String FACET_METADATA_TABLE_NAME = "facetMetadata";
+
+    public FacetQueryTestTableHelper(String instanceName, Logger log, RebuildingScannerTestHelper.TEARDOWN teardown,
+                    RebuildingScannerTestHelper.INTERRUPT interrupt)
+                    throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException {
+        super(instanceName, log, teardown, interrupt);
+    }
+
+    @Override
+    public void printTables(Authorizations auths) throws TableNotFoundException {
+        super.printTables(auths);
+        PrintUtility.printTable(client, auths, FACET_TABLE_NAME);
+        PrintUtility.printTable(client, auths, FACET_METADATA_TABLE_NAME);
+        PrintUtility.printTable(client, auths, FACET_HASH_TABLE_NAME);
+    }
+
+    @Override
+    public void dumpTables(Authorizations auths) throws TableNotFoundException {
+        super.dumpTables(auths);
+        dumpTable(FACET_TABLE_NAME, auths);
+        dumpTable(FACET_HASH_TABLE_NAME, auths);
+        dumpTable(FACET_METADATA_TABLE_NAME, auths);
+    }
+
+    @Override
+    protected void createTables() throws AccumuloSecurityException, AccumuloException, TableNotFoundException, TableExistsException {
+        super.createTables();
+
+        TableOperations tops = client.tableOperations();
+        deleteAndCreateTable(tops, FACET_TABLE_NAME);
+        deleteAndCreateTable(tops, FACET_HASH_TABLE_NAME);
+        deleteAndCreateTable(tops, FACET_METADATA_TABLE_NAME);
+    }
+
+    @Override
+    public void configureTables(MockAccumuloRecordWriter writer) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+        super.configureTables(writer);
+
+        configureAShardRelatedTable(writer, new FacetTableConfigHelper(), FacetHandler.FACET_TABLE_NAME, FACET_TABLE_NAME);
+        configureAShardRelatedTable(writer, new FacetTableConfigHelper(), FacetHandler.FACET_METADATA_TABLE_NAME, FACET_METADATA_TABLE_NAME);
+        configureAShardRelatedTable(writer, new FacetTableConfigHelper(), FacetHandler.FACET_HASH_TABLE_NAME, FACET_HASH_TABLE_NAME);
+    }
+}
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/facets/FacetedCitiesDataType.java b/warehouse/query-core/src/test/java/datawave/query/tables/facets/FacetedCitiesDataType.java
index 55ff25c1551..24ca1a2c6b1 100644
--- a/warehouse/query-core/src/test/java/datawave/query/tables/facets/FacetedCitiesDataType.java
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/facets/FacetedCitiesDataType.java
@@ -6,10 +6,8 @@
 import datawave.ingest.data.TypeRegistry;
 import datawave.ingest.mapreduce.handler.facet.FacetHandler;
 import datawave.ingest.mapreduce.handler.shard.AbstractColumnBasedHandler;
-import datawave.query.QueryTestTableHelper;
 import datawave.query.testframework.CitiesDataType;
 import datawave.query.testframework.FieldConfig;
-import datawave.webservice.query.Query;
 
 public class FacetedCitiesDataType extends CitiesDataType {
 
@@ -24,8 +22,8 @@ public FacetedCitiesDataType(final String city, final String ingestFile, final F
         this.hConf.set(this.dataType + ".facet.category.name" + ".continent", "CONTINENT;STATE,CITY");
         this.hConf.set(this.dataType + ".facet.category.name" + ".code", "CODE;STATE,CITY");
 
-        this.hConf.set(FacetHandler.FACET_TABLE_NAME, QueryTestTableHelper.FACET_TABLE_NAME);
-        this.hConf.set(FacetHandler.FACET_METADATA_TABLE_NAME, QueryTestTableHelper.FACET_METADATA_TABLE_NAME);
-        this.hConf.set(FacetHandler.FACET_HASH_TABLE_NAME, QueryTestTableHelper.FACET_HASH_TABLE_NAME);
+        this.hConf.set(FacetHandler.FACET_TABLE_NAME, FacetQueryTestTableHelper.FACET_TABLE_NAME);
+        this.hConf.set(FacetHandler.FACET_METADATA_TABLE_NAME, FacetQueryTestTableHelper.FACET_METADATA_TABLE_NAME);
+        this.hConf.set(FacetHandler.FACET_HASH_TABLE_NAME, FacetQueryTestTableHelper.FACET_HASH_TABLE_NAME);
     }
 }
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/facets/FacetedQueryLogicTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/facets/FacetedQueryLogicTest.java
index 99a0f04509a..de2827d6019 100644
--- a/warehouse/query-core/src/test/java/datawave/query/tables/facets/FacetedQueryLogicTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/facets/FacetedQueryLogicTest.java
@@ -43,7 +43,6 @@
 import datawave.query.testframework.GenericCityFields;
 import datawave.query.testframework.QueryLogicTestHarness;
 import datawave.query.testframework.QueryLogicTestHarness.DocumentChecker;
-import datawave.query.testframework.cardata.CarsDataType;
 import datawave.query.util.DateIndexHelperFactory;
 import datawave.query.util.MetadataHelperFactory;
 import datawave.security.authorization.DatawavePrincipal;
@@ -75,9 +74,12 @@ public static void setupClass() throws Exception {
         dataTypes.add(new FacetedCitiesDataType(CitiesDataType.CityEntry.paris, generic));
         dataTypes.add(new FacetedCitiesDataType(CitiesDataType.CityEntry.rome, generic));
 
+        FacetQueryTestTableHelper facetQueryTestTableHelper = new FacetQueryTestTableHelper(FacetedQueryLogicTest.class.getName(), log, TEARDOWN.EVERY_OTHER,
+                        INTERRUPT.NEVER);
         accumuloSetup.setData(FileType.CSV, dataTypes);
         accumuloSetup.setAuthorizations(CitiesDataType.getTestAuths());
-        client = accumuloSetup.loadTables(log, TEARDOWN.EVERY_OTHER, INTERRUPT.NEVER);
+
+        client = accumuloSetup.loadTables(facetQueryTestTableHelper);
     }
 
     @Before
@@ -90,9 +92,9 @@ public void querySetUp() throws IOException {
         FacetedQueryLogic facetLogic = new FacetedQueryLogic();
         facetLogic.setFacetedSearchType(FacetedSearchType.FIELD_VALUE_FACETS);
 
-        facetLogic.setFacetTableName(QueryTestTableHelper.FACET_TABLE_NAME);
-        facetLogic.setFacetMetadataTableName(QueryTestTableHelper.FACET_METADATA_TABLE_NAME);
-        facetLogic.setFacetHashTableName(QueryTestTableHelper.FACET_HASH_TABLE_NAME);
+        facetLogic.setFacetTableName(FacetQueryTestTableHelper.FACET_TABLE_NAME);
+        facetLogic.setFacetMetadataTableName(FacetQueryTestTableHelper.FACET_METADATA_TABLE_NAME);
+        facetLogic.setFacetHashTableName(FacetQueryTestTableHelper.FACET_HASH_TABLE_NAME);
 
         facetLogic.setMaximumFacetGrouping(200);
         facetLogic.setMinimumFacet(1);
@@ -150,7 +152,7 @@ public void testQueryPrecomputedFacets() throws Exception {
         expected.add("STATE; veneto -- veneto//1");
         expected.add("STATE; viana do castelo -- viana do castelo//1");
 
-        String query = CitiesDataType.CityField.CONTINENT.name() + " == 'Europe'";
+        String query = CityField.CONTINENT.name() + " == 'Europe'";
 
         runTest(query, Collections.emptyMap(), expected);
     }
@@ -190,7 +192,7 @@ public void runTest(String query, Map options, Set expect
     @Override
     protected void testInit() {
         this.auths = CitiesDataType.getTestAuths();
-        this.documentKey = CitiesDataType.CityField.EVENT_ID.name();
+        this.documentKey = CityField.EVENT_ID.name();
     }
 
     @Override
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java
new file mode 100644
index 00000000000..e90659b2f07
--- /dev/null
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java
@@ -0,0 +1,174 @@
+package datawave.query.tables.ssdeep;
+
+import static datawave.query.tables.ssdeep.util.SSDeepTestUtil.BUCKET_COUNT;
+import static datawave.query.tables.ssdeep.util.SSDeepTestUtil.BUCKET_ENCODING_BASE;
+import static datawave.query.tables.ssdeep.util.SSDeepTestUtil.BUCKET_ENCODING_LENGTH;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.commons.collections4.iterators.TransformIterator;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+import com.google.common.collect.Sets;
+
+import datawave.helpers.PrintUtility;
+import datawave.ingest.mapreduce.handler.ssdeep.SSDeepIndexHandler;
+import datawave.marking.MarkingFunctions;
+import datawave.microservice.querymetric.QueryMetricFactoryImpl;
+import datawave.query.RebuildingScannerTestHelper;
+import datawave.query.tables.ssdeep.testframework.SSDeepDataType;
+import datawave.query.tables.ssdeep.testframework.SSDeepFields;
+import datawave.query.tables.ssdeep.testframework.SSDeepQueryTestTableHelper;
+import datawave.query.tables.ssdeep.util.SSDeepTestUtil;
+import datawave.query.testframework.AbstractFunctionalQuery;
+import datawave.query.testframework.AccumuloSetup;
+import datawave.query.testframework.DataTypeHadoopConfig;
+import datawave.query.testframework.FieldConfig;
+import datawave.query.testframework.FileType;
+import datawave.query.testframework.QueryLogicTestHarness;
+import datawave.security.authorization.DatawavePrincipal;
+import datawave.security.authorization.DatawaveUser;
+import datawave.security.authorization.SubjectIssuerDNPair;
+import datawave.webservice.common.connection.AccumuloConnectionFactory;
+import datawave.webservice.query.QueryImpl;
+import datawave.webservice.query.logic.AbstractQueryLogicTransformer;
+import datawave.webservice.query.logic.QueryLogic;
+import datawave.webservice.query.result.event.DefaultResponseObjectFactory;
+import datawave.webservice.query.result.event.EventBase;
+import datawave.webservice.query.result.event.FieldBase;
+import datawave.webservice.query.result.event.ResponseObjectFactory;
+import datawave.webservice.query.runner.RunningQuery;
+import datawave.webservice.result.EventQueryResponseBase;
+
+public class SSDeepIngestQueryTest extends AbstractFunctionalQuery {
+
+    @ClassRule
+    public static AccumuloSetup accumuloSetup = new AccumuloSetup();
+
+    private static final Logger log = Logger.getLogger(SSDeepIngestQueryTest.class);
+
+    SSDeepSimilarityQueryLogic similarityQueryLogic;
+
+    @BeforeClass
+    public static void filterSetup() throws Exception {
+        log.setLevel(Level.DEBUG);
+        Logger printLog = Logger.getLogger(PrintUtility.class);
+        printLog.setLevel(Level.DEBUG);
+
+        Collection dataTypes = new ArrayList<>();
+        FieldConfig generic = new SSDeepFields();
+        dataTypes.add(new SSDeepDataType(SSDeepDataType.SSDeepEntry.ssdeep, generic));
+
+        SSDeepQueryTestTableHelper ssDeepQueryTestTableHelper = new SSDeepQueryTestTableHelper(SSDeepIngestQueryTest.class.getName(), log,
+                        RebuildingScannerTestHelper.TEARDOWN.EVERY_OTHER, RebuildingScannerTestHelper.INTERRUPT.NEVER);
+        accumuloSetup.setData(FileType.CSV, dataTypes);
+        client = accumuloSetup.loadTables(ssDeepQueryTestTableHelper);
+    }
+
+    @Before
+    public void setupQuery() {
+        MarkingFunctions markingFunctions = new MarkingFunctions.Default();
+        ResponseObjectFactory responseFactory = new DefaultResponseObjectFactory();
+
+        similarityQueryLogic = new SSDeepSimilarityQueryLogic();
+        similarityQueryLogic.setTableName(SSDeepIndexHandler.DEFAULT_SSDEEP_INDEX_TABLE_NAME);
+        similarityQueryLogic.setMarkingFunctions(markingFunctions);
+        similarityQueryLogic.setResponseObjectFactory(responseFactory);
+        similarityQueryLogic.setBucketEncodingBase(BUCKET_ENCODING_BASE);
+        similarityQueryLogic.setBucketEncodingLength(BUCKET_ENCODING_LENGTH);
+        similarityQueryLogic.setIndexBuckets(BUCKET_COUNT);
+
+        // init must set auths
+        testInit();
+
+        SubjectIssuerDNPair dn = SubjectIssuerDNPair.of("userDn", "issuerDn");
+        DatawaveUser user = new DatawaveUser(dn, DatawaveUser.UserType.USER, Sets.newHashSet(this.auths.toString().split(",")), null, null, -1L);
+        principal = new DatawavePrincipal(Collections.singleton(user));
+
+        testHarness = new QueryLogicTestHarness(this);
+    }
+
+    protected void testInit() {
+        this.auths = SSDeepDataType.getTestAuths();
+        this.documentKey = SSDeepDataType.SSDeepField.EVENT_ID.name();
+    }
+
+    public SSDeepIngestQueryTest() {
+        super(SSDeepDataType.getManager());
+    }
+
+    @SuppressWarnings("rawtypes")
+    @Test
+    public void testSSDeepSimilarity() throws Exception {
+        log.info("------ testSSDeepSimilarity ------");
+        @SuppressWarnings("SpellCheckingInspection")
+        String testSSDeep = "384:nv/fP9FmWVMdRFj2aTgSO+u5QT4ZE1PIVS:nDmWOdRFNTTs504cQS";
+        String query = "CHECKSUM_SSDEEP:" + testSSDeep;
+        EventQueryResponseBase response = runSSDeepQuery(query, similarityQueryLogic, 0);
+
+        List events = response.getEvents();
+        Assert.assertEquals(1, events.size());
+        Map> observedEvents = extractObservedEvents(events);
+
+        SSDeepTestUtil.assertSSDeepSimilarityMatch(testSSDeep, testSSDeep, "38.0", "100", observedEvents);
+    }
+
+    @SuppressWarnings("rawtypes")
+    public EventQueryResponseBase runSSDeepQuery(String query, QueryLogic queryLogic, int minScoreThreshold) throws Exception {
+        QueryImpl q = new QueryImpl();
+        q.setQuery(query);
+        q.setId(UUID.randomUUID());
+        q.setPagesize(Integer.MAX_VALUE);
+        q.setQueryAuthorizations(auths.toString());
+
+        if (minScoreThreshold > 0) {
+            q.addParameter(SSDeepSimilarityQueryTransformer.MIN_SSDEEP_SCORE_PARAMETER, String.valueOf(minScoreThreshold));
+        }
+
+        RunningQuery runner = new RunningQuery(client, AccumuloConnectionFactory.Priority.NORMAL, queryLogic, q, "", principal, new QueryMetricFactoryImpl());
+        TransformIterator transformIterator = runner.getTransformIterator();
+        AbstractQueryLogicTransformer transformer = (AbstractQueryLogicTransformer) transformIterator.getTransformer();
+        return (EventQueryResponseBase) transformer.createResponse(runner.next());
+    }
+
+    /** Extract the events from a set of results into an easy to manage data structure for validation */
+    @SuppressWarnings({"rawtypes", "unchecked"})
+    public Map> extractObservedEvents(List events) {
+        int eventCount = events.size();
+        Map> observedEvents = new HashMap<>();
+        if (eventCount > 0) {
+            for (EventBase e : events) {
+                Map observedFields = new HashMap<>();
+                String querySsdeep = "UNKNOWN_QUERY";
+                String matchingSsdeep = "UNKNOWN_MATCH";
+
+                List fields = e.getFields();
+                for (FieldBase f : fields) {
+                    if (f.getName().equals("QUERY_SSDEEP")) {
+                        querySsdeep = f.getValueString();
+                    }
+                    if (f.getName().equals("MATCHING_SSDEEP")) {
+                        matchingSsdeep = f.getValueString();
+                    }
+                    observedFields.put(f.getName(), f.getValueString());
+                }
+
+                String eventKey = querySsdeep + "#" + matchingSsdeep;
+                observedEvents.put(eventKey, observedFields);
+            }
+        }
+        return observedEvents;
+    }
+}
diff --git a/warehouse/query-core/src/test/java/datawave/query/SSDeepQueryTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepQueryTest.java
similarity index 96%
rename from warehouse/query-core/src/test/java/datawave/query/SSDeepQueryTest.java
rename to warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepQueryTest.java
index a9700423ab8..8aafd08b3b8 100644
--- a/warehouse/query-core/src/test/java/datawave/query/SSDeepQueryTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepQueryTest.java
@@ -1,4 +1,4 @@
-package datawave.query;
+package datawave.query.tables.ssdeep;
 
 import static org.junit.Assert.fail;
 
@@ -33,16 +33,16 @@
 
 import datawave.accumulo.inmemory.InMemoryAccumuloClient;
 import datawave.accumulo.inmemory.InMemoryInstance;
+import datawave.ingest.mapreduce.handler.ssdeep.SSDeepIndexHandler;
 import datawave.marking.MarkingFunctions;
 import datawave.microservice.querymetric.QueryMetricFactoryImpl;
-import datawave.query.tables.SSDeepSimilarityQueryLogic;
 import datawave.query.testframework.AbstractDataTypeConfig;
-import datawave.query.transformer.SSDeepSimilarityQueryTransformer;
 import datawave.security.authorization.DatawavePrincipal;
 import datawave.security.authorization.DatawaveUser;
 import datawave.security.authorization.SubjectIssuerDNPair;
 import datawave.util.ssdeep.BucketAccumuloKeyGenerator;
 import datawave.util.ssdeep.NGramByteHashGenerator;
+import datawave.util.ssdeep.NGramGenerator;
 import datawave.util.ssdeep.NGramTuple;
 import datawave.webservice.common.connection.AccumuloConnectionFactory;
 import datawave.webservice.query.QueryImpl;
@@ -78,9 +78,9 @@ public class SSDeepQueryTest {
 
     public static void indexSSDeepTestData(AccumuloClient accumuloClient) throws Exception {
         // configuration
-        String ssdeepTableName = "ssdeepIndex";
-        int ngramSize = 7;
-        int minHashSize = 3;
+        String ssdeepTableName = SSDeepIndexHandler.DEFAULT_SSDEEP_INDEX_TABLE_NAME;
+        int ngramSize = NGramGenerator.DEFAULT_NGRAM_SIZE;
+        int minHashSize = NGramGenerator.DEFAULT_MIN_HASH_SIZE;
 
         // input
         Stream ssdeepLines = Stream.of(TEST_SSDEEPS);
@@ -118,7 +118,7 @@ public static void indexSSDeepTestData(AccumuloClient accumuloClient) throws Exc
 
     @BeforeClass
     public static void loadData() throws Exception {
-        final String tableName = "ssdeepIndex";
+        final String tableName = SSDeepIndexHandler.DEFAULT_SSDEEP_INDEX_TABLE_NAME;
 
         InMemoryInstance i = new InMemoryInstance("ssdeepTestInstance");
         accumuloClient = new InMemoryAccumuloClient("root", i);
@@ -140,7 +140,7 @@ public static void loadData() throws Exception {
     @Before
     public void setUpQuery() {
         logic = new SSDeepSimilarityQueryLogic();
-        logic.setTableName("ssdeepIndex");
+        logic.setTableName(SSDeepIndexHandler.DEFAULT_SSDEEP_INDEX_TABLE_NAME);
         logic.setMarkingFunctions(new MarkingFunctions.Default());
         logic.setResponseObjectFactory(new DefaultResponseObjectFactory());
         logic.setBucketEncodingBase(BUCKET_ENCODING_BASE);
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTest.java
new file mode 100644
index 00000000000..3833da7494f
--- /dev/null
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTest.java
@@ -0,0 +1,162 @@
+package datawave.query.tables.ssdeep;
+
+import static datawave.query.tables.ssdeep.util.SSDeepTestUtil.BUCKET_COUNT;
+import static datawave.query.tables.ssdeep.util.SSDeepTestUtil.BUCKET_ENCODING_BASE;
+import static datawave.query.tables.ssdeep.util.SSDeepTestUtil.BUCKET_ENCODING_LENGTH;
+import static datawave.query.tables.ssdeep.util.SSDeepTestUtil.TEST_SSDEEPS;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.commons.collections4.iterators.TransformIterator;
+import org.apache.log4j.Logger;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.google.common.collect.Sets;
+
+import datawave.accumulo.inmemory.InMemoryAccumuloClient;
+import datawave.accumulo.inmemory.InMemoryInstance;
+import datawave.ingest.mapreduce.handler.ssdeep.SSDeepIndexHandler;
+import datawave.marking.MarkingFunctions;
+import datawave.microservice.querymetric.QueryMetricFactoryImpl;
+import datawave.query.tables.ssdeep.util.SSDeepTestUtil;
+import datawave.query.testframework.AbstractDataTypeConfig;
+import datawave.security.authorization.DatawavePrincipal;
+import datawave.security.authorization.DatawaveUser;
+import datawave.security.authorization.SubjectIssuerDNPair;
+import datawave.webservice.common.connection.AccumuloConnectionFactory;
+import datawave.webservice.query.QueryImpl;
+import datawave.webservice.query.result.event.DefaultResponseObjectFactory;
+import datawave.webservice.query.result.event.EventBase;
+import datawave.webservice.query.runner.RunningQuery;
+import datawave.webservice.result.EventQueryResponseBase;
+
+public class SSDeepSimilarityQueryTest {
+
+    private static final Logger log = Logger.getLogger(SSDeepSimilarityQueryTest.class);
+
+    private static final Authorizations auths = AbstractDataTypeConfig.getTestAuths();
+
+    protected static AccumuloClient accumuloClient;
+
+    protected SSDeepSimilarityQueryLogic logic;
+
+    protected DatawavePrincipal principal;
+
+    @BeforeClass
+    public static void loadData() throws Exception {
+        final String tableName = SSDeepIndexHandler.DEFAULT_SSDEEP_INDEX_TABLE_NAME;
+
+        InMemoryInstance inMemoryInstance = new InMemoryInstance("ssdeepTestInstance");
+        accumuloClient = new InMemoryAccumuloClient("root", inMemoryInstance);
+
+        /* create the table */
+        TableOperations tops = accumuloClient.tableOperations();
+        if (tops.exists(tableName)) {
+            tops.delete(tableName);
+        }
+        tops.create(tableName);
+
+        /* add ssdeep data to the table */
+        SSDeepTestUtil.loadSSDeepIndexTextData(accumuloClient);
+
+        /* dump the table */
+        logSSDeepTestData();
+    }
+
+    @Before
+    public void setUpQuery() {
+        logic = new SSDeepSimilarityQueryLogic();
+        logic.setTableName(SSDeepIndexHandler.DEFAULT_SSDEEP_INDEX_TABLE_NAME);
+        logic.setMarkingFunctions(new MarkingFunctions.Default());
+        logic.setResponseObjectFactory(new DefaultResponseObjectFactory());
+        logic.setBucketEncodingBase(BUCKET_ENCODING_BASE);
+        logic.setBucketEncodingLength(BUCKET_ENCODING_LENGTH);
+        logic.setIndexBuckets(BUCKET_COUNT);
+
+        SubjectIssuerDNPair dn = SubjectIssuerDNPair.of("userDn", "issuerDn");
+        DatawaveUser user = new DatawaveUser(dn, DatawaveUser.UserType.USER, Sets.newHashSet(auths.toString().split(",")), null, null, -1L);
+        principal = new DatawavePrincipal(Collections.singleton(user));
+    }
+
+    @Test
+    public void testSingleQueryNoMinScore() throws Exception {
+        runSingleQuery(false);
+    }
+
+    @Test
+    public void testSingleQueryMinScore() throws Exception {
+        runSingleQuery(true);
+    }
+
+    private static void logSSDeepTestData() throws TableNotFoundException {
+        Scanner scanner = accumuloClient.createScanner(SSDeepIndexHandler.DEFAULT_SSDEEP_INDEX_TABLE_NAME, auths);
+        Iterator> iterator = scanner.iterator();
+        log.debug("*************** ssdeepIndex ********************");
+        while (iterator.hasNext()) {
+            Map.Entry entry = iterator.next();
+            log.debug(entry);
+        }
+        scanner.close();
+    }
+
+    @SuppressWarnings("rawtypes")
+    public void runSingleQuery(boolean applyMinScoreThreshold) throws Exception {
+        String query = "CHECKSUM_SSDEEP:" + TEST_SSDEEPS[2];
+
+        final int minScoreThreshold = applyMinScoreThreshold ? 65 : 0;
+        final int expectedEventCount = applyMinScoreThreshold ? 2 : 3;
+
+        EventQueryResponseBase response = runSSDeepQuery(query, minScoreThreshold);
+        List events = response.getEvents();
+        int eventCount = events.size();
+        Map> observedEvents = SSDeepTestUtil.extractObservedEvents(events);
+
+        Assert.assertEquals(expectedEventCount, eventCount);
+
+        // find the fields for the self match example.
+        SSDeepTestUtil.assertSSDeepSimilarityMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[2], "65.0", "100", observedEvents);
+
+        // find and validate the fields for the partial match example.
+        SSDeepTestUtil.assertSSDeepSimilarityMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[3], "51.0", "96", observedEvents);
+
+        if (applyMinScoreThreshold)
+            SSDeepTestUtil.assertNoMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[3], observedEvents);
+        else
+            SSDeepTestUtil.assertSSDeepSimilarityMatch(TEST_SSDEEPS[2], TEST_SSDEEPS[4], "9.0", "63", observedEvents);
+    }
+
+    @SuppressWarnings("rawtypes")
+    public EventQueryResponseBase runSSDeepQuery(String query, int minScoreThreshold) throws Exception {
+
+        QueryImpl q = new QueryImpl();
+        q.setQuery(query);
+        q.setId(UUID.randomUUID());
+        q.setPagesize(Integer.MAX_VALUE);
+        q.setQueryAuthorizations(auths.toString());
+
+        if (minScoreThreshold > 0) {
+            q.addParameter(SSDeepSimilarityQueryTransformer.MIN_SSDEEP_SCORE_PARAMETER, String.valueOf(minScoreThreshold));
+        }
+
+        RunningQuery runner = new RunningQuery(accumuloClient, AccumuloConnectionFactory.Priority.NORMAL, this.logic, q, "", principal,
+                        new QueryMetricFactoryImpl());
+        TransformIterator transformIterator = runner.getTransformIterator();
+        SSDeepSimilarityQueryTransformer transformer = (SSDeepSimilarityQueryTransformer) transformIterator.getTransformer();
+        return (EventQueryResponseBase) transformer.createResponse(runner.next());
+    }
+
+}
diff --git a/warehouse/query-core/src/test/java/datawave/query/transformer/SSDeepSimilarityQueryTransformerTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformerTest.java
similarity index 96%
rename from warehouse/query-core/src/test/java/datawave/query/transformer/SSDeepSimilarityQueryTransformerTest.java
rename to warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformerTest.java
index ca9e5fd6622..e22c300d1d6 100644
--- a/warehouse/query-core/src/test/java/datawave/query/transformer/SSDeepSimilarityQueryTransformerTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepSimilarityQueryTransformerTest.java
@@ -1,4 +1,4 @@
-package datawave.query.transformer;
+package datawave.query.tables.ssdeep;
 
 import java.util.AbstractMap;
 import java.util.ArrayList;
@@ -21,8 +21,7 @@
 
 import datawave.marking.MarkingFunctions;
 import datawave.query.config.SSDeepSimilarityQueryConfiguration;
-import datawave.util.ssdeep.ChunkSizeEncoding;
-import datawave.util.ssdeep.IntegerEncoding;
+import datawave.query.tables.ssdeep.SSDeepSimilarityQueryTransformer;
 import datawave.util.ssdeep.NGramTuple;
 import datawave.util.ssdeep.SSDeepHash;
 import datawave.webservice.query.Query;
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepDataManager.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepDataManager.java
new file mode 100644
index 00000000000..077319a6660
--- /dev/null
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepDataManager.java
@@ -0,0 +1,72 @@
+package datawave.query.tables.ssdeep.testframework;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.log4j.Logger;
+import org.junit.Assert;
+
+import au.com.bytecode.opencsv.CSVReader;
+import datawave.query.tables.ssdeep.testframework.SSDeepDataType.SSDeepField;
+import datawave.query.testframework.AbstractDataManager;
+import datawave.query.testframework.BaseRawData;
+import datawave.query.testframework.RawData;
+
+/**
+ * Provides a mapping of the raw ingest data for query analysis. This will allow dynamic calculation of expected results and modification of the test data
+ * without impacting the test cases. Each entry in the file will be transformed to a POJO entry.
+ * 
    
+ * 
    
+ * This class is immutable.
+ */
+public class SSDeepDataManager extends AbstractDataManager {
+
+    private static final Logger log = Logger.getLogger(SSDeepDataManager.class);
+
+    public SSDeepDataManager() {
+        super(SSDeepField.EVENT_ID.name(), SSDeepField.PROCESSING_DATE.name(), SSDeepField.getFieldsMetadata());
+    }
+
+    @Override
+    public void addTestData(final URI file, final String datatype, final Set indexes) throws IOException {
+        Assert.assertFalse("datatype has already been configured(" + datatype + ")", this.rawData.containsKey(datatype));
+        try (final Reader reader = Files.newBufferedReader(Paths.get(file)); final CSVReader csv = new CSVReader(reader, ',', '\"', '\0')) {
+            String[] data;
+            int count = 0;
+            Set ssdeepData = new HashSet<>();
+            while (null != (data = csv.readNext())) {
+                final RawData raw = new SSDeepRawData(datatype, data);
+                ssdeepData.add(raw);
+                count++;
+            }
+            this.rawData.put(datatype, ssdeepData);
+            this.rawDataIndex.put(datatype, indexes);
+            log.info("ssdeep test data(" + file + ") count(" + count + ")");
+        }
+    }
+
+    @Override
+    public List getHeaders() {
+        return SSDeepField.headers();
+    }
+
+    /**
+     * POJO for a single raw data entry.
+     */
+    private static class SSDeepRawData extends BaseRawData {
+        SSDeepRawData(final String datatype, final String fields[]) {
+            super(datatype, fields, SSDeepField.headers(), SSDeepField.getFieldsMetadata());
+        }
+
+        @Override
+        protected boolean hasExtraFields() {
+            return true;
+        }
+    }
+}
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepDataType.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepDataType.java
new file mode 100644
index 00000000000..6e2672330bd
--- /dev/null
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepDataType.java
@@ -0,0 +1,266 @@
+package datawave.query.tables.ssdeep.testframework;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.log4j.Logger;
+
+import datawave.data.normalizer.Normalizer;
+import datawave.data.type.NoOpType;
+import datawave.ingest.csv.config.helper.ExtendedCSVHelper;
+import datawave.ingest.data.TypeRegistry;
+import datawave.ingest.data.config.CSVHelper;
+import datawave.ingest.data.config.ingest.BaseIngestHelper;
+import datawave.ingest.input.reader.EventRecordReader;
+import datawave.ingest.mapreduce.handler.shard.AbstractColumnBasedHandler;
+import datawave.ingest.mapreduce.handler.ssdeep.SSDeepIndexHandler;
+import datawave.marking.MarkingFunctions;
+import datawave.query.testframework.AbstractDataTypeConfig;
+import datawave.query.testframework.FieldConfig;
+import datawave.query.testframework.RawDataManager;
+import datawave.query.testframework.RawMetaData;
+
+/**
+ * Contains all the relevant data needed to configure the ssdeep data type.
+ */
+public class SSDeepDataType extends AbstractDataTypeConfig {
+
+    private static final Logger log = Logger.getLogger(SSDeepDataType.class);
+
+    /**
+     * Contains predefined names for the ssdeep datatype. Each enumeration will contain the path of the data ingest file. Currently, there is only one
+     */
+    public enum SSDeepEntry {
+        // default provided cities with datatype name
+        ssdeep("input/ssdeep-data.csv", "ssdeep");
+
+        private final String ingestFile;
+        private final String datatype;
+
+        SSDeepEntry(final String file, final String name) {
+            this.ingestFile = file;
+            this.datatype = name;
+        }
+
+        public String getIngestFile() {
+            return this.ingestFile;
+        }
+
+        /**
+         * Returns the datatype for the entry.
+         *
+         * @return datatype for instance
+         */
+        public String getDataType() {
+            return this.datatype;
+        }
+    }
+
+    /**
+     * Defines the data fields for cities datatype.
+     */
+    @SuppressWarnings("SpellCheckingInspection")
+    public enum SSDeepField {
+        // order is important, should match the order in the csv files
+        PROCESSING_DATE(Normalizer.LC_NO_DIACRITICS_NORMALIZER),
+        EVENT_ID(Normalizer.LC_NO_DIACRITICS_NORMALIZER),
+        LANGUAGE(Normalizer.LC_NO_DIACRITICS_NORMALIZER),
+        ORIGINAL_SIZE(Normalizer.LC_NO_DIACRITICS_NORMALIZER),
+        PROCESSED_SIZE(Normalizer.LC_NO_DIACRITICS_NORMALIZER),
+        FILE_TYPE(Normalizer.LC_NO_DIACRITICS_NORMALIZER),
+        MD5(Normalizer.LC_NO_DIACRITICS_NORMALIZER),
+        SHA1(Normalizer.LC_NO_DIACRITICS_NORMALIZER),
+        SHA256(Normalizer.LC_NO_DIACRITICS_NORMALIZER),
+        EVENT_DATE(Normalizer.LC_NO_DIACRITICS_NORMALIZER),
+        SECURITY_MARKING(Normalizer.LC_NO_DIACRITICS_NORMALIZER),
+        // The folowing fields are 'extra' fields indicated with the K=V structure in the CSV.
+        FILE_DATE(Normalizer.LC_NO_DIACRITICS_NORMALIZER, true),
+        FILE_NAME(Normalizer.LC_NO_DIACRITICS_NORMALIZER, true),
+        CHECKSUM_SSDEEP(Normalizer.NOOP_NORMALIZER, true),
+        IMAGEHEIGHT(Normalizer.NUMBER_NORMALIZER, true),
+        IMAGEWIDTH(Normalizer.NUMBER_NORMALIZER, true),
+        PARENT_FILETYPE(Normalizer.LC_NO_DIACRITICS_NORMALIZER, true),
+        ACCESS_CONTROLS(Normalizer.LC_NO_DIACRITICS_NORMALIZER, true);
+
+        private static final List HEADERS;
+
+        static {
+            HEADERS = Stream.of(SSDeepField.values()).filter(Predicate.not(SSDeepField::isExtraField)).map(Enum::name).collect(Collectors.toList());
+        }
+
+        public static List headers() {
+            return HEADERS;
+        }
+
+        private static final Map fieldMetadata;
+
+        static {
+            fieldMetadata = new HashMap<>();
+            for (SSDeepField field : SSDeepField.values()) {
+                fieldMetadata.put(field.name().toLowerCase(), field.metadata);
+            }
+        }
+
+        /**
+         * Retrieves the enumeration that matches the specified field.
+         *
+         * @param field
+         *            string representation of the field.
+         * @return enumeration value
+         * @throws AssertionError
+         *             field does not match any of the enumeration values
+         */
+        public static SSDeepField getField(final String field) {
+            for (final SSDeepField f : SSDeepField.values()) {
+                if (f.name().equalsIgnoreCase(field)) {
+                    return f;
+                }
+            }
+
+            throw new AssertionError("invalid SSDeep field(" + field + ")");
+        }
+
+        /**
+         * Returns mapping of ip address fields to the metadata for the field.
+         *
+         * @return populate map
+         */
+        public static Map getFieldsMetadata() {
+            return fieldMetadata;
+        }
+
+        private final RawMetaData metadata;
+
+        /**
+         * A flag to set if we expect the field to be not returned as part of the headers. These fields are represented in the CSC as fieldName=fieldValue
+         * pairs.
+         */
+        private final boolean extraField;
+
+        SSDeepField(final Normalizer normalizer) {
+            this(normalizer, false);
+        }
+
+        SSDeepField(final Normalizer normalizer, final boolean extraField) {
+            this.extraField = extraField;
+            // we don't use multivalued fields for this datatype, so don't bother setting it.
+            this.metadata = new RawMetaData(this.name(), normalizer, false);
+        }
+
+        /**
+         * Returns the metadata for this field.
+         *
+         * @return metadata
+         */
+        public RawMetaData getMetadata() {
+            return metadata;
+        }
+
+        /**
+         * Returns whether the field is an 'extra' field and should not be included in the headers.
+         *
+         * @return metadata
+         */
+        public boolean isExtraField() {
+            return extraField;
+        }
+    }
+
+    // ==================================
+    // data manager info
+    private static final RawDataManager ssdeepManager = new SSDeepDataManager();
+
+    public static RawDataManager getManager() {
+        return ssdeepManager;
+    }
+
+    /**
+     * Creates a ssdeep datatype entry with all the key/value configuration settings.
+     *
+     * @param ssdeep
+     *            entry for ingest containing datatype and ingest file
+     * @param config
+     *            hadoop field configuration
+     * @throws IOException
+     *             unable to load ingest file
+     * @throws URISyntaxException
+     *             unable to resolve ingest file
+     */
+    public SSDeepDataType(final SSDeepEntry ssdeep, final FieldConfig config) throws IOException, URISyntaxException {
+        this(ssdeep.getDataType(), ssdeep.getIngestFile(), config);
+    }
+
+    /**
+     * Constructor for ssdeep ingest files that are not defined in the class {@link SSDeepEntry}.
+     *
+     * @param ssdeep
+     *            name of the ssdeep datatype
+     * @param ingestFile
+     *            ingest file path
+     * @param config
+     *            hadoop field configuration
+     * @throws IOException
+     *             error loading test data
+     * @throws URISyntaxException
+     *             invalid test data file
+     */
+    public SSDeepDataType(final String ssdeep, final String ingestFile, final FieldConfig config) throws IOException, URISyntaxException {
+        super(ssdeep, ingestFile, config, ssdeepManager);
+        // NOTE: see super for default settings
+
+        // set datatype settings
+        this.hConf.set(this.dataType + "." + SSDeepField.CHECKSUM_SSDEEP.name() + BaseIngestHelper.FIELD_TYPE, NoOpType.class.getName());
+        this.hConf.set(this.dataType + EventRecordReader.Properties.EVENT_DATE_FIELD_NAME, SSDeepField.PROCESSING_DATE.name());
+        this.hConf.set(this.dataType + EventRecordReader.Properties.EVENT_DATE_FIELD_FORMAT, SSDEEP_DATE_FIELD_FORMAT);
+
+        this.hConf.set(this.dataType + ExtendedCSVHelper.Properties.EVENT_ID_FIELD_NAME, SSDeepField.EVENT_ID.name());
+
+        // fields
+        this.hConf.set(this.dataType + CSVHelper.DATA_HEADER, String.join(",", SSDeepField.headers()));
+        this.hConf.set(this.dataType + CSVHelper.PROCESS_EXTRA_FIELDS, "true");
+
+        // ssdeep index handler
+        this.hConf.set(this.dataType + TypeRegistry.HANDLER_CLASSES,
+                        String.join(",", AbstractColumnBasedHandler.class.getName(), SSDeepIndexHandler.class.getName()));
+        this.hConf.set(this.dataType + SSDeepIndexHandler.SSDEEP_FIELD_SET, "CHECKSUM_SSDEEP");
+        this.hConf.set(SSDeepIndexHandler.SSDEEP_INDEX_TABLE_NAME, SSDeepQueryTestTableHelper.SSDEEP_INDEX_TABLE_NAME);
+
+        log.debug(this.toString());
+    }
+
+    private static final String SSDEEP_DATE_FIELD_FORMAT = "yyyy-MM-dd hh:mm:ss";
+
+    private static final String[] AUTH_VALUES = new String[] {"public"};
+    private static final Authorizations TEST_AUTHS = new Authorizations(AUTH_VALUES);
+    private static final Authorizations EXPANSION_AUTHS = new Authorizations("ct-a", "b-ct", "not-b-ct");
+
+    public static Authorizations getTestAuths() {
+        return TEST_AUTHS;
+    }
+
+    public static Authorizations getExpansionAuths() {
+        return EXPANSION_AUTHS;
+    }
+
+    @Override
+    public String getSecurityMarkingFieldNames() {
+        return ""; // TODO: while the ssdeep data has security markings, we don't use them in the tests yet.
+    }
+
+    @Override
+    public String getSecurityMarkingFieldDomains() {
+        return MarkingFunctions.Default.COLUMN_VISIBILITY;
+    }
+
+    @Override
+    public String toString() {
+        return this.getClass().getSimpleName() + "{" + super.toString() + "}";
+    }
+}
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepFields.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepFields.java
new file mode 100644
index 00000000000..fe13198aedf
--- /dev/null
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepFields.java
@@ -0,0 +1,40 @@
+package datawave.query.tables.ssdeep.testframework;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+import datawave.query.testframework.AbstractFields;
+
+/**
+ * Base field configuration settings for the data in the ssdeep-data CSV file. The default settings are:
+ * 
      
+ * 
    • forward index: md5, checksum ssdeep
      • 
+ * 
    • indexonly: none
      • 
+ * 
    • reverse: none
      • 
+ * 
    • multivalue fields: none
      • 
+ * 
    • composite index: none
      • 
+ * 
    • virtual fields: none
      • 
+ * 
    
+ */
+public class SSDeepFields extends AbstractFields {
+
+    private static final Collection index = Arrays.asList("CHECKSUM_SSDEEP", SSDeepDataType.SSDeepField.EVENT_ID.name(),
+                    SSDeepDataType.SSDeepField.MD5.name());
+    private static final Collection indexOnly = new HashSet<>();
+    private static final Collection reverse = new HashSet<>();
+    private static final Collection multivalue = Arrays.asList();
+
+    private static final Collection> composite = new HashSet<>();
+    private static final Collection> virtual = new HashSet<>();
+
+    public SSDeepFields() {
+        super(index, indexOnly, reverse, multivalue, composite, virtual);
+    }
+
+    @Override
+    public String toString() {
+        return this.getClass().getSimpleName() + "{" + super.toString() + "}";
+    }
+}
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepQueryTestTableHelper.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepQueryTestTableHelper.java
new file mode 100644
index 00000000000..c976f94b57b
--- /dev/null
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/testframework/SSDeepQueryTestTableHelper.java
@@ -0,0 +1,65 @@
+package datawave.query.tables.ssdeep.testframework;
+
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.deltaspike.core.api.jmx.Table;
+import org.apache.log4j.Logger;
+
+import datawave.helpers.PrintUtility;
+import datawave.ingest.mapreduce.handler.ssdeep.SSDeepIndexHandler;
+import datawave.ingest.table.config.SSDeepIndexTableConfigHelper;
+import datawave.query.MockAccumuloRecordWriter;
+import datawave.query.QueryTestTableHelper;
+import datawave.query.RebuildingScannerTestHelper;
+
+public class SSDeepQueryTestTableHelper extends QueryTestTableHelper {
+    public static final String SSDEEP_INDEX_TABLE_NAME = SSDeepIndexHandler.DEFAULT_SSDEEP_INDEX_TABLE_NAME;
+
+    public SSDeepQueryTestTableHelper(AccumuloClient client, Logger log)
+                    throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException {
+        super(client, log);
+    }
+
+    public SSDeepQueryTestTableHelper(String instanceName, Logger log)
+                    throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException {
+        this(instanceName, log, RebuildingScannerTestHelper.TEARDOWN.EVERY_OTHER, RebuildingScannerTestHelper.INTERRUPT.EVERY_OTHER);
+    }
+
+    public SSDeepQueryTestTableHelper(String instanceName, Logger log, RebuildingScannerTestHelper.TEARDOWN teardown,
+                    RebuildingScannerTestHelper.INTERRUPT interrupt)
+                    throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException {
+        super(instanceName, log, teardown, interrupt);
+    }
+
+    @Override
+    public void printTables(Authorizations auths) throws TableNotFoundException {
+        super.printTables(auths);
+        PrintUtility.printTable(client, auths, SSDEEP_INDEX_TABLE_NAME);
+    }
+
+    @Override
+    public void dumpTables(Authorizations auths) throws TableNotFoundException {
+        super.dumpTables(auths);
+        dumpTable(SSDEEP_INDEX_TABLE_NAME, auths);
+    }
+
+    @Override
+    protected void createTables() throws AccumuloSecurityException, AccumuloException, TableNotFoundException, TableExistsException {
+        super.createTables();
+
+        TableOperations tops = client.tableOperations();
+        deleteAndCreateTable(tops, SSDEEP_INDEX_TABLE_NAME);
+    }
+
+    @Override
+    public void configureTables(MockAccumuloRecordWriter writer) throws AccumuloSecurityException, AccumuloException, TableNotFoundException {
+        super.configureTables(writer);
+
+        configureAShardRelatedTable(writer, new SSDeepIndexTableConfigHelper(), SSDeepIndexHandler.SSDEEP_INDEX_TABLE_NAME, SSDEEP_INDEX_TABLE_NAME);
+    }
+}
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/util/SSDeepTestUtil.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/util/SSDeepTestUtil.java
new file mode 100644
index 00000000000..bdc7670ffa6
--- /dev/null
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/util/SSDeepTestUtil.java
@@ -0,0 +1,158 @@
+package datawave.query.tables.ssdeep.util;
+
+import static org.junit.Assert.fail;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.log4j.Logger;
+import org.junit.Assert;
+
+import datawave.ingest.mapreduce.handler.ssdeep.SSDeepIndexHandler;
+import datawave.query.tables.ssdeep.SSDeepSimilarityQueryTest;
+import datawave.util.ssdeep.BucketAccumuloKeyGenerator;
+import datawave.util.ssdeep.NGramByteHashGenerator;
+import datawave.util.ssdeep.NGramGenerator;
+import datawave.util.ssdeep.NGramTuple;
+import datawave.webservice.query.result.event.EventBase;
+import datawave.webservice.query.result.event.FieldBase;
+
+public class SSDeepTestUtil {
+
+    private static final Logger log = Logger.getLogger(SSDeepSimilarityQueryTest.class);
+
+    @SuppressWarnings("SpellCheckingInspection")
+    public static String[] TEST_SSDEEPS = {"12288:002r/VG4GjeZHkwuPikQ7lKH5p5H9x1beZHkwulizQ1lK55pGxlXTd8zbW:002LVG4GjeZEXi37l6Br1beZEdic1lmu",
+            "6144:02C3nq73v1kHGhs6y7ppFj93NRW6/ftZTgC6e8o4toHZmk6ZxoXb0ns:02C4cGCLjj9Swfj9koHEk6/Fns",
+            "3072:02irbxzGAFYDMxud7fKg3dXVmbOn5u46Kjnz/G8VYrs123D6pIJLIOSP:02MKlWQ7Sg3d4bO968rm7JO",
+            "3072:03jscyaGAFYDMxud7fKg3dXVmbOn5u46Kjnz/G8VYrs123D6pIJLIOSP:03NLmXR7Sg3d4bO968rm7JO",
+            "3072:03jscyaZZZZZYYYYXXXWWdXVmbOn5u46KjnzWWWXXXXYYYYYYZZZZZZZ:03NLmXR7ZZZYYXW9WXYYZZZ",
+            "48:1aBhsiUw69/UXX0x0qzNkVkydf2klA8a7Z35:155w69MXAlNkmkWTF5", "196608:wEEE+EEEEE0LEEEEEEEEEEREEEEhEEETEEEEEWUEEEJEEEEcEEEEEEEE3EEEEEEN:",
+            "1536:0YgNvw/OmgPgiQeI+25Nh6+RS5Qa8LmbyfAiIRgizy1cBx76UKYbD+iD/RYgNvw6:", "12288:222222222222222222222222222222222:"};
+
+    public static final int BUCKET_COUNT = BucketAccumuloKeyGenerator.DEFAULT_BUCKET_COUNT;
+    public static final int BUCKET_ENCODING_BASE = BucketAccumuloKeyGenerator.DEFAULT_BUCKET_ENCODING_BASE;
+    public static final int BUCKET_ENCODING_LENGTH = BucketAccumuloKeyGenerator.DEFAULT_BUCKET_ENCODING_LENGTH;
+
+    public static void loadSSDeepIndexTextData(AccumuloClient accumuloClient) throws Exception {
+        // configuration
+        String ssdeepTableName = SSDeepIndexHandler.DEFAULT_SSDEEP_INDEX_TABLE_NAME;
+        int ngramSize = NGramGenerator.DEFAULT_NGRAM_SIZE;
+        int minHashSize = NGramGenerator.DEFAULT_MIN_HASH_SIZE;
+
+        // input
+        Stream ssdeepLines = Stream.of(TEST_SSDEEPS);
+
+        // processing
+        final NGramByteHashGenerator nGramGenerator = new NGramByteHashGenerator(ngramSize, BUCKET_COUNT, minHashSize);
+        final BucketAccumuloKeyGenerator accumuloKeyGenerator = new BucketAccumuloKeyGenerator(BUCKET_COUNT, BUCKET_ENCODING_BASE, BUCKET_ENCODING_LENGTH);
+
+        // output
+        BatchWriterConfig batchWriterConfig = new BatchWriterConfig();
+        final BatchWriter bw = accumuloClient.createBatchWriter(ssdeepTableName, batchWriterConfig);
+
+        // operations
+        ssdeepLines.forEach(s -> {
+            try {
+                Iterator> it = nGramGenerator.call(s);
+                while (it.hasNext()) {
+                    ImmutablePair nt = it.next();
+                    ImmutablePair at = accumuloKeyGenerator.call(nt);
+                    Key k = at.getKey();
+                    Mutation m = new Mutation(k.getRow());
+                    ColumnVisibility cv = new ColumnVisibility(k.getColumnVisibility());
+                    m.put(k.getColumnFamily(), k.getColumnQualifier(), cv, k.getTimestamp(), at.getValue());
+                    bw.addMutation(m);
+                }
+                bw.flush();
+            } catch (Exception e) {
+                log.error("Exception loading ssdeep hashes", e);
+                fail("Exception while loading ssdeep hashes: " + e.getMessage());
+            }
+        });
+
+        bw.close();
+    }
+
+    /**
+     * assert that a match exists between the specified query and matching ssdeep and that the match has the expected properties
+     *
+     * @param querySsdeep
+     *            the query ssdeep we expect to find in the match results
+     * @param matchingSsdeep
+     *            the matching ssdeep we expect to find in the match results.
+     * @param matchScore
+     *            the base match score
+     * @param weightedScore
+     *            the weighted match score.
+     * @param observedEvents
+     *            the map of observed events, created by extractObservedEvents on the event list obtained from query execution.
+     */
+    public static void assertSSDeepSimilarityMatch(String querySsdeep, String matchingSsdeep, String matchScore, String weightedScore,
+                    Map> observedEvents) {
+        final Map observedFields = observedEvents.get(querySsdeep + "#" + matchingSsdeep);
+        Assert.assertNotNull("Observed fields was null", observedFields);
+        Assert.assertFalse("Observed fields was unexpectedly empty", observedFields.isEmpty());
+        Assert.assertEquals(matchScore, observedFields.remove("MATCH_SCORE"));
+        Assert.assertEquals(weightedScore, observedFields.remove("WEIGHTED_SCORE"));
+        Assert.assertEquals(querySsdeep, observedFields.remove("QUERY_SSDEEP"));
+        Assert.assertEquals(matchingSsdeep, observedFields.remove("MATCHING_SSDEEP"));
+        Assert.assertTrue("Observed unexpected field(s) in full match: " + observedFields, observedFields.isEmpty());
+    }
+
+    /**
+     * Assert that the results do not contain a match between the specified query and matching ssdeep
+     *
+     * @param querySsdeep
+     *            the query ssdeep we do not expect to find in the match results
+     * @param matchingSsdeep
+     *            the matching ssdeep we do not expect to find in the match results
+     * @param observedEvents
+     *            the map of the observed events, created by extractObservedEvents on the event list obtained from query execution.
+     */
+    public static void assertNoMatch(String querySsdeep, String matchingSsdeep, Map> observedEvents) {
+        final Map observedFields = observedEvents.get(querySsdeep + "#" + matchingSsdeep);
+        Assert.assertTrue("Observed fields was not empty", observedFields.isEmpty());
+
+    }
+
+    /** Extract the events from a set of results into an easy to manage data structure for validation */
+    @SuppressWarnings({"rawtypes", "unchecked"})
+    public static Map> extractObservedEvents(List events) {
+        int eventCount = events.size();
+        Map> observedEvents = new HashMap<>();
+        if (eventCount > 0) {
+            for (EventBase e : events) {
+                Map observedFields = new HashMap<>();
+                String querySsdeep = "UNKNOWN_QUERY";
+                String matchingSsdeep = "UNKNOWN_MATCH";
+
+                List fields = e.getFields();
+                for (FieldBase f : fields) {
+                    if (f.getName().equals("QUERY_SSDEEP")) {
+                        querySsdeep = f.getValueString();
+                    }
+                    if (f.getName().equals("MATCHING_SSDEEP")) {
+                        matchingSsdeep = f.getValueString();
+                    }
+                    observedFields.put(f.getName(), f.getValueString());
+                }
+
+                String eventKey = querySsdeep + "#" + matchingSsdeep;
+                observedEvents.put(eventKey, observedFields);
+            }
+        }
+        return observedEvents;
+    }
+}
diff --git a/warehouse/query-core/src/test/java/datawave/query/testframework/AccumuloSetup.java b/warehouse/query-core/src/test/java/datawave/query/testframework/AccumuloSetup.java
index 64401ca23c8..f08d3a6f1aa 100644
--- a/warehouse/query-core/src/test/java/datawave/query/testframework/AccumuloSetup.java
+++ b/warehouse/query-core/src/test/java/datawave/query/testframework/AccumuloSetup.java
@@ -38,7 +38,6 @@
 import org.junit.Assert;
 import org.junit.rules.ExternalResource;
 
-import datawave.helpers.PrintUtility;
 import datawave.ingest.config.RawRecordContainerImpl;
 import datawave.ingest.data.RawRecordContainer;
 import datawave.ingest.input.reader.event.EventSequenceFileRecordReader;
@@ -47,7 +46,6 @@
 import datawave.query.MockAccumuloRecordWriter;
 import datawave.query.QueryTestTableHelper;
 import datawave.query.RebuildingScannerTestHelper;
-import datawave.util.TableName;
 
 public class AccumuloSetup extends ExternalResource {
 
@@ -166,11 +164,46 @@ public AccumuloClient loadTables(final Logger parentLog) throws AccumuloExceptio
      *            log of parent
      * @return connector to Accumulo
      * @throws AccumuloException
-     *             , AccumuloSecurityException, IOException, InterruptedException, TableExistsException, TableNotFoundException Accumulo error conditions
+     *             Accumulo error conditions
+     * @throws AccumuloSecurityException
+     *             Accumulo error conditions
+     * @throws IOException
+     *             Accumulo error conditions
+     * @throws InterruptedException
+     *             Accumulo error conditions
+     * @throws TableExistsException
+     *             Accumulo error conditions
+     * @throws TableNotFoundException
+     *             Accumulo error conditions
      */
     public AccumuloClient loadTables(final Logger parentLog, final RebuildingScannerTestHelper.TEARDOWN teardown,
                     RebuildingScannerTestHelper.INTERRUPT interrupt) throws AccumuloException, AccumuloSecurityException, IOException, InterruptedException,
                     TableExistsException, TableNotFoundException, URISyntaxException {
+        final QueryTestTableHelper tableHelper = new QueryTestTableHelper(this.getClass().getName(), parentLog, teardown, interrupt);
+        return this.loadTables(tableHelper);
+    }
+
+    /**
+     * Creates the Accumulo shard ids and ingests the data into the tables. Uses a CSV file for loading test data.
+     *
+     * @param tableHelper
+     *            the table helper to use
+     * @return connector to Accumulo
+     * @throws AccumuloException
+     *             Accumulo error conditions
+     * @throws AccumuloSecurityException
+     *             Accumulo error conditions
+     * @throws IOException
+     *             Accumulo error conditions
+     * @throws InterruptedException
+     *             Accumulo error conditions
+     * @throws TableExistsException
+     *             Accumulo error conditions
+     * @throws TableNotFoundException
+     *             Accumulo error conditions
+     */
+    public AccumuloClient loadTables(final QueryTestTableHelper tableHelper) throws AccumuloException, AccumuloSecurityException, IOException,
+                    InterruptedException, TableExistsException, TableNotFoundException, URISyntaxException {
         log.debug("------------- loadTables -------------");
 
         if (this.fileFormat != FileType.GROUPING) {
@@ -178,7 +211,6 @@ public AccumuloClient loadTables(final Logger parentLog, final RebuildingScanner
             Assert.assertFalse("data types have not been specified", this.dataTypes.isEmpty());
         }
 
-        QueryTestTableHelper tableHelper = new QueryTestTableHelper(AccumuloSetup.class.getName(), parentLog, teardown, interrupt);
         final AccumuloClient client = tableHelper.client;
         tableHelper.configureTables(this.recordWriter);
 
@@ -191,15 +223,7 @@ public AccumuloClient loadTables(final Logger parentLog, final RebuildingScanner
             }
         }
 
-        PrintUtility.printTable(client, auths, QueryTestTableHelper.METADATA_TABLE_NAME);
-        PrintUtility.printTable(client, auths, TableName.SHARD);
-        PrintUtility.printTable(client, auths, TableName.SHARD_INDEX);
-        PrintUtility.printTable(client, auths, TableName.SHARD_RINDEX);
-
-        // TODO: elsewhere?
-        PrintUtility.printTable(client, auths, QueryTestTableHelper.FACET_TABLE_NAME);
-        PrintUtility.printTable(client, auths, QueryTestTableHelper.FACET_METADATA_TABLE_NAME);
-        PrintUtility.printTable(client, auths, QueryTestTableHelper.FACET_HASH_TABLE_NAME);
+        tableHelper.printTables(auths);
 
         return client;
     }
diff --git a/warehouse/query-core/src/test/java/datawave/query/testframework/BaseRawData.java b/warehouse/query-core/src/test/java/datawave/query/testframework/BaseRawData.java
index 206fe30dde5..a825c44dd74 100644
--- a/warehouse/query-core/src/test/java/datawave/query/testframework/BaseRawData.java
+++ b/warehouse/query-core/src/test/java/datawave/query/testframework/BaseRawData.java
@@ -81,8 +81,7 @@ public BaseRawData(final String datatype, final String[] fields, final List hdrs = getHeaders();
-        // ensure headers match field input
-        Assert.assertEquals(hdrs.size(), fields.length);
+        Assert.assertTrue("The number of fields must be equal or greater than the number of headers", fields.length >= hdrs.size());
 
         for (int n = 0; n < hdrs.size(); n++) {
             String header = hdrs.get(n);
@@ -125,6 +124,19 @@ public void processFields(final String datatype, final String[] fields) {
                 this.event.put(key, values);
             }
         }
+
+        if (fields.length > hdrs.size() && hasExtraFields()) {
+            // extra fields are in the form of fieldName=fieldValue.
+            for (int i = hdrs.size(); i < fields.length; i++) {
+                String value = fields[i];
+            }
+        } else if (fields.length != hdrs.size()) {
+            Assert.fail("The number of fields did not equal the number of headers and hasExtraFields() returned false indicating that there were no extra fields expected");
+        }
+    }
+
+    protected boolean hasExtraFields() {
+        return false;
     }
 
     /**
diff --git a/warehouse/query-core/src/test/resources/input/ssdeep-data.csv b/warehouse/query-core/src/test/resources/input/ssdeep-data.csv
new file mode 100644
index 00000000000..aa1fa009a61
--- /dev/null
+++ b/warehouse/query-core/src/test/resources/input/ssdeep-data.csv
@@ -0,0 +1,14 @@
+2020-10-31 01:27:37,HAR1401.pdf-1-att-1.png,NONE,,7549,PNG,e19a4fec27ceed3152f939b7effccf1e,e6d8136560e1c163b03ff47d0d285f70e431e24b,9867d8380ebfbb12e0e6c303f96e413f1caf0fc85ef94f5592b91f960f2499b8,2020-10-31 01:27:36,B>>>>G,FILE_DATE=2020-10-31 01:27:36,FILE_NAME=HAR1401.pdf-1-att-1.png,CHECKSUM_SSDEEP=192:1sSAdFXYvPToUCpabe0qHEbM0NkaA80W31bixZzS7:1r4X6oU6ZHEbp1biW7,IMAGEHEIGHT=126,IMAGEWIDTH=126,PARENT_FILETYPE=PNG,"ACCESS_CONTROLS=MON:JULY"
+2020-10-31 01:26:25,AAR1202.pdf-1-att-1.png,NONE,,7549,PNG,e19a4fec27ceed3152f939b7effccf1e,e6d8136560e1c163b03ff47d0d285f70e431e24b,9867d8380ebfbb12e0e6c303f96e413f1caf0fc85ef94f5592b91f960f2499b8,2020-10-31 01:18:19,A>>E>>I,FILE_DATE=2020-10-31 01:18:19,FILE_NAME=AAR1202.pdf-1-att-1.png,CHECKSUM_SSDEEP=192:1sSAdFXYvPToUCpabe0qHEbM0NkaA80W31bixZzS7:1r4X6oU6ZHEbp1biW7,IMAGEHEIGHT=126,IMAGEWIDTH=126,PARENT_FILETYPE=PNG,"ACCESS_CONTROLS=MON:JANUARY"
+2020-10-31 01:27:36,HAR1302.pdf-1-att-3.png,NONE,,7549,PNG,e19a4fec27ceed3152f939b7effccf1e,e6d8136560e1c163b03ff47d0d285f70e431e24b,9867d8380ebfbb12e0e6c303f96e413f1caf0fc85ef94f5592b91f960f2499b8,2020-10-31 01:27:35,B>>D>F>>G,FILE_DATE=2020-10-31 01:27:35,FILE_NAME=HAR1302.pdf-1-att-3.png,CHECKSUM_SSDEEP=192:1sSAdFXYvPToUCpabe0qHEbM0NkaA80W31bixZzS7:1r4X6oU6ZHEbp1biW7,IMAGEHEIGHT=126,IMAGEWIDTH=126,PARENT_FILETYPE=PNG,"ACCESS_CONTROLS=COL:BLUE MON:OCTOBER"
+2020-10-31 01:27:38,HAR1401.pdf-1-att-3.png,NONE,,7549,PNG,e19a4fec27ceed3152f939b7effccf1e,e6d8136560e1c163b03ff47d0d285f70e431e24b,9867d8380ebfbb12e0e6c303f96e413f1caf0fc85ef94f5592b91f960f2499b8,2020-10-31 01:27:37,A>>>>G,FILE_DATE=2020-10-31 01:27:37,FILE_NAME=HAR1401.pdf-1-att-3.png,CHECKSUM_SSDEEP=192:1sSAdFXYvPToUCpabe0qHEbM0NkaA80W31bixZzS7:1r4X6oU6ZHEbp1biW7,IMAGEHEIGHT=126,IMAGEWIDTH=126,PARENT_FILETYPE=PNG,"ACCESS_CONTROLS=MON:JANUARY"
+2020-10-31 01:27:48,HAR1601.pdf-1-att-3.png,NONE,,7549,PNG,e19a4fec27ceed3152f939b7effccf1e,e6d8136560e1c163b03ff47d0d285f70e431e24b,9867d8380ebfbb12e0e6c303f96e413f1caf0fc85ef94f5592b91f960f2499b8,2020-10-31 01:27:38,A>>>>H,FILE_DATE=2020-10-31 01:27:38,FILE_NAME=HAR1601.pdf-1-att-3.png,CHECKSUM_SSDEEP=192:1sSAdFXYvPToUCpabe0qHEbM0NkaA80W31bixZzS7:1r4X6oU6ZHEbp1biW7,IMAGEHEIGHT=126,IMAGEWIDTH=126,PARENT_FILETYPE=PNG,
+2020-10-31 01:28:40,HZM1201S.pdf-1-att-1.png,NONE,,7549,PNG,e19a4fec27ceed3152f939b7effccf1e,e6d8136560e1c163b03ff47d0d285f70e431e24b,9867d8380ebfbb12e0e6c303f96e413f1caf0fc85ef94f5592b91f960f2499b8,2020-10-31 01:28:39,C>>E>>H,FILE_DATE=2020-10-31 01:28:39,FILE_NAME=HZM1201S.pdf-1-att-1.png,CHECKSUM_SSDEEP=192:1sSAdFXYvPToUCpabe0qHEbM0NkaA80W31bixZzS7:1r4X6oU6ZHEbp1biW7,IMAGEHEIGHT=126,IMAGEWIDTH=126,PARENT_FILETYPE=PNG,"ACCESS_CONTROLS=COL:BLUE"
+2020-10-31 01:18:25,AAR1403.pdf-1-att-2.jpg,NONE,,17892,JPEG,a3a4312e262a767602d0855797a042ee,05c6dc4223682325ff712d23d8d937d421a2ee17,fbd20e77d8e01cdafc0e1957a41c721d4f989e4bfb6619b68a230aa4488a2b64,2020-10-31 01:18:19,A>>>>G,FILE_DATE=2020-10-31 01:18:19,FILE_NAME=AAR1403.pdf-1-att-2.jpg,CHECKSUM_SSDEEP=384:nv/fP9FmWVMdRFj2aTgSO+u5QT4ZE1PIVS:nDmWOdRFNTTs504cQS,IMAGEHEIGHT=307,IMAGEWIDTH=612,PARENT_FILETYPE=JPEG,"ACCESS_CONTROLS=COL:GREY MON:JANUARY"
+2020-10-31 01:18:25,AAR1503.pdf-1-att-2.jpg,NONE,,17892,JPEG,a3a4312e262a767602d0855797a042ee,05c6dc4223682325ff712d23d8d937d421a2ee17,fbd20e77d8e01cdafc0e1957a41c721d4f989e4bfb6619b68a230aa4488a2b64,2020-10-31 01:18:19,B>>D>>G,FILE_DATE=2020-10-31 01:18:19,FILE_NAME=AAR1503.pdf-1-att-2.jpg,CHECKSUM_SSDEEP=384:nv/fP9FmWVMdRFj2aTgSO+u5QT4ZE1PIVS:nDmWOdRFNTTs504cQS,IMAGEHEIGHT=307,IMAGEWIDTH=612,PARENT_FILETYPE=JPEG,"ACCESS_CONTROLS=COL:BLUE"
+2020-10-31 01:18:55,AAR1603.pdf-1-att-1.jpg,NONE,,17892,JPEG,a3a4312e262a767602d0855797a042ee,05c6dc4223682325ff712d23d8d937d421a2ee17,fbd20e77d8e01cdafc0e1957a41c721d4f989e4bfb6619b68a230aa4488a2b64,2020-10-31 01:18:19,B>>>>H,FILE_DATE=2020-10-31 01:18:19,FILE_NAME=AAR1603.pdf-1-att-1.jpg,CHECKSUM_SSDEEP=384:nv/fP9FmWVMdRFj2aTgSO+u5QT4ZE1PIVS:nDmWOdRFNTTs504cQS,IMAGEHEIGHT=307,IMAGEWIDTH=612,PARENT_FILETYPE=JPEG,"ACCESS_CONTROLS=MON:APRIL"
+2020-10-31 01:18:34,AAR1404.pdf-1-att-2.jpg,NONE,,17892,JPEG,a3a4312e262a767602d0855797a042ee,05c6dc4223682325ff712d23d8d937d421a2ee17,fbd20e77d8e01cdafc0e1957a41c721d4f989e4bfb6619b68a230aa4488a2b64,2020-10-31 01:18:19,A>>>>G,FILE_DATE=2020-10-31 01:18:19,FILE_NAME=AAR1404.pdf-1-att-2.jpg,CHECKSUM_SSDEEP=384:nv/fP9FmWVMdRFj2aTgSO+u5QT4ZE1PIVS:nDmWOdRFNTTs504cQS,IMAGEHEIGHT=307,IMAGEWIDTH=612,PARENT_FILETYPE=JPEG,"ACCESS_CONTROLS=COL:BLUE"
+2017-04-02 13:57:30,AAR7808.pdf-1-att-48,NONE,67096,67096,TIFF,3b3d3551ee44d0ece3045853e1f2ea33,fb86f018806c3fa83b9b4487f26b92c56b342f8d,a75f7d749262b8a5aa0bedf55f8856f2bcf99b0f84f5bf709078d61887e1605d,2017-04-02 13:47:24,C>>D>E>>G,FILE_DATE=2020-10-31 01:20:52,FILE_NAME=AAR7808.pdf-1-att-48.tif,CHECKSUM_SSDEEP=1536:a+xKllrcbBbTKWHKOaA29xbXY58enGBDMrINPzSjAaOQBP:aDldcdDm923GG08jAadV,IMAGEHEIGHT=3200,IMAGEWIDTH=2576,PARENT_FILETYPE=PDF,"ACCESS_CONTROLS=COL:BLUE"
+2017-04-02 13:57:30,AAR7808.pdf-1-att-49,NONE,107984,107984,TIFF,bd3069e6bc2247cc6d62102f246aad97,b8cd63bf4e5bda4ec973c2c85b3a79e700e19657,68bbcf0701beb8cabd7cfa0f1a397d6a8b55fb197f42a844ba88c1494d21c25c,2017-04-02 13:47:24,A>>E>>G,FILE_DATE=2020-10-31 01:20:52,FILE_NAME=AAR7808.pdf-1-att-49.tif,CHECKSUM_SSDEEP=3072:WsvD5V0/eca2XMzkt4VyqW0iwIF4eMfqxU4I7:WsvL3Z2mkyVyqhiD4e83,IMAGEHEIGHT=3200,IMAGEWIDTH=2560,PARENT_FILETYPE=PDF,"ACCESS_CONTROLS=COL:PURPLE"
+2020-10-31 01:35:57,AAR7808.pdf-1-att-49.tif,NONE,,107984,TIFF,bd3069e6bc2247cc6d62102f246aad97,b8cd63bf4e5bda4ec973c2c85b3a79e700e19657,68bbcf0701beb8cabd7cfa0f1a397d6a8b55fb197f42a844ba88c1494d21c25c,2020-10-31 01:20:52,A>>>>G,FILE_DATE=2020-10-31 01:20:52,FILE_NAME=AAR7808.pdf-1-att-49.tif,CHECKSUM_SSDEEP=3072:WsvD5V0/eca2XMzkt4VyqW0iwIF4eMfqxU4I7:WsvL3Z2mkyVyqhiD4e83,IMAGEHEIGHT=3200,IMAGEWIDTH=2560,PARENT_FILETYPE=TIFF,"ACCESS_CONTROLS=COL:BLUE"
+2020-10-31 01:42:33,AAR7808.pdf-1-att-48.tif,NONE,,67096,TIFF,3b3d3551ee44d0ece3045853e1f2ea33,fb86f018806c3fa83b9b4487f26b92c56b342f8d,a75f7d749262b8a5aa0bedf55f8856f2bcf99b0f84f5bf709078d61887e1605d,2020-10-31 01:20:52,A>>>>G,FILE_DATE=2020-10-31 01:20:52,FILE_NAME=AAR7808.pdf-1-att-48.tif,CHECKSUM_SSDEEP=1536:a+xKllrcbBbTKWHKOaA29xbXY58enGBDMrINPzSjAaOQBP:aDldcdDm923GG08jAadV,IMAGEHEIGHT=3200,IMAGEWIDTH=2576,PARENT_FILETYPE=TIFF,"ACCESS_CONTROLS=COL:BLUE"
\ No newline at end of file
diff --git a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/NGramGenerator.java b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/NGramGenerator.java
index 07cfa5f98c3..e8cb81f8b10 100644
--- a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/NGramGenerator.java
+++ b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/NGramGenerator.java
@@ -15,12 +15,27 @@
 /** Generates NGrams of SSDeep Hashes for indexing or query */
 public class NGramGenerator implements Serializable {
 
+    /**
+     * The paper "Optimizing ssDeep for use at Scale" employed an ngram size of 7, and this seems to work well for our purposes based on testing.
+     */
+    public static final int DEFAULT_NGRAM_SIZE = 7;
+
+    /**
+     * the minimum size (chunkSize * chunkLength) required for input hashes, which roughly corresponds to the size of the original object that was hashed. In
+     * this case, we won't generate ngrams for hashes that are based on input smaller than 64 bytes.
+     */
+    public static final int DEFAULT_MIN_HASH_SIZE = 64;
+
     private static final Logger log = LoggerFactory.getLogger(NGramGenerator.class);
 
     final int ngramSize;
     final int maxRepeatedChars;
     final int minHashSize;
 
+    public NGramGenerator() {
+        this(DEFAULT_NGRAM_SIZE, SSDeepHash.DEFAULT_MAX_REPEATED_CHARACTERS, DEFAULT_MIN_HASH_SIZE);
+    }
+
     /**
      * Generate NGrams of the specified size
      *
@@ -28,7 +43,7 @@ public class NGramGenerator implements Serializable {
      *            the size of ngrams to generate.
      */
     public NGramGenerator(int ngramSize) {
-        this(ngramSize, 0, 64);
+        this(ngramSize, SSDeepHash.DEFAULT_MAX_REPEATED_CHARACTERS, DEFAULT_MIN_HASH_SIZE);
     }
 
     /**
diff --git a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepHash.java b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepHash.java
index 95e0f64316d..bb00e6eb3d0 100644
--- a/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepHash.java
+++ b/warehouse/ssdeep-common/src/main/java/datawave/util/ssdeep/SSDeepHash.java
@@ -9,7 +9,10 @@
 /** An Immutable SSDeepHash object */
 public final class SSDeepHash implements Serializable, Comparable {
 
-    /** The default number of max repeated characters to produce when normalizing hashes */
+    /**
+     * The paper on Optimizing ssDeep for use at Scale" suggested that any hash with more than this many repeated characters should have that run of characters
+     * reduced to this number of characters and suggested that 3 was a good number. This works well for our purposes as well.
+     */
     public static final int DEFAULT_MAX_REPEATED_CHARACTERS = 3;
 
     public static final int MIN_CHUNK_SIZE = 3;
diff --git a/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/NGramGeneratorTest.java b/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/NGramGeneratorTest.java
index ee2b370410b..ac522ac4b39 100644
--- a/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/NGramGeneratorTest.java
+++ b/warehouse/ssdeep-common/src/test/java/datawave/util/ssdeep/NGramGeneratorTest.java
@@ -36,7 +36,7 @@ public class NGramGeneratorTest {
 
     @Test
     public void testGenerateNGrams() {
-        int ngramSize = 7;
+        int ngramSize = NGramGenerator.DEFAULT_NGRAM_SIZE;
         NGramGenerator eng = new NGramGenerator(ngramSize);
         SSDeepHash[] hashes = Arrays.stream(GOOD_HASHES).limit(2).map(SSDeepHash::parse).toArray(SSDeepHash[]::new);
         for (int i = 0; i < hashes.length; i++) {
diff --git a/web-services/deploy/configuration/src/main/resources/datawave/query/SSDeepQueryLogicFactory.xml b/web-services/deploy/configuration/src/main/resources/datawave/query/SSDeepQueryLogicFactory.xml
index 628b5bddee3..29c840eee61 100644
--- a/web-services/deploy/configuration/src/main/resources/datawave/query/SSDeepQueryLogicFactory.xml
+++ b/web-services/deploy/configuration/src/main/resources/datawave/query/SSDeepQueryLogicFactory.xml
@@ -6,7 +6,7 @@
     http://www.springframework.org/schema/beans/spring-beans-4.0.xsd">
 
     
-    
+    
         
         
         

From cd3f1e1ceae730a00c6025ffedaffc1f0ce225c8 Mon Sep 17 00:00:00 2001
From: alerman 
Date: Fri, 9 Feb 2024 13:07:20 -0500
Subject: [PATCH 19/50] Update LookupUUIDUtil.java

---
 .../java/datawave/webservice/query/util/LookupUUIDUtil.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web-services/query/src/main/java/datawave/webservice/query/util/LookupUUIDUtil.java b/web-services/query/src/main/java/datawave/webservice/query/util/LookupUUIDUtil.java
index 00a3149315e..ee95c329774 100644
--- a/web-services/query/src/main/java/datawave/webservice/query/util/LookupUUIDUtil.java
+++ b/web-services/query/src/main/java/datawave/webservice/query/util/LookupUUIDUtil.java
@@ -1069,7 +1069,7 @@ private EventQueryResponseBase validatePagedResponse(final BaseQueryResponse res
      *
      * @param lookupContext additional information about the lookup purpose or type.
      *
-     * @param uuidTypeValueTerm A token to evaluate as a possible UUID field/value term
+     * @param possibleUUIDTerm A token to evaluate as a possible UUID field/value term
      *
      * @param logicName The existing assigned query logic name, if any
      *

From 464b2db5e7e1fc0c43805b60369ee8bb8102adec Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Fri, 9 Feb 2024 21:38:52 +0000
Subject: [PATCH 20/50] Fix pruning logic for IngestTypePruningVisitor after
 removal of JexlNodes.children() method

---
 .../visitors/IngestTypePruningVisitor.java    |  4 +-
 .../IngestTypePruningVisitorTest.java         | 68 +++++++++----------
 2 files changed, 36 insertions(+), 36 deletions(-)

diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypePruningVisitor.java b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypePruningVisitor.java
index b182d73da3c..e02f351909e 100644
--- a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypePruningVisitor.java
+++ b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypePruningVisitor.java
@@ -195,7 +195,7 @@ public Object visit(ASTAndNode node, Object data) {
             pruningTypes = ingestTypes;
         }
 
-        for (int i = 0; i < node.jjtGetNumChildren(); i++) {
+        for (int i = node.jjtGetNumChildren() - 1; i >= 0; i--) {
             node.jjtGetChild(i).jjtAccept(this, pruningTypes);
         }
 
@@ -303,7 +303,7 @@ private boolean shouldPrune(Set ingestTypes, Set includes) {
     }
 
     private Set pruneJunction(JexlNode node, Object data) {
-        for (int i = 0; i < node.jjtGetNumChildren(); i++) {
+        for (int i = node.jjtGetNumChildren() - 1; i >= 0; i--) {
             node.jjtGetChild(i).jjtAccept(this, data);
         }
         return Collections.emptySet();
diff --git a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/IngestTypePruningVisitorTest.java b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/IngestTypePruningVisitorTest.java
index 323d1e970ec..e5f246b2577 100644
--- a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/IngestTypePruningVisitorTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/IngestTypePruningVisitorTest.java
@@ -1,13 +1,13 @@
 package datawave.query.jexl.visitors;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import org.apache.commons.jexl3.parser.ASTJexlScript;
 import org.apache.log4j.Logger;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
+import org.junit.BeforeClass;
+import org.junit.Test;
 
 import datawave.data.type.LcType;
 import datawave.query.jexl.JexlASTHelper;
@@ -21,7 +21,7 @@ public class IngestTypePruningVisitorTest {
     private static final TypeMetadata typeMetadata = new TypeMetadata();
     private final ASTValidator validator = new ASTValidator();
 
-    @BeforeAll
+    @BeforeClass
     public static void setup() {
         typeMetadata.put("A", "ingestType1", LcType.class.getTypeName());
         typeMetadata.put("A", "ingestType2", LcType.class.getTypeName());
@@ -36,7 +36,7 @@ public static void setup() {
     }
 
     @Test
-    void testNoOps() {
+    public void testNoOps() {
         //  @formatter:off
         String[] queries = {
                         "A == '1' || B == '2'",
@@ -51,7 +51,7 @@ void testNoOps() {
 
     // test cases for no pruning, multiple node types
     @Test
-    void testNoOpsWithMultipleLeafTypes() {
+    public void testNoOpsWithMultipleLeafTypes() {
         //  @formatter:off
         String[] queries = {
                         "A == '1' && B == '2'",
@@ -74,7 +74,7 @@ void testNoOpsWithMultipleLeafTypes() {
 
     // case where two nodes do not share an ingest type
     @Test
-    void testEmptyIntersection() {
+    public void testEmptyIntersection() {
         //  @formatter:off
         String[] queries = {
                         "A == '1' && C == '3'",
@@ -100,7 +100,7 @@ void testEmptyIntersection() {
     // ingestType 1 = A, B
     // ingestType 2 = C
     @Test
-    void testPruneNestedUnion() {
+    public void testPruneNestedUnion() {
         // prune C term
         String query = "A == '1' && (B == '2' || C == '3')";
         String expected = "A == '1' && B == '2'";
@@ -120,7 +120,7 @@ void testPruneNestedUnion() {
     // ingestType 1 = A, B
     // ingestType 2 = C
     @Test
-    void testPruneComplexNestedUnion() {
+    public void testPruneComplexNestedUnion() {
         // double nested C term pruned
         String query = "A == '1' && (B == '2' || (C == '3' && C == '5'))";
         String expected = "A == '1' && B == '2'";
@@ -138,14 +138,14 @@ void testPruneComplexNestedUnion() {
     }
 
     @Test
-    void testOtherComplexNestedUnion() {
+    public void testOtherComplexNestedUnion() {
         // doesn't matter how complex the nesting is, C term should drive pruning
         String query = "C == '1' && (B == '2' || B == '3' || (A == '4' && A == '5'))";
         test(query, null);
     }
 
     @Test
-    void testDoubleNestedPruning() {
+    public void testDoubleNestedPruning() {
         // base case, should be fine
         String query = "(A == '1' || B == '2') && (A == '3' || B == '4')";
         test(query, query);
@@ -160,7 +160,7 @@ void testDoubleNestedPruning() {
     }
 
     @Test
-    void testDoubleNestedUnionWithRangeStreamPruning() {
+    public void testDoubleNestedUnionWithRangeStreamPruning() {
         // this case demonstrates how a top level query could pass ingest type pruning
         // but still get modified by range stream pruning. In some cases further pruning
         // by this visitor would be necessary.
@@ -195,7 +195,7 @@ void testDoubleNestedUnionWithRangeStreamPruning() {
     }
 
     @Test
-    void testOverlappingExclusions() {
+    public void testOverlappingExclusions() {
         TypeMetadata metadata = new TypeMetadata();
         metadata.put("A", "ingestType1", LcType.class.getTypeName());
         metadata.put("A", "ingestType2", LcType.class.getTypeName());
@@ -217,7 +217,7 @@ void testOverlappingExclusions() {
     }
 
     @Test
-    void testYetAnotherComplexNestedUnion() {
+    public void testYetAnotherComplexNestedUnion() {
         TypeMetadata metadata = new TypeMetadata();
         metadata.put("A", "ingestType1", LcType.class.getTypeName());
         metadata.put("B", "ingestType1", LcType.class.getTypeName());
@@ -245,7 +245,7 @@ void testYetAnotherComplexNestedUnion() {
     }
 
     @Test
-    void testIntersectionsWithNonIndexedFields() {
+    public void testIntersectionsWithNonIndexedFields() {
         //  @formatter:off
         String[] queries = {
                         //  D term is not indexed
@@ -265,7 +265,7 @@ void testIntersectionsWithNonIndexedFields() {
     }
 
     @Test
-    void testIntersectionsWithIncompleteUnions() {
+    public void testIntersectionsWithIncompleteUnions() {
         //  @formatter:off
         String[] queries = {
                         "A == '1' && (B == 2 || filter:includeRegex(D, 'value.*'))",
@@ -279,7 +279,7 @@ void testIntersectionsWithIncompleteUnions() {
     }
 
     @Test
-    void testIntersectionsWithQueryFunctions() {
+    public void testIntersectionsWithQueryFunctions() {
         // each function type
 
         //  @formatter:off
@@ -301,7 +301,7 @@ void testIntersectionsWithQueryFunctions() {
     }
 
     @Test
-    void testIntersectionsWithMarkers() {
+    public void testIntersectionsWithMarkers() {
         // all marker node types
         //  @formatter:off
         String[] queries = {
@@ -340,7 +340,7 @@ void testIntersectionsWithMarkers() {
     }
 
     @Test
-    void testMultiFieldedMarkers() {
+    public void testMultiFieldedMarkers() {
         // case 1: delayed intersection of non-intersecting ingestTypes should remove itself
         String query = "((_Delayed_ = true) && (A == '1' && C == '2'))";
         test(query, null);
@@ -359,7 +359,7 @@ void testMultiFieldedMarkers() {
     }
 
     @Test
-    void testDelayedBoundedMarker() {
+    public void testDelayedBoundedMarker() {
         String query = "((_Delayed_ = true) && ((_Bounded_ = true) && (A > '2' && A < '4')))";
         test(query, query);
 
@@ -372,7 +372,7 @@ void testDelayedBoundedMarker() {
     }
 
     @Test
-    void testDelayedEvaluationOnlyMarker() {
+    public void testDelayedEvaluationOnlyMarker() {
         String query = "((_Delayed_ = true) && ((_Eval_ = true) && (A == '1')))";
         test(query, query);
 
@@ -382,7 +382,7 @@ void testDelayedEvaluationOnlyMarker() {
     }
 
     @Test
-    void testDelayedListMarker() {
+    public void testDelayedListMarker() {
         String query = "((_Delayed_ = true) && ((_List_ = true) && ((id = 'some-bogus-id') && (field = 'A') && (params = '{\"values\":[\"a\",\"b\",\"c\"]}'))))";
         test(query, query);
 
@@ -392,7 +392,7 @@ void testDelayedListMarker() {
     }
 
     @Test
-    void testDelayedTermMarker() {
+    public void testDelayedTermMarker() {
         String query = "((_Delayed_ = true) && ((_Term_ = true) && (A =~ 'ba.*')))";
         test(query, query);
 
@@ -402,7 +402,7 @@ void testDelayedTermMarker() {
     }
 
     @Test
-    void testDelayedValueMarker() {
+    public void testDelayedValueMarker() {
         String query = "((_Delayed_ = true) && ((_Value_ = true) && (A =~ 'ba.*' && B =~ 'ba.*')))";
         test(query, query);
 
@@ -416,13 +416,13 @@ void testDelayedValueMarker() {
     }
 
     @Test
-    void testMultiFieldedFunctions() {
+    public void testMultiFieldedFunctions() {
         String query = "A == '1' && filter:compare(A,'==','ANY','C')";
         test(query, query);
     }
 
     @Test
-    void testEvaluationOnlyField() {
+    public void testEvaluationOnlyField() {
         // evaluation only fields are not guaranteed to have an 'e' column in
         // the datawave metadata table. In this case the Z term has no entry.
         String query = "A == '1' && Z == '2'";
@@ -430,19 +430,19 @@ void testEvaluationOnlyField() {
     }
 
     @Test
-    void testPruneNegation() {
+    public void testPruneNegation() {
         String query = "A == '1' || !((_Delayed_ = true) && (A == '1' && C == '2'))";
         test(query, "A == '1'");
     }
 
     @Test
-    void testFullyPrunedTree() {
+    public void testFullyPrunedTree() {
         String query = "(false)";
         test(query, query);
     }
 
     @Test
-    void testIdentifiers() {
+    public void testIdentifiers() {
         String query = "A == '1' && $123 == '123'";
         test(query, query);
 
@@ -451,13 +451,13 @@ void testIdentifiers() {
     }
 
     @Test
-    void testArithmetic() {
+    public void testArithmetic() {
         String query = "A == '1' && 1 + 1 == 3";
         test(query, query);
     }
 
     @Test
-    void testPruneNestedMarker() {
+    public void testPruneNestedMarker() {
         TypeMetadata metadata = new TypeMetadata();
         metadata.put("A", "ingestType1", LcType.class.getTypeName());
         metadata.put("A", "ingestType2", LcType.class.getTypeName());
@@ -498,7 +498,7 @@ private void test(String query, String expected, TypeMetadata metadata) {
 
             ASTJexlScript expectedScript = JexlASTHelper.parseAndFlattenJexlQuery(expected);
             TreeEqualityVisitor.Comparison comparison = TreeEqualityVisitor.checkEquality(expectedScript, pruned);
-            assertTrue(comparison.isEqual(), "Jexl tree comparison failed with reason: " + comparison.getReason());
+            assertTrue("Jexl tree comparison failed with reason: " + comparison.getReason(), comparison.isEqual());
 
         } catch (Exception e) {
             e.printStackTrace();

From c046a456b7aeec4b164dd2aaeaeeb68279e016fa Mon Sep 17 00:00:00 2001
From: Drew Farris 
Date: Thu, 15 Feb 2024 12:52:50 -0500
Subject: [PATCH 21/50] Re-introduce RebuildingScannerTestHelper to
 QueryTestTableHelper (#2279)

* @jwomeara observed that the `QueryTestTableHelper` was no longer using the `RebuildingScannerTestHelper`, and was using a regular `InMemoryInstance` instead, so we were not testing teardowns and interrupts.
* Updated `RebuildingScannerTestHelper` to use `AccumuloClient` instead of `AccumuloConnection` incorporating changes from the `feature/queryMicroservices` branch
* The only test broken by this is the `SSDeepIngestQueryTest`, so `@Ignore`-ing for now until that test can be fixed by me.

Co-authored-by: Whitney O'Meara 
---
 .../datawave/query/QueryTestTableHelper.java  |   9 +-
 .../query/RebuildingScannerTestHelper.java    | 139 ++++--------------
 .../tables/ssdeep/SSDeepIngestQueryTest.java  |   2 +
 3 files changed, 37 insertions(+), 113 deletions(-)

diff --git a/warehouse/query-core/src/test/java/datawave/query/QueryTestTableHelper.java b/warehouse/query-core/src/test/java/datawave/query/QueryTestTableHelper.java
index 72888245c57..03ded2685b1 100644
--- a/warehouse/query-core/src/test/java/datawave/query/QueryTestTableHelper.java
+++ b/warehouse/query-core/src/test/java/datawave/query/QueryTestTableHelper.java
@@ -12,6 +12,7 @@
 import org.apache.accumulo.core.client.TableExistsException;
 import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Value;
@@ -21,12 +22,9 @@
 import org.apache.hadoop.io.Text;
 import org.apache.log4j.Logger;
 
-import datawave.accumulo.inmemory.InMemoryAccumuloClient;
 import datawave.accumulo.inmemory.InMemoryInstance;
 import datawave.helpers.PrintUtility;
-import datawave.ingest.mapreduce.handler.facet.FacetHandler;
 import datawave.ingest.mapreduce.handler.shard.ShardedDataTypeHandler;
-import datawave.ingest.table.config.FacetTableConfigHelper;
 import datawave.ingest.table.config.MetadataTableConfigHelper;
 import datawave.ingest.table.config.ShardTableConfigHelper;
 import datawave.ingest.table.config.TableConfigHelper;
@@ -64,7 +62,7 @@ public QueryTestTableHelper(String instanceName, Logger log, RebuildingScannerTe
                     throws AccumuloSecurityException, AccumuloException, TableExistsException, TableNotFoundException {
         // create mock instance and connector
         InMemoryInstance i = new InMemoryInstance(instanceName);
-        this.client = new InMemoryAccumuloClient("root", i);
+        this.client = RebuildingScannerTestHelper.getClient(i, "root", new PasswordToken(""), teardown, interrupt);
         this.log = log;
 
         createTables();
@@ -82,7 +80,6 @@ public void dumpTables(Authorizations auths) throws TableNotFoundException {
     }
 
     public void dumpTable(String table, Authorizations auths) throws TableNotFoundException {
-        TableOperations tops = client.tableOperations();
         Scanner scanner = client.createScanner(table, auths);
         Iterator> iterator = scanner.iterator();
         System.out.println("*************** " + table + " ********************");
@@ -121,7 +118,7 @@ protected void deleteAndCreateTable(TableOperations tops, String tableName)
     }
 
     /**
-     * Configures all of the default tables and associates a {@link BatchWriterConfig} object for ach table.
+     * Configures all the default tables and associates a {@link BatchWriterConfig} object for ach table.
      *
      * @param writer
      *            a mock writer
diff --git a/warehouse/query-core/src/test/java/datawave/query/RebuildingScannerTestHelper.java b/warehouse/query-core/src/test/java/datawave/query/RebuildingScannerTestHelper.java
index 8a9cd2b8e8a..ff5396df4d6 100644
--- a/warehouse/query-core/src/test/java/datawave/query/RebuildingScannerTestHelper.java
+++ b/warehouse/query-core/src/test/java/datawave/query/RebuildingScannerTestHelper.java
@@ -10,29 +10,16 @@
 import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 
+import org.apache.accumulo.core.client.AccumuloClient;
 import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.BatchDeleter;
 import org.apache.accumulo.core.client.BatchScanner;
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.ConditionalWriter;
-import org.apache.accumulo.core.client.ConditionalWriterConfig;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.core.client.IteratorSetting;
-import org.apache.accumulo.core.client.MultiTableBatchWriter;
 import org.apache.accumulo.core.client.Scanner;
 import org.apache.accumulo.core.client.ScannerBase;
 import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.client.admin.InstanceOperations;
-import org.apache.accumulo.core.client.admin.NamespaceOperations;
-import org.apache.accumulo.core.client.admin.ReplicationOperations;
-import org.apache.accumulo.core.client.admin.SecurityOperations;
-import org.apache.accumulo.core.client.admin.TableOperations;
 import org.apache.accumulo.core.client.sample.SamplerConfiguration;
 import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
-import org.apache.accumulo.core.client.security.tokens.PasswordToken;
 import org.apache.accumulo.core.data.ByteSequence;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.PartialKey;
@@ -42,12 +29,10 @@
 import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 import org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException;
 import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.util.ByteBufferUtil;
-import org.apache.accumulo.core.util.TextUtil;
 import org.apache.hadoop.io.Text;
 
+import datawave.accumulo.inmemory.InMemoryAccumuloClient;
 import datawave.accumulo.inmemory.InMemoryBatchScanner;
-import datawave.accumulo.inmemory.InMemoryConnector;
 import datawave.accumulo.inmemory.InMemoryInstance;
 import datawave.accumulo.inmemory.InMemoryScanner;
 import datawave.accumulo.inmemory.InMemoryScannerBase;
@@ -71,7 +56,7 @@ public enum TEARDOWN {
         RANDOM_SANS_CONSISTENCY(RandomTeardownWithoutConsistency.class),
         EVERY_OTHER_SANS_CONSISTENCY(EveryOtherTeardownWithoutConsistency.class);
 
-        private Class tclass;
+        private final Class tclass;
 
         TEARDOWN(Class tclass) {
             this.tclass = tclass;
@@ -93,7 +78,7 @@ public enum INTERRUPT {
         FI_EVERY_OTHER(FiEveryOtherInterrupt.class),
         RANDOM_HIGH(HighRandomInterrupt.class);
 
-        private Class iclass;
+        private final Class iclass;
 
         INTERRUPT(Class iclass) {
             this.iclass = iclass;
@@ -260,24 +245,24 @@ public boolean interrupt(Key key) {
         }
     }
 
-    public static Connector getConnector(InMemoryInstance i, String user, byte[] pass, TEARDOWN teardown, INTERRUPT interrupt)
-                    throws AccumuloException, AccumuloSecurityException {
-        return new RebuildingConnector((InMemoryConnector) (i.getConnector(user, new PasswordToken(pass))), teardown, interrupt);
+    public static AccumuloClient getClient(InMemoryInstance i, String user, byte[] pass, TEARDOWN teardown, INTERRUPT interrupt)
+                    throws AccumuloSecurityException {
+        return new RebuildingAccumuloClient(user, i, teardown, interrupt);
     }
 
-    public static Connector getConnector(InMemoryInstance i, String user, ByteBuffer pass, TEARDOWN teardown, INTERRUPT interrupt)
-                    throws AccumuloException, AccumuloSecurityException {
-        return new RebuildingConnector((InMemoryConnector) (i.getConnector(user, ByteBufferUtil.toBytes(pass))), teardown, interrupt);
+    public static AccumuloClient getClient(InMemoryInstance i, String user, ByteBuffer pass, TEARDOWN teardown, INTERRUPT interrupt)
+                    throws AccumuloSecurityException {
+        return new RebuildingAccumuloClient(user, i, teardown, interrupt);
     }
 
-    public static Connector getConnector(InMemoryInstance i, String user, CharSequence pass, TEARDOWN teardown, INTERRUPT interrupt)
-                    throws AccumuloException, AccumuloSecurityException {
-        return new RebuildingConnector((InMemoryConnector) (i.getConnector(user, TextUtil.getBytes(new Text(pass.toString())))), teardown, interrupt);
+    public static AccumuloClient getClient(InMemoryInstance i, String user, CharSequence pass, TEARDOWN teardown, INTERRUPT interrupt)
+                    throws AccumuloSecurityException {
+        return new RebuildingAccumuloClient(user, i, teardown, interrupt);
     }
 
-    public static Connector getConnector(InMemoryInstance i, String principal, AuthenticationToken token, TEARDOWN teardown, INTERRUPT interrupt)
-                    throws AccumuloException, AccumuloSecurityException {
-        return new RebuildingConnector((InMemoryConnector) (i.getConnector(principal, token)), teardown, interrupt);
+    public static AccumuloClient getClient(InMemoryInstance i, String principal, AuthenticationToken token, TEARDOWN teardown, INTERRUPT interrupt)
+                    throws AccumuloSecurityException {
+        return new RebuildingAccumuloClient(principal, i, teardown, interrupt);
     }
 
     public interface TeardownListener {
@@ -287,14 +272,14 @@ public interface TeardownListener {
     }
 
     public static class RebuildingIterator implements Iterator> {
-        private InMemoryScannerBase baseScanner;
+        private final InMemoryScannerBase baseScanner;
         private Iterator> delegate;
         private final ScannerRebuilder scanner;
         private final TeardownListener teardown;
         private final InterruptListener interruptListener;
         private Map.Entry next = null;
         private Map.Entry lastKey = null;
-        private KryoDocumentDeserializer deserializer = new KryoDocumentDeserializer();
+        private final KryoDocumentDeserializer deserializer = new KryoDocumentDeserializer();
         private boolean initialized = false;
 
         public RebuildingIterator(InMemoryScannerBase baseScanner, ScannerRebuilder scanner, TeardownListener teardown, InterruptListener interruptListener) {
@@ -389,7 +374,7 @@ private void findNext() {
                             }
                         } else if (hasNext) {
                             if (teardown.checkConsistency()) {
-                                // if we are dealing with a final document, then both prebuild and postbuild must be returning a final document
+                                // if we are dealing with a final document, then both pre-build and post-build must be returning a final document
                                 // otherwise the keys need to be identical save for the timestamp
                                 boolean nextFinal = FinalDocumentTrackingIterator.isFinalDocumentKey(next.getKey());
                                 boolean rebuildNextFinal = FinalDocumentTrackingIterator.isFinalDocumentKey(rebuildNext.getKey());
@@ -617,99 +602,39 @@ public void setRanges(Collection ranges) {
         }
     }
 
-    public static class RebuildingConnector extends Connector {
-        private final InMemoryConnector delegate;
+    public static class RebuildingAccumuloClient extends InMemoryAccumuloClient {
         private final TEARDOWN teardown;
         private final INTERRUPT interrupt;
 
-        public RebuildingConnector(InMemoryConnector delegate, TEARDOWN teardown, INTERRUPT interrupt) {
-            this.delegate = delegate;
+        public RebuildingAccumuloClient(String user, InMemoryInstance instance, TEARDOWN teardown, INTERRUPT interrupt) throws AccumuloSecurityException {
+            super(user, instance);
             this.teardown = teardown;
             this.interrupt = interrupt;
         }
 
         @Override
         public BatchScanner createBatchScanner(String s, Authorizations authorizations, int i) throws TableNotFoundException {
-            return new RebuildingBatchScanner((InMemoryBatchScanner) (delegate.createBatchScanner(s, authorizations, i)), teardown, interrupt);
+            return new RebuildingBatchScanner((InMemoryBatchScanner) (super.createBatchScanner(s, authorizations, i)), teardown, interrupt);
         }
 
         @Override
-        @Deprecated
-        public BatchDeleter createBatchDeleter(String s, Authorizations authorizations, int i, long l, long l1, int i1) throws TableNotFoundException {
-            return delegate.createBatchDeleter(s, authorizations, i, l, l1, i1);
+        public BatchScanner createBatchScanner(String s, Authorizations authorizations) throws TableNotFoundException {
+            return new RebuildingBatchScanner((InMemoryBatchScanner) (super.createBatchScanner(s, authorizations)), teardown, interrupt);
         }
 
         @Override
-        public BatchDeleter createBatchDeleter(String s, Authorizations authorizations, int i, BatchWriterConfig batchWriterConfig)
-                        throws TableNotFoundException {
-            return delegate.createBatchDeleter(s, authorizations, i, batchWriterConfig);
-        }
-
-        @Override
-        @Deprecated
-        public BatchWriter createBatchWriter(String s, long l, long l1, int i) throws TableNotFoundException {
-            return delegate.createBatchWriter(s, l, l1, i);
-        }
-
-        @Override
-        public BatchWriter createBatchWriter(String s, BatchWriterConfig batchWriterConfig) throws TableNotFoundException {
-            return delegate.createBatchWriter(s, batchWriterConfig);
-        }
-
-        @Override
-        @Deprecated
-        public MultiTableBatchWriter createMultiTableBatchWriter(long l, long l1, int i) {
-            return delegate.createMultiTableBatchWriter(l, l1, i);
-        }
-
-        @Override
-        public MultiTableBatchWriter createMultiTableBatchWriter(BatchWriterConfig batchWriterConfig) {
-            return delegate.createMultiTableBatchWriter(batchWriterConfig);
+        public BatchScanner createBatchScanner(String s) throws TableNotFoundException, AccumuloSecurityException, AccumuloException {
+            return new RebuildingBatchScanner((InMemoryBatchScanner) (super.createBatchScanner(s)), teardown, interrupt);
         }
 
         @Override
         public Scanner createScanner(String s, Authorizations authorizations) throws TableNotFoundException {
-            return new RebuildingScanner((InMemoryScanner) (delegate.createScanner(s, authorizations)), teardown, interrupt);
-        }
-
-        @Override
-        public ConditionalWriter createConditionalWriter(String s, ConditionalWriterConfig conditionalWriterConfig) throws TableNotFoundException {
-            return delegate.createConditionalWriter(s, conditionalWriterConfig);
-        }
-
-        @Override
-        public Instance getInstance() {
-            return delegate.getInstance();
-        }
-
-        @Override
-        public String whoami() {
-            return delegate.whoami();
-        }
-
-        @Override
-        public TableOperations tableOperations() {
-            return delegate.tableOperations();
-        }
-
-        @Override
-        public NamespaceOperations namespaceOperations() {
-            return delegate.namespaceOperations();
-        }
-
-        @Override
-        public SecurityOperations securityOperations() {
-            return delegate.securityOperations();
-        }
-
-        @Override
-        public InstanceOperations instanceOperations() {
-            return delegate.instanceOperations();
+            return new RebuildingScanner((InMemoryScanner) (super.createScanner(s, authorizations)), teardown, interrupt);
         }
 
         @Override
-        public ReplicationOperations replicationOperations() {
-            return delegate.replicationOperations();
+        public Scanner createScanner(String s) throws TableNotFoundException, AccumuloSecurityException, AccumuloException {
+            return new RebuildingScanner((InMemoryScanner) (super.createScanner(s)), teardown, interrupt);
         }
     }
 
@@ -741,8 +666,8 @@ public void fetchColumnFamily(Text col) {
         }
 
         @Override
-        public void fetchColumn(Text colFam, Text colQual) {
-            delegate.fetchColumn(colFam, colQual);
+        public void fetchColumn(Text colFam, Text colQualifier) {
+            delegate.fetchColumn(colFam, colQualifier);
         }
 
         @Override
diff --git a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java
index e90659b2f07..e657bfa0927 100644
--- a/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/tables/ssdeep/SSDeepIngestQueryTest.java
@@ -19,6 +19,7 @@
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import com.google.common.collect.Sets;
@@ -111,6 +112,7 @@ public SSDeepIngestQueryTest() {
 
     @SuppressWarnings("rawtypes")
     @Test
+    @Ignore
     public void testSSDeepSimilarity() throws Exception {
         log.info("------ testSSDeepSimilarity ------");
         @SuppressWarnings("SpellCheckingInspection")

From 6be206aea4901af797ab63a6cd28302f7d93bf1d Mon Sep 17 00:00:00 2001
From: Whitney O'Meara 
Date: Thu, 15 Feb 2024 21:46:05 +0000
Subject: [PATCH 22/50] Updated metadata-utils submodule

---
 core/utils/metadata-utils | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/utils/metadata-utils b/core/utils/metadata-utils
index 5724fc2b602..d56dbeb3ca9 160000
--- a/core/utils/metadata-utils
+++ b/core/utils/metadata-utils
@@ -1 +1 @@
-Subproject commit 5724fc2b602342424283009515fb1ebd658a1c06
+Subproject commit d56dbeb3ca921450991f44ed93346d55ba8dc3a0

From 81d1cb3a4650a8c412398a45f029284b91e13856 Mon Sep 17 00:00:00 2001
From: Whitney O'Meara 
Date: Thu, 15 Feb 2024 21:46:52 +0000
Subject: [PATCH 23/50] Updated datawave quickstart version in docker compose

---
 docker/docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index 80952e120e0..cac31a23ef0 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -8,7 +8,7 @@ services:
     profiles:
       - quickstart
     command: ["datawave-bootstrap.sh", "--web"]
-    image: datawave/quickstart-compose:6.5.0-SNAPSHOT
+    image: datawave/quickstart-compose:6.7.0-SNAPSHOT
     environment:
       - DW_CONTAINER_HOST=quickstart
       - DW_DATAWAVE_WEB_JAVA_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:8787 -Duser.timezone=GMT -Dfile.encoding=UTF-8 -Djava.net.preferIPv4Stack=true

From 13ef78b6e9b9750e82a6090de2d8b304b9b8b410 Mon Sep 17 00:00:00 2001
From: Whitney O'Meara 
Date: Fri, 16 Feb 2024 21:01:23 +0000
Subject: [PATCH 24/50] Updated uuidTypes for docker compose properties

---
 properties/compose.properties | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/properties/compose.properties b/properties/compose.properties
index aab3bdaed0f..b7adf656b17 100644
--- a/properties/compose.properties
+++ b/properties/compose.properties
@@ -170,28 +170,36 @@ event.query.data.decorators= \
 lookup.uuid.uuidTypes= \
            \
 \n             \
-\n             \
+\n             \
+\n         \
+\n         \
+\n             \
+\n            \
+\n              \
+\n                 \
+\n               \
+\n             \
 \n         \
 \n         \
 \n             \
-\n             \
+\n             \
 \n             \
 \n         \
 \n         \
 \n             \
-\n             \
+\n             \
 \n         \
 \n         \
 \n             \
-\n             \
+\n             \
 \n         \
 \n         \
 \n             \
-\n             \
+\n             \
 \n         \
 \n         \
 \n             \
-\n             \
+\n             \
 \n        
 
 query.metrics.marking=(PUBLIC)

From fad9276cb514cc3afd01c1dca36a8c427c85a028 Mon Sep 17 00:00:00 2001
From: Drew Farris 
Date: Fri, 16 Feb 2024 21:41:39 +0000
Subject: [PATCH 25/50] Corrected the parent pom for ssdeep-common module

---
 warehouse/ssdeep-common/pom.xml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/warehouse/ssdeep-common/pom.xml b/warehouse/ssdeep-common/pom.xml
index 108a9b5aa7e..ee7fd9ef88d 100644
--- a/warehouse/ssdeep-common/pom.xml
+++ b/warehouse/ssdeep-common/pom.xml
@@ -3,9 +3,8 @@
     4.0.0
     
         gov.nsa.datawave
-        datawave-parent
+        datawave-warehouse-parent
         6.7.0-SNAPSHOT
-        ../../pom.xml
     
     datawave-ssdeep-common
     

From 66aa834a7f086eb8a96dcd307329fe8eee37e1f1 Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Tue, 20 Feb 2024 10:33:21 +0000
Subject: [PATCH 26/50] Add config option for reducing field sets per tablet

---
 .../datawave/query/config/ShardQueryConfiguration.java | 10 ++++++++++
 .../java/datawave/query/tables/ShardQueryLogic.java    |  8 ++++++++
 .../query/tables/async/event/VisitorFunction.java      |  2 +-
 .../query/config/ShardQueryConfigurationTest.java      |  2 ++
 .../resources/datawave/query/QueryLogicFactory.xml     |  1 +
 5 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java b/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java
index 23bdb8e1f92..6fbef8a1b4c 100644
--- a/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java
+++ b/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java
@@ -104,6 +104,7 @@ public class ShardQueryConfiguration extends GenericQueryConfiguration implement
     private boolean pruneQueryByIngestTypes = false;
     // should this query reduce the set of fields prior to serialization
     private boolean reduceQueryFields = false;
+    private boolean reduceQueryFieldsPerShard = false;
     private boolean reduceTypeMetadata = false;
     private boolean reduceTypeMetadataPerShard = false;
     private boolean sequentialScheduler = false;
@@ -484,6 +485,7 @@ public ShardQueryConfiguration(ShardQueryConfiguration other) {
         this.setEnforceUniqueTermsWithinExpressions(other.getEnforceUniqueTermsWithinExpressions());
         this.setPruneQueryByIngestTypes(other.getPruneQueryByIngestTypes());
         this.setReduceQueryFields(other.getReduceQueryFields());
+        this.setReduceQueryFieldsPerShard(other.getReduceQueryFieldsPerShard());
         this.setReduceTypeMetadata(other.getReduceTypeMetadata());
         this.setReduceTypeMetadataPerShard(other.getReduceTypeMetadataPerShard());
         this.setParseTldUids(other.getParseTldUids());
@@ -2095,6 +2097,14 @@ public void setReduceQueryFields(boolean reduceQueryFields) {
         this.reduceQueryFields = reduceQueryFields;
     }
 
+    public boolean getReduceQueryFieldsPerShard() {
+        return reduceQueryFieldsPerShard;
+    }
+
+    public void setReduceQueryFieldsPerShard(boolean reduceQueryFieldsPerShard) {
+        this.reduceQueryFieldsPerShard = reduceQueryFieldsPerShard;
+    }
+
     public boolean getReduceTypeMetadata() {
         return reduceTypeMetadata;
     }
diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java
index dbeed1383a9..951e85f4cfe 100644
--- a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java
+++ b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java
@@ -2275,6 +2275,14 @@ public void setReduceQueryFields(boolean reduceQueryFields) {
         this.getConfig().setReduceQueryFields(reduceQueryFields);
     }
 
+    public boolean getReduceQueryFieldsPerShard() {
+        return this.getConfig().getReduceQueryFieldsPerShard();
+    }
+
+    public void setReduceQueryFieldsPerShard(boolean reduceQueryFieldsPerShard) {
+        this.getConfig().setReduceQueryFieldsPerShard(reduceQueryFieldsPerShard);
+    }
+
     public boolean getReduceTypeMetadata() {
         return getConfig().getReduceTypeMetadata();
     }
diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/async/event/VisitorFunction.java b/warehouse/query-core/src/main/java/datawave/query/tables/async/event/VisitorFunction.java
index a0eed9f3d4b..55df1a39d3e 100644
--- a/warehouse/query-core/src/main/java/datawave/query/tables/async/event/VisitorFunction.java
+++ b/warehouse/query-core/src/main/java/datawave/query/tables/async/event/VisitorFunction.java
@@ -303,7 +303,7 @@ public ScannerChunk apply(@Nullable ScannerChunk input) {
 
                     pruneEmptyOptions(newIteratorSetting);
 
-                    if (config.getReduceQueryFields()) {
+                    if (config.getReduceQueryFieldsPerShard()) {
                         reduceQueryFields(script, newIteratorSetting);
                     }
 
diff --git a/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java b/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java
index 65d3131230f..de9a3a5db96 100644
--- a/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java
@@ -135,6 +135,8 @@ public void setUp() throws Exception {
         updatedValues.put("enforceUniqueTermsWithinExpressions", true);
         defaultValues.put("reduceQueryFields", false);
         updatedValues.put("reduceQueryFields", true);
+        defaultValues.put("reduceQueryFieldsPerShard", false);
+        updatedValues.put("reduceQueryFieldsPerShard", true);
         defaultValues.put("reduceTypeMetadata", false);
         updatedValues.put("reduceTypeMetadata", true);
         defaultValues.put("reduceTypeMetadataPerShard", false);
diff --git a/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml b/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml
index b0dc5428a8e..7e2c2bc64cf 100644
--- a/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml
+++ b/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml
@@ -250,6 +250,7 @@
         
         
         
+        
         
         
         

From c6ab9c7d782fb7c29c24da28a54290127f16d5bb Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Fri, 9 Feb 2024 21:38:19 +0000
Subject: [PATCH 27/50] Configure surefire to run JUnit5, Junit4 tests

---
 core/pom.xml                                 | 23 +++++++----
 pom.xml                                      | 40 +++++++++++++++-----
 warehouse/accumulo-extensions/pom.xml        | 11 ++++++
 warehouse/common/pom.xml                     | 10 +++++
 warehouse/core/pom.xml                       | 22 +++++++++++
 warehouse/index-stats/pom.xml                | 10 +++++
 warehouse/ingest-core/pom.xml                | 10 +++++
 warehouse/ingest-csv/pom.xml                 | 10 +++++
 warehouse/ingest-json/pom.xml                | 10 +++++
 warehouse/ingest-nyctlc/pom.xml              | 11 ++++++
 warehouse/ingest-wikipedia/pom.xml           | 10 +++++
 warehouse/ops-tools/config-compare/pom.xml   | 10 +++++
 warehouse/ops-tools/index-validation/pom.xml | 10 +++++
 warehouse/pom.xml                            | 12 ++++++
 warehouse/query-core/pom.xml                 | 10 +++++
 warehouse/ssdeep-common/pom.xml              | 10 +++++
 web-services/atom/pom.xml                    | 10 +++++
 web-services/cached-results/pom.xml          | 10 +++++
 web-services/client/pom.xml                  | 10 +++++
 web-services/common-util/pom.xml             | 10 +++++
 web-services/common/pom.xml                  | 10 +++++
 web-services/map-reduce/pom.xml              | 10 +++++
 web-services/model/pom.xml                   | 10 +++++
 web-services/modification/pom.xml            | 10 +++++
 web-services/query/pom.xml                   | 10 +++++
 web-services/security/pom.xml                | 10 +++++
 26 files changed, 302 insertions(+), 17 deletions(-)

diff --git a/core/pom.xml b/core/pom.xml
index df1b7baeee6..9c40497bdbf 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -18,11 +18,6 @@
         scm:git:ssh://git@fixme/core.git
         HEAD
     
-    
-        4.13.2
-        5.8.2
-        5.8.2
-    
     
         
             
@@ -34,19 +29,16 @@
             
                 org.junit.jupiter
                 junit-jupiter-api
-                ${version.junit.jupiter}
                 test
             
             
                 org.junit.jupiter
                 junit-jupiter-engine
-                ${version.junit.jupiter}
                 test
             
             
                 org.junit.vintage
                 junit-vintage-engine
-                ${version.junit.vintage}
                 test
             
         
@@ -59,16 +51,19 @@
         
             junit
             junit
+            ${version.junit}
             test
         
         
             org.junit.jupiter
             junit-jupiter-engine
+            ${version.junit.jupiter}
             test
         
         
             org.junit.vintage
             junit-vintage-engine
+            ${version.junit.vintage}
             test
         
     
@@ -90,6 +85,18 @@
                         true
                         random
                     
+                    
+                        
+                            org.junit.jupiter
+                            junit-jupiter-engine
+                            ${version.junit.jupiter}
+                        
+                        
+                            org.junit.vintage
+                            junit-vintage-engine
+                            ${version.junit.vintage}
+                        
+                    
                 
             
         
diff --git a/pom.xml b/pom.xml
index 351566a582c..a3886846c89 100644
--- a/pom.xml
+++ b/pom.xml
@@ -85,8 +85,8 @@
         20231013
         1.19.0
         4.13.2
-        5.5.2
-        5.5.2
+        5.10.2
+        5.10.2
         2.7.2b2
         2.20
         2.20
@@ -119,7 +119,7 @@
         5.2.2.RELEASE
         ${version.spring}
         2.9.6
-        3.0.0-M6
+        3.2.5
         0.17.0
         3.1.1.Final
         3.1.1.Final
@@ -1149,6 +1149,18 @@
                 ${version.junit.jupiter}
                 test
                 
+            
+                org.junit.jupiter
+                junit-jupiter-engine
+                ${version.junit.jupiter}
+                test
+            
+            
+                org.junit.vintage
+                junit-vintage-engine
+                ${version.junit.vintage}
+                test
+            
             
                 org.mockito
                 mockito-core
@@ -1471,9 +1483,14 @@
                     
                     
                         
-                            org.apache.maven.surefire
-                            surefire-junit47
-                            ${version.surefire.plugin}
+                            org.junit.jupiter
+                            junit-jupiter-engine
+                            ${version.junit.jupiter}
+                        
+                        
+                            org.junit.vintage
+                            junit-vintage-engine
+                            ${version.junit.vintage}
                         
                     
                     
@@ -1558,9 +1575,14 @@
                     ${version.surefire.plugin}
                     
                         
-                            org.apache.maven.surefire
-                            surefire-junit47
-                            ${version.surefire.plugin}
+                            org.junit.jupiter
+                            junit-jupiter-engine
+                            ${version.junit.jupiter}
+                        
+                        
+                            org.junit.vintage
+                            junit-vintage-engine
+                            ${version.junit.vintage}
                         
                     
                 
diff --git a/warehouse/accumulo-extensions/pom.xml b/warehouse/accumulo-extensions/pom.xml
index 14f445611d2..9cb037953e5 100644
--- a/warehouse/accumulo-extensions/pom.xml
+++ b/warehouse/accumulo-extensions/pom.xml
@@ -42,6 +42,7 @@
         
             junit
             junit
+            ${version.junit}
             test
         
         
@@ -54,6 +55,16 @@
             jmock-junit4
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
     
     
         
diff --git a/warehouse/common/pom.xml b/warehouse/common/pom.xml
index 8991e037ffb..f95de245364 100644
--- a/warehouse/common/pom.xml
+++ b/warehouse/common/pom.xml
@@ -55,5 +55,15 @@
             junit
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-api
+            test
+        
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
     
 
diff --git a/warehouse/core/pom.xml b/warehouse/core/pom.xml
index 9f37e568dbf..dde77c0a59a 100644
--- a/warehouse/core/pom.xml
+++ b/warehouse/core/pom.xml
@@ -198,6 +198,16 @@
             easymock
             test
             
+        
+            org.junit.jupiter
+            junit-jupiter-api
+            test
+        
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
     
     
         
@@ -237,6 +247,18 @@
                         ${project.build.directory}
                     
                 
+                
+                    
+                        org.junit.jupiter
+                        junit-jupiter-engine
+                        ${version.junit.jupiter}
+                    
+                    
+                        org.junit.vintage
+                        junit-vintage-engine
+                        ${version.junit.vintage}
+                    
+                
             
         
     
diff --git a/warehouse/index-stats/pom.xml b/warehouse/index-stats/pom.xml
index 9d1af44b5fc..fd123f15304 100644
--- a/warehouse/index-stats/pom.xml
+++ b/warehouse/index-stats/pom.xml
@@ -59,6 +59,16 @@
             javassist
             test
             
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
     
     
         
diff --git a/warehouse/ingest-core/pom.xml b/warehouse/ingest-core/pom.xml
index ce13224d0e0..3ac8433edc4 100644
--- a/warehouse/ingest-core/pom.xml
+++ b/warehouse/ingest-core/pom.xml
@@ -202,6 +202,16 @@
             javassist
             test
             
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
     
     
         
diff --git a/warehouse/ingest-csv/pom.xml b/warehouse/ingest-csv/pom.xml
index 35b1c2df03c..16e12e17776 100644
--- a/warehouse/ingest-csv/pom.xml
+++ b/warehouse/ingest-csv/pom.xml
@@ -120,6 +120,16 @@
             javassist
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
     
     
         
diff --git a/warehouse/ingest-json/pom.xml b/warehouse/ingest-json/pom.xml
index 5d7c4f1a8a7..3cda7ebbe5f 100644
--- a/warehouse/ingest-json/pom.xml
+++ b/warehouse/ingest-json/pom.xml
@@ -125,6 +125,16 @@
             javassist
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
     
     
         
diff --git a/warehouse/ingest-nyctlc/pom.xml b/warehouse/ingest-nyctlc/pom.xml
index fa661596a31..ae17424c068 100644
--- a/warehouse/ingest-nyctlc/pom.xml
+++ b/warehouse/ingest-nyctlc/pom.xml
@@ -36,6 +36,17 @@
             datawave-query-core
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
+
     
     
         
diff --git a/warehouse/ingest-wikipedia/pom.xml b/warehouse/ingest-wikipedia/pom.xml
index fea39cd417e..6a7b5db56fa 100644
--- a/warehouse/ingest-wikipedia/pom.xml
+++ b/warehouse/ingest-wikipedia/pom.xml
@@ -110,6 +110,16 @@
             javassist
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
     
     
         
diff --git a/warehouse/ops-tools/config-compare/pom.xml b/warehouse/ops-tools/config-compare/pom.xml
index dde7abee524..c49e63faff1 100644
--- a/warehouse/ops-tools/config-compare/pom.xml
+++ b/warehouse/ops-tools/config-compare/pom.xml
@@ -58,5 +58,15 @@
             1.3
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
     
 
diff --git a/warehouse/ops-tools/index-validation/pom.xml b/warehouse/ops-tools/index-validation/pom.xml
index ff10e398236..65fe126998c 100644
--- a/warehouse/ops-tools/index-validation/pom.xml
+++ b/warehouse/ops-tools/index-validation/pom.xml
@@ -79,6 +79,16 @@
             1.3
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
     
     
         
diff --git a/warehouse/pom.xml b/warehouse/pom.xml
index f383da9e34e..c9115a807b1 100644
--- a/warehouse/pom.xml
+++ b/warehouse/pom.xml
@@ -476,6 +476,18 @@
                             file:${project.build.testOutputDirectory}/log4j.properties
                         
                     
+                    
+                        
+                            org.junit.jupiter
+                            junit-jupiter-engine
+                            ${version.junit.jupiter}
+                        
+                        
+                            org.junit.vintage
+                            junit-vintage-engine
+                            ${version.junit.vintage}
+                        
+                    
                     
                         
                             default-test
diff --git a/warehouse/query-core/pom.xml b/warehouse/query-core/pom.xml
index 42a8e1d2c99..20ad75eb873 100644
--- a/warehouse/query-core/pom.xml
+++ b/warehouse/query-core/pom.xml
@@ -350,6 +350,16 @@
             junit-jupiter-api
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
         
             org.mockito
             mockito-core
diff --git a/warehouse/ssdeep-common/pom.xml b/warehouse/ssdeep-common/pom.xml
index ee7fd9ef88d..34299e08780 100644
--- a/warehouse/ssdeep-common/pom.xml
+++ b/warehouse/ssdeep-common/pom.xml
@@ -23,5 +23,15 @@
             junit
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
     
 
diff --git a/web-services/atom/pom.xml b/web-services/atom/pom.xml
index 4a925cf581b..d45ec3571ca 100644
--- a/web-services/atom/pom.xml
+++ b/web-services/atom/pom.xml
@@ -97,6 +97,16 @@
             jboss-transaction-api_1.2_spec
             provided
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
     
     
         ${project.artifactId}
diff --git a/web-services/cached-results/pom.xml b/web-services/cached-results/pom.xml
index de70d9d80cb..bf1bc1691e4 100644
--- a/web-services/cached-results/pom.xml
+++ b/web-services/cached-results/pom.xml
@@ -124,6 +124,16 @@
             javassist
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
     
     
         ${project.artifactId}
diff --git a/web-services/client/pom.xml b/web-services/client/pom.xml
index 587d5f818f7..29cdd50c2a9 100644
--- a/web-services/client/pom.xml
+++ b/web-services/client/pom.xml
@@ -176,6 +176,16 @@
             junit-jupiter-api
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
     
     
         ${project.artifactId}
diff --git a/web-services/common-util/pom.xml b/web-services/common-util/pom.xml
index c75e95b7f1d..c23e05e869c 100644
--- a/web-services/common-util/pom.xml
+++ b/web-services/common-util/pom.xml
@@ -187,6 +187,16 @@
             javassist
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
     
     
         ${project.artifactId}
diff --git a/web-services/common/pom.xml b/web-services/common/pom.xml
index 5df8eaa097d..22ead5adb64 100644
--- a/web-services/common/pom.xml
+++ b/web-services/common/pom.xml
@@ -255,6 +255,16 @@
             javassist
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
         
             org.springframework
             spring-test
diff --git a/web-services/map-reduce/pom.xml b/web-services/map-reduce/pom.xml
index e61c1bd1304..26e0eddf68f 100644
--- a/web-services/map-reduce/pom.xml
+++ b/web-services/map-reduce/pom.xml
@@ -174,6 +174,16 @@
             junit
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
         
             org.springframework
             spring-expression
diff --git a/web-services/model/pom.xml b/web-services/model/pom.xml
index d136098c61c..f9e58f0acb1 100644
--- a/web-services/model/pom.xml
+++ b/web-services/model/pom.xml
@@ -133,6 +133,16 @@
             javassist
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
     
     
         ${project.artifactId}
diff --git a/web-services/modification/pom.xml b/web-services/modification/pom.xml
index 138d81fe4ae..15bfcd97ea8 100644
--- a/web-services/modification/pom.xml
+++ b/web-services/modification/pom.xml
@@ -82,6 +82,16 @@
             junit
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
     
     
         ${project.artifactId}
diff --git a/web-services/query/pom.xml b/web-services/query/pom.xml
index b91638faf34..ba27a7067c4 100644
--- a/web-services/query/pom.xml
+++ b/web-services/query/pom.xml
@@ -230,6 +230,16 @@
             javassist
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
     
     
         ${project.artifactId}
diff --git a/web-services/security/pom.xml b/web-services/security/pom.xml
index b739d4aa839..313d0f389fb 100644
--- a/web-services/security/pom.xml
+++ b/web-services/security/pom.xml
@@ -249,6 +249,16 @@
             weld-core-impl
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
         
             org.springframework
             spring-expression

From 06b9fe2bac09ef9a951ab63548070aa72fd4b8d7 Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Mon, 12 Feb 2024 11:15:09 +0000
Subject: [PATCH 28/50] Add missed module

---
 .../deploy/spring-framework-integration/pom.xml        | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/web-services/deploy/spring-framework-integration/pom.xml b/web-services/deploy/spring-framework-integration/pom.xml
index 52459f2a6e4..6f724e3b4a0 100644
--- a/web-services/deploy/spring-framework-integration/pom.xml
+++ b/web-services/deploy/spring-framework-integration/pom.xml
@@ -100,6 +100,16 @@
             junit
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+            test
+        
         
             org.jboss.arquillian.container
             arquillian-weld-ee-embedded-1.1

From 0d4659055688c821329f31a7d4264b9bfde220e4 Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Mon, 12 Feb 2024 14:07:06 +0000
Subject: [PATCH 29/50] Update comments

---
 .../datawave/query/jexl/visitors/IngestTypePruningVisitor.java  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypePruningVisitor.java b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypePruningVisitor.java
index e02f351909e..7f694788642 100644
--- a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypePruningVisitor.java
+++ b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IngestTypePruningVisitor.java
@@ -195,6 +195,7 @@ public Object visit(ASTAndNode node, Object data) {
             pruningTypes = ingestTypes;
         }
 
+        // must traverse the children in reverse order because this visitor prunes as it visits
         for (int i = node.jjtGetNumChildren() - 1; i >= 0; i--) {
             node.jjtGetChild(i).jjtAccept(this, pruningTypes);
         }
@@ -303,6 +304,7 @@ private boolean shouldPrune(Set ingestTypes, Set includes) {
     }
 
     private Set pruneJunction(JexlNode node, Object data) {
+        // must traverse the children in reverse order because this visitor prunes as it visits
         for (int i = node.jjtGetNumChildren() - 1; i >= 0; i--) {
             node.jjtGetChild(i).jjtAccept(this, data);
         }

From e356252485f814f8e1ce4f431b2d5d53d48e3d17 Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Mon, 12 Feb 2024 17:40:12 +0000
Subject: [PATCH 30/50] Update unit tests to junit5

---
 warehouse/accumulo-extensions/pom.xml         |  1 -
 .../audit/SplitSelectorExtractorTest.java     | 30 ++++----
 .../iterators/TermFrequencyIteratorTest.java  | 46 ++++++-------
 .../iterators/key/util/FiKeyUtilTest.java     | 22 +++---
 .../iterators/key/util/TFKeyUtilTest.java     | 24 +++----
 .../AncestorChildExpansionIteratorTest.java   | 68 +++++++++----------
 .../query/jexl/DatawaveJexlContextTest.java   | 14 ++--
 7 files changed, 102 insertions(+), 103 deletions(-)

diff --git a/warehouse/accumulo-extensions/pom.xml b/warehouse/accumulo-extensions/pom.xml
index 9cb037953e5..a69ab3678c1 100644
--- a/warehouse/accumulo-extensions/pom.xml
+++ b/warehouse/accumulo-extensions/pom.xml
@@ -42,7 +42,6 @@
         
             junit
             junit
-            ${version.junit}
             test
         
         
diff --git a/warehouse/query-core/src/test/java/datawave/audit/SplitSelectorExtractorTest.java b/warehouse/query-core/src/test/java/datawave/audit/SplitSelectorExtractorTest.java
index 3d4cc8cd31d..eb587e3fe76 100644
--- a/warehouse/query-core/src/test/java/datawave/audit/SplitSelectorExtractorTest.java
+++ b/warehouse/query-core/src/test/java/datawave/audit/SplitSelectorExtractorTest.java
@@ -1,23 +1,23 @@
 package datawave.audit;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
 import static org.junit.jupiter.api.Assertions.assertAll;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.util.List;
 
 import org.apache.commons.lang.math.IntRange;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 
 import com.google.common.collect.Lists;
 
 import datawave.webservice.query.QueryImpl;
 
-public class SplitSelectorExtractorTest {
+class SplitSelectorExtractorTest {
 
     @Test
-    public void extractSelectorsLuceneQuery1() {
+    void extractSelectorsLuceneQuery1() {
 
         SplitSelectorExtractor extractor = new SplitSelectorExtractor();
         QueryImpl q = new QueryImpl();
@@ -28,7 +28,7 @@ public void extractSelectorsLuceneQuery1() {
     }
 
     @Test
-    public void extractSelectorsLuceneQuery2() {
+    void extractSelectorsLuceneQuery2() {
 
         SplitSelectorExtractor extractor = new SplitSelectorExtractor();
         extractor.setSeparatorCharacter(";");
@@ -40,7 +40,7 @@ public void extractSelectorsLuceneQuery2() {
     }
 
     @Test
-    public void extractSelectorsLuceneQuery3() {
+    void extractSelectorsLuceneQuery3() {
 
         SplitSelectorExtractor extractor = new SplitSelectorExtractor();
         extractor.setSeparatorCharacter("\0");
@@ -52,7 +52,7 @@ public void extractSelectorsLuceneQuery3() {
     }
 
     @Test
-    public void extractSelectorsLuceneQuery4() {
+    void extractSelectorsLuceneQuery4() {
 
         SplitSelectorExtractor extractor = new SplitSelectorExtractor();
         extractor.setSeparatorParameter("delimiter");
@@ -65,7 +65,7 @@ public void extractSelectorsLuceneQuery4() {
     }
 
     @Test
-    public void rangeTest1() {
+    void rangeTest1() {
         SplitSelectorExtractor extractor = new SplitSelectorExtractor();
         List useSplitRanges = extractor.parseUseSplitsRanges("0");
         //  @formatter:off
@@ -77,7 +77,7 @@ public void rangeTest1() {
     }
 
     @Test
-    public void rangeTest2() {
+    void rangeTest2() {
         SplitSelectorExtractor extractor = new SplitSelectorExtractor();
         List useSplitRanges = extractor.parseUseSplitsRanges("0-2");
         //  @formatter:off
@@ -91,7 +91,7 @@ public void rangeTest2() {
     }
 
     @Test
-    public void rangeTest3() {
+    void rangeTest3() {
         SplitSelectorExtractor extractor = new SplitSelectorExtractor();
         List useSplitRanges = extractor.parseUseSplitsRanges("0-2,4");
         //  @formatter:off
@@ -104,7 +104,7 @@ public void rangeTest3() {
     }
 
     @Test
-    public void rangeTest4() {
+    void rangeTest4() {
         SplitSelectorExtractor extractor = new SplitSelectorExtractor();
         List useSplitRanges = extractor.parseUseSplitsRanges("2,4");
         //  @formatter:off
@@ -117,7 +117,7 @@ public void rangeTest4() {
     }
 
     @Test
-    public void rangeTest5() {
+    void rangeTest5() {
         SplitSelectorExtractor extractor = new SplitSelectorExtractor();
         List useSplitRanges = extractor.parseUseSplitsRanges("2,4,6-");
         //  @formatter:off
@@ -134,7 +134,7 @@ public void rangeTest5() {
     }
 
     @Test
-    public void rangeTest6() {
+    void rangeTest6() {
         SplitSelectorExtractor extractor = new SplitSelectorExtractor();
         List useSplitRanges = extractor.parseUseSplitsRanges(" 2, 4 , 6- ");
         //  @formatter:off
diff --git a/warehouse/query-core/src/test/java/datawave/core/iterators/TermFrequencyIteratorTest.java b/warehouse/query-core/src/test/java/datawave/core/iterators/TermFrequencyIteratorTest.java
index 75b6e9b6797..9db5aa87045 100644
--- a/warehouse/query-core/src/test/java/datawave/core/iterators/TermFrequencyIteratorTest.java
+++ b/warehouse/query-core/src/test/java/datawave/core/iterators/TermFrequencyIteratorTest.java
@@ -1,11 +1,11 @@
 package datawave.core.iterators;
 
 import static datawave.query.Constants.NULL;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
 import static org.junit.jupiter.api.Assertions.assertAll;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.IOException;
 import java.util.AbstractMap.SimpleEntry;
@@ -23,7 +23,7 @@
 import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.io.Text;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Lists;
@@ -32,13 +32,13 @@
 
 import datawave.query.iterator.SortedListKeyValueIterator;
 
-public class TermFrequencyIteratorTest {
+class TermFrequencyIteratorTest {
 
     private final String lowers = "abcdefghijklmnopqrstuvwxyz";
     private final String uppers = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
 
     @Test
-    public void testDocRange_singleKey_parent() throws IOException {
+    void testDocRange_singleKey_parent() throws IOException {
         Key start = new Key("20200314_0", "tf", "datatype\0uid3");
         Key end = new Key("20200314_0", "tf", "datatype\0uid3\uffff");
         Range r = new Range(start, false, end, true);
@@ -62,7 +62,7 @@ public void testDocRange_singleKey_parent() throws IOException {
     }
 
     @Test
-    public void testDocRange_singleKey_parent_multiField() throws IOException {
+    void testDocRange_singleKey_parent_multiField() throws IOException {
         Key start = new Key("20200314_0", "tf", "datatype\0uid3");
         Key end = new Key("20200314_0", "tf", "datatype\0uid3\uffff");
         Range r = new Range(start, false, end, true);
@@ -89,7 +89,7 @@ public void testDocRange_singleKey_parent_multiField() throws IOException {
     }
 
     @Test
-    public void testDocRange_singleKey_child() throws IOException {
+    void testDocRange_singleKey_child() throws IOException {
         Key start = new Key("20200314_0", "tf", "datatype\0uid3");
         Key end = new Key("20200314_0", "tf", "datatype\0uid3\uffff");
         Range r = new Range(start, false, end, true);
@@ -113,7 +113,7 @@ public void testDocRange_singleKey_child() throws IOException {
     }
 
     @Test
-    public void testDocRange_singleKey_child_multiField() throws IOException {
+    void testDocRange_singleKey_child_multiField() throws IOException {
         Key start = new Key("20200314_0", "tf", "datatype\0uid3");
         Key end = new Key("20200314_0", "tf", "datatype\0uid3\uffff");
         Range r = new Range(start, false, end, true);
@@ -140,7 +140,7 @@ public void testDocRange_singleKey_child_multiField() throws IOException {
     }
 
     @Test
-    public void testDocRange_minMaxKeys() throws IOException {
+    void testDocRange_minMaxKeys() throws IOException {
         Key start = new Key("20200314_0", "tf", "datatype\0uid0");
         Key end = new Key("20200314_0", "tf", "datatype\0uid0.1\uffff");
         Range r = new Range(start, false, end, true);
@@ -165,7 +165,7 @@ public void testDocRange_minMaxKeys() throws IOException {
     }
 
     @Test
-    public void testDocRange_rotatingChildKeys() throws IOException {
+    void testDocRange_rotatingChildKeys() throws IOException {
         Key start = new Key("20200314_0", "tf", "datatype\0uid0");
         Key end = new Key("20200314_0", "tf", "datatype\0uid0\uffff");
         Range r = new Range(start, false, end, true);
@@ -198,7 +198,7 @@ public void testDocRange_rotatingChildKeys() throws IOException {
     }
 
     @Test
-    public void testParentFirstChildAggregation() throws IOException {
+    void testParentFirstChildAggregation() throws IOException {
         Key start = new Key("20200314_0", "tf", "datatype\0uid0");
         Key end = new Key("20200314_0", "tf", "datatype\0uid0.1\uffff");
         Range r = new Range(start, false, end, true);
@@ -223,7 +223,7 @@ public void testParentFirstChildAggregation() throws IOException {
     }
 
     @Test
-    public void testFullScanRange_singleKey_first() throws IOException {
+    void testFullScanRange_singleKey_first() throws IOException {
         Key start = new Key("20200314_0", "tf", "datatype\0uid0");
         Key end = new Key("20200314_0", "tf", "datatype\0uid9.9\uffff");
         Range r = new Range(start, false, end, true);
@@ -247,7 +247,7 @@ public void testFullScanRange_singleKey_first() throws IOException {
     }
 
     @Test
-    public void testFullScanRange_singleKey_middle() throws IOException {
+    void testFullScanRange_singleKey_middle() throws IOException {
         Key start = new Key("20200314_0", "tf", "datatype\0uid0");
         Key end = new Key("20200314_0", "tf", "datatype\0uid9.9\uffff");
         Range r = new Range(start, false, end, true);
@@ -271,7 +271,7 @@ public void testFullScanRange_singleKey_middle() throws IOException {
     }
 
     @Test
-    public void testFullScanRange_minMaxKeys() throws IOException {
+    void testFullScanRange_minMaxKeys() throws IOException {
         Key start = new Key("20200314_0", "tf", "datatype\0uid0");
         Key end = new Key("20200314_0", "tf", "datatype\0uid9.9\uffff");
         Range r = new Range(start, false, end, true);
@@ -296,7 +296,7 @@ public void testFullScanRange_minMaxKeys() throws IOException {
     }
 
     @Test
-    public void testFullScanRange_rotatingSingleKeyPerParent() throws IOException {
+    void testFullScanRange_rotatingSingleKeyPerParent() throws IOException {
         Key start = new Key("20200314_0", "tf", "datatype\0uid0");
         Key end = new Key("20200314_0", "tf", "datatype\0uid9.9\uffff");
         Range r = new Range(start, false, end, true);
@@ -330,7 +330,7 @@ public void testFullScanRange_rotatingSingleKeyPerParent() throws IOException {
 
     // Roll through every field in the search space and assert proper hit count
     @Test
-    public void testSearchEveryField() throws IOException {
+    void testSearchEveryField() throws IOException {
         // Full scan range
         Key start = new Key("20200314_0", "tf", "datatype\0uid0");
         Key end = new Key("20200314_0", "tf", "datatype\0uid9.9\uffff");
@@ -365,12 +365,12 @@ public void testSearchEveryField() throws IOException {
                 hits.add(tfIter.getTopKey());
                 tfIter.next();
             }
-            assertEquals("Expected to get 20 hits for field " + field + "but was " + hits.size(), 20, hits.size());
+            assertEquals(20, hits.size(), "Expected to get 20 hits for field " + field + "but was " + hits.size());
         }
     }
 
     @Test
-    public void testGetNextSeekRange() throws IOException {
+    void testGetNextSeekRange() throws IOException {
         SortedKeyValueIterator tfIter = buildIterAcrossValuesWithNulls();
         Multimap fieldValues = buildFieldValues("FIELD_A", "value_c", "value_d");
 
@@ -399,7 +399,7 @@ public void testGetNextSeekRange() throws IOException {
     }
 
     @Test
-    public void testTreeSetHigher() {
+    void testTreeSetHigher() {
         TreeSet values = Sets.newTreeSet(Lists.newArrayList("value_a", "value_b", "value_c"));
         //  @formatter:off
         assertAll(
@@ -413,7 +413,7 @@ public void testTreeSetHigher() {
     }
 
     @Test
-    public void testGetDistance() {
+    void testGetDistance() {
         TermFrequencyIterator iter = new TermFrequencyIterator();
         //  @formatter:off
         assertAll(
@@ -501,7 +501,7 @@ public Multimap buildFieldValues(String field, String... values)
     }
 
     @Test
-    public void testGetValueFromParts() {
+    void testGetValueFromParts() {
         // CQ = datatype\0uid\0fieldValue\0fieldName
         TermFrequencyIterator iter = new TermFrequencyIterator();
 
diff --git a/warehouse/query-core/src/test/java/datawave/core/iterators/key/util/FiKeyUtilTest.java b/warehouse/query-core/src/test/java/datawave/core/iterators/key/util/FiKeyUtilTest.java
index f5ed018d020..d03ca6bd15e 100644
--- a/warehouse/query-core/src/test/java/datawave/core/iterators/key/util/FiKeyUtilTest.java
+++ b/warehouse/query-core/src/test/java/datawave/core/iterators/key/util/FiKeyUtilTest.java
@@ -1,21 +1,21 @@
 package datawave.core.iterators.key.util;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThrows;
-import static org.junit.Assert.assertTrue;
 import static org.junit.jupiter.api.Assertions.assertAll;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import org.apache.accumulo.core.data.Key;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 
-public class FiKeyUtilTest {
+class FiKeyUtilTest {
 
     private final Key fikey = new Key("row", "fi\0FIELD", "value\0datatype\0uid");
     private final Key fikeyWithNulls = new Key("row", "fi\0FIELD", "v\0a\0l\0u\0e\0datatype\0uid");
     private final Key fikeyWithChildUid = new Key("row", "fi\0FIELD", "value\0datatype\0uid.12.37");
 
     @Test
-    public void testSimpleParse() {
+    void testSimpleParse() {
         assertTrue(FiKeyUtil.instanceOf(fikey));
         //  @formatter:off
         assertAll(
@@ -28,7 +28,7 @@ public void testSimpleParse() {
     }
 
     @Test
-    public void testParsingValueWithNulls() {
+    void testParsingValueWithNulls() {
         assertTrue(FiKeyUtil.instanceOf(fikeyWithNulls));
         //  @formatter:off
         assertAll(
@@ -41,7 +41,7 @@ public void testParsingValueWithNulls() {
     }
 
     @Test
-    public void testParseChildUid() {
+    void testParseChildUid() {
         assertTrue(FiKeyUtil.instanceOf(fikeyWithChildUid));
         //  @formatter:off
         assertAll(
@@ -54,7 +54,7 @@ public void testParseChildUid() {
     }
 
     @Test
-    public void testParseNoValue() {
+    void testParseNoValue() {
         Key key = new Key("row", "fi\0FIELD", "datatype\0uid");
         assertThrows(IllegalArgumentException.class, () -> {
             assertEquals(".getValueString() should have thrown an IllegalArgumentException error: ", "", FiKeyUtil.getValueString(key));
@@ -62,7 +62,7 @@ public void testParseNoValue() {
     }
 
     @Test
-    public void testParseNoDatatype() {
+    void testParseNoDatatype() {
         Key key = new Key("row", "fi\0FIELD", "value\0uid");
         assertThrows(IllegalArgumentException.class, () -> {
             assertEquals(".getDatatypeString() should have thrown an IllegalArgumentException error: ", "", FiKeyUtil.getDatatypeString(key));
@@ -72,7 +72,7 @@ public void testParseNoDatatype() {
     // getUidString simply returns everything after the last null byte. If the column qualifier is wrong
     // then the method will return the wrong value
     @Test
-    public void testParseNoUid() {
+    void testParseNoUid() {
         Key key = new Key("row", "fi\0FIELD", "value\0datatype");
         assertEquals("datatype", FiKeyUtil.getUidString(key));
     }
diff --git a/warehouse/query-core/src/test/java/datawave/core/iterators/key/util/TFKeyUtilTest.java b/warehouse/query-core/src/test/java/datawave/core/iterators/key/util/TFKeyUtilTest.java
index dcf2aa95f67..b112bd6a170 100644
--- a/warehouse/query-core/src/test/java/datawave/core/iterators/key/util/TFKeyUtilTest.java
+++ b/warehouse/query-core/src/test/java/datawave/core/iterators/key/util/TFKeyUtilTest.java
@@ -1,21 +1,21 @@
 package datawave.core.iterators.key.util;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThrows;
-import static org.junit.Assert.assertTrue;
 import static org.junit.jupiter.api.Assertions.assertAll;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import org.apache.accumulo.core.data.Key;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 
-public class TFKeyUtilTest {
+class TFKeyUtilTest {
 
     private final Key tfKey = new Key("row", "tf", "datatype\0uid\0value\0FIELD");
     private final Key tfKeyWithNulls = new Key("row", "tf", "datatype\0uid\0v\0a\0l\0u\0e\0FIELD");
     private final Key tfKeyWithChildUid = new Key("row", "tf", "datatype\0uid.11.22\0value\0FIELD");
 
     @Test
-    public void testSimpleParse() {
+    void testSimpleParse() {
         assertTrue(TFKeyUtil.instanceOf(tfKey));
         //  @formatter:off
         assertAll(
@@ -28,7 +28,7 @@ public void testSimpleParse() {
     }
 
     @Test
-    public void testParseValueWithNulls() {
+    void testParseValueWithNulls() {
         assertTrue(TFKeyUtil.instanceOf(tfKeyWithNulls));
         //  @formatter:off
         assertAll(
@@ -41,7 +41,7 @@ public void testParseValueWithNulls() {
     }
 
     @Test
-    public void testParseChildUid() {
+    void testParseChildUid() {
         assertTrue(TFKeyUtil.instanceOf(tfKeyWithChildUid));
         //  @formatter:off
         assertAll(
@@ -56,13 +56,13 @@ public void testParseChildUid() {
     // malformed keys will still parse
 
     @Test
-    public void testParseNoField() {
+    void testParseNoField() {
         Key k = new Key("row", "tf", "datatype\0uid\0value");
         assertEquals("value", TFKeyUtil.getFieldString(k));
     }
 
     @Test
-    public void testParseNoValue() {
+    void testParseNoValue() {
         assertThrows(IllegalArgumentException.class, () -> {
             Key k = new Key("row", "tf", "datatype\0uid\0FIELD");
             assertEquals(".getValueString() should have thrown an IllegalArgumentException error: ", "", TFKeyUtil.getValueString(k));
@@ -70,13 +70,13 @@ public void testParseNoValue() {
     }
 
     @Test
-    public void testParseNoDatatype() {
+    void testParseNoDatatype() {
         Key k = new Key("row", "tf", "uid\0value\0FIELD");
         assertEquals("uid", TFKeyUtil.getDatatypeString(k));
     }
 
     @Test
-    public void testParseNoUid() {
+    void testParseNoUid() {
         Key k = new Key("row", "tf", "datatype\0value\0FIELD");
         assertEquals("value", TFKeyUtil.getUidString(k));
     }
diff --git a/warehouse/query-core/src/test/java/datawave/query/ancestor/AncestorChildExpansionIteratorTest.java b/warehouse/query-core/src/test/java/datawave/query/ancestor/AncestorChildExpansionIteratorTest.java
index f4bf043f626..0b94c379e32 100644
--- a/warehouse/query-core/src/test/java/datawave/query/ancestor/AncestorChildExpansionIteratorTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/ancestor/AncestorChildExpansionIteratorTest.java
@@ -1,11 +1,11 @@
 package datawave.query.ancestor;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertThrows;
-import static org.junit.Assert.assertTrue;
 import static org.junit.jupiter.api.Assertions.assertAll;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.IOException;
 import java.util.AbstractMap;
@@ -19,15 +19,15 @@
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
-import org.junit.Before;
-import org.junit.Test;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 
 import datawave.query.Constants;
 import datawave.query.function.AncestorEquality;
 import datawave.query.function.Equality;
 import datawave.query.util.IteratorToSortedKeyValueIterator;
 
-public class AncestorChildExpansionIteratorTest {
+class AncestorChildExpansionIteratorTest {
     private final List children = Arrays.asList("a", "a.1", "a.1.1", "a.1.2", "a.1.2.1", "a.10", "a.2", "a.3", "a.4", "a.4.1", "a.4.1.1", "a.4.1.2",
                     "a.4.2", "a.5", "a.6", "a.7", "a.8", "a.9");
 
@@ -36,7 +36,7 @@ public class AncestorChildExpansionIteratorTest {
     private IteratorToSortedKeyValueIterator baseIterator;
     private Equality equality;
 
-    @Before
+    @BeforeEach
     public void setup() {
         baseValues = new ArrayList<>();
         equality = new AncestorEquality();
@@ -47,62 +47,62 @@ public void setup() {
     // basic iterator contract verification
 
     @Test
-    public void testUninitializedHasTop() {
+    void testUninitializedHasTop() {
         assertThrows(IllegalStateException.class, () -> {
             iterator.hasTop();
         });
     }
 
     @Test
-    public void testUninitializedgetTopKey() {
+    void testUninitializedgetTopKey() {
         assertThrows(IllegalStateException.class, () -> {
             iterator.getTopKey();
         });
     }
 
     @Test
-    public void testUninitializedgetTopValue() {
+    void testUninitializedgetTopValue() {
         assertThrows(IllegalStateException.class, () -> {
             iterator.getTopValue();
         });
     }
 
     @Test
-    public void testUninitializedNext() {
+    void testUninitializedNext() {
         assertThrows(IllegalStateException.class, () -> {
             iterator.next();
         });
     }
 
     @Test
-    public void testSeekEnablesHasTop() throws IOException {
-        iterator.seek(new Range(), Collections.EMPTY_LIST, false);
+    void testSeekEnablesHasTop() throws IOException {
+        iterator.seek(new Range(), Collections.emptySet(), false);
         assertFalse(iterator.hasTop());
     }
 
     @Test
-    public void testNoTopGetTopKeyError() {
+    void testNoTopGetTopKeyError() {
         assertThrows(NoSuchElementException.class, () -> {
-            iterator.seek(new Range(), Collections.EMPTY_LIST, false);
-            assertFalse(".hasTop() returned true on EMPTY_LIST", iterator.hasTop());
+            iterator.seek(new Range(), Collections.emptySet(), false);
+            assertFalse(iterator.hasTop(), ".hasTop() returned true on EMPTY_LIST");
             iterator.getTopKey();
         });
     }
 
     @Test
-    public void testNoTopGetTopValueError() {
+    void testNoTopGetTopValueError() {
         assertThrows(NoSuchElementException.class, () -> {
-            iterator.seek(new Range(), Collections.EMPTY_LIST, false);
-            assertFalse(".hasTop() returned true on EMPTY_LIST", iterator.hasTop());
+            iterator.seek(new Range(), Collections.emptySet(), false);
+            assertFalse(iterator.hasTop(), ".hasTop() returned true on EMPTY_LIST");
             iterator.getTopValue();
         });
     }
 
     @Test
-    public void testNoTopNextError() {
+    void testNoTopNextError() {
         assertThrows(NoSuchElementException.class, () -> {
-            iterator.seek(new Range(), Collections.EMPTY_LIST, false);
-            assertFalse(".hasTop() returned true on EMPTY_LIST", iterator.hasTop());
+            iterator.seek(new Range(), Collections.emptySet(), false);
+            assertFalse(iterator.hasTop(), ".hasTop() returned true on EMPTY_LIST");
             iterator.next();
         });
     }
@@ -133,12 +133,12 @@ private void assertKey(Key key, String uid) {
     }
 
     @Test
-    public void testSingleChildMatch() throws IOException {
+    void testSingleChildMatch() throws IOException {
         baseValues.add(new AbstractMap.SimpleImmutableEntry<>(generateFiKey("a.3"), new Value()));
         baseIterator = new IteratorToSortedKeyValueIterator(baseValues.iterator());
         iterator = new AncestorChildExpansionIterator(baseIterator, children, equality);
 
-        iterator.seek(new Range(), Collections.EMPTY_LIST, false);
+        iterator.seek(new Range(), Collections.emptySet(), false);
 
         assertTrue(iterator.hasTop());
         Key topKey = iterator.getTopKey();
@@ -151,12 +151,12 @@ public void testSingleChildMatch() throws IOException {
     }
 
     @Test
-    public void testFamilyMatch() throws IOException {
+    void testFamilyMatch() throws IOException {
         baseValues.add(new AbstractMap.SimpleImmutableEntry<>(generateFiKey("a.1"), new Value()));
         baseIterator = new IteratorToSortedKeyValueIterator(baseValues.iterator());
         iterator = new AncestorChildExpansionIterator(baseIterator, children, equality);
 
-        iterator.seek(new Range(), Collections.EMPTY_LIST, false);
+        iterator.seek(new Range(), Collections.emptySet(), false);
 
         assertTrue(iterator.hasTop());
         Key topKey = iterator.getTopKey();
@@ -190,14 +190,14 @@ public void testFamilyMatch() throws IOException {
     }
 
     @Test
-    public void testFamilyMatchWithOverlaps() throws IOException {
+    void testFamilyMatchWithOverlaps() throws IOException {
         baseValues.add(new AbstractMap.SimpleImmutableEntry<>(generateFiKey("a.1"), new Value()));
         baseValues.add(new AbstractMap.SimpleImmutableEntry<>(generateFiKey("a.1.1"), new Value()));
         baseValues.add(new AbstractMap.SimpleImmutableEntry<>(generateFiKey("a.1.2.1"), new Value()));
         baseIterator = new IteratorToSortedKeyValueIterator(baseValues.iterator());
         iterator = new AncestorChildExpansionIterator(baseIterator, children, equality);
 
-        iterator.seek(new Range(), Collections.EMPTY_LIST, false);
+        iterator.seek(new Range(), Collections.emptySet(), false);
 
         assertTrue(iterator.hasTop());
         Key topKey = iterator.getTopKey();
@@ -231,13 +231,13 @@ public void testFamilyMatchWithOverlaps() throws IOException {
     }
 
     @Test
-    public void testChildIndexAdvanceOnIteratorNext() throws IOException {
+    void testChildIndexAdvanceOnIteratorNext() throws IOException {
         baseValues.add(new AbstractMap.SimpleImmutableEntry<>(generateFiKey("a.3"), new Value()));
         baseValues.add(new AbstractMap.SimpleImmutableEntry<>(generateFiKey("a.4.1"), new Value()));
         baseIterator = new IteratorToSortedKeyValueIterator(baseValues.iterator());
         iterator = new AncestorChildExpansionIterator(baseIterator, children, equality);
 
-        iterator.seek(new Range(), Collections.EMPTY_LIST, false);
+        iterator.seek(new Range(), Collections.emptySet(), false);
 
         assertTrue(iterator.hasTop());
         Key topKey = iterator.getTopKey();
@@ -271,7 +271,7 @@ public void testChildIndexAdvanceOnIteratorNext() throws IOException {
     }
 
     @Test
-    public void testMultipleGappedRanges() throws IOException {
+    void testMultipleGappedRanges() throws IOException {
         baseValues.add(new AbstractMap.SimpleImmutableEntry<>(generateFiKey("a.1"), new Value()));
         baseValues.add(new AbstractMap.SimpleImmutableEntry<>(generateFiKey("a.3"), new Value()));
         baseValues.add(new AbstractMap.SimpleImmutableEntry<>(generateFiKey("a.4.1.1"), new Value()));
@@ -279,7 +279,7 @@ public void testMultipleGappedRanges() throws IOException {
         baseIterator = new IteratorToSortedKeyValueIterator(baseValues.iterator());
         iterator = new AncestorChildExpansionIterator(baseIterator, children, equality);
 
-        iterator.seek(new Range(), Collections.EMPTY_LIST, false);
+        iterator.seek(new Range(), Collections.emptySet(), false);
 
         assertTrue(iterator.hasTop());
         Key topKey = iterator.getTopKey();
diff --git a/warehouse/query-core/src/test/java/datawave/query/jexl/DatawaveJexlContextTest.java b/warehouse/query-core/src/test/java/datawave/query/jexl/DatawaveJexlContextTest.java
index 1f2d3730f00..355b4656ce8 100644
--- a/warehouse/query-core/src/test/java/datawave/query/jexl/DatawaveJexlContextTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/jexl/DatawaveJexlContextTest.java
@@ -1,20 +1,20 @@
 package datawave.query.jexl;
 
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
 import static org.junit.jupiter.api.Assertions.assertAll;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import org.apache.commons.jexl3.JexlEngine;
 import org.apache.commons.jexl3.internal.Engine;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 
-public class DatawaveJexlContextTest {
+class DatawaveJexlContextTest {
 
     private final JexlEngine engine = new Engine();
     private final DatawaveJexlContext context = new DatawaveJexlContext();
 
     @Test
-    public void testEscapingSingleQuote() {
+    void testEscapingSingleQuote() {
         // @formatter:off
         assertAll(
                         // an escaped single quote within single quotes will match a single quote in the context
@@ -61,7 +61,7 @@ public void testEscapingSingleQuote() {
     }
 
     @Test
-    public void testEscapingDoubleQuote() {
+    void testEscapingDoubleQuote() {
         // @formatter:off
         assertAll(
                         // an escaped double quote within double quotes will match a double quote in the context
@@ -109,7 +109,7 @@ public void testEscapingDoubleQuote() {
     }
 
     @Test
-    public void testEscapingUnicode() {
+    void testEscapingUnicode() {
         // @formatter:off
         assertAll(() -> {
             context.set("F1", "Str∂");

From a29c4636b868aea4b16f32425ebffc7aa6d8516a Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Mon, 12 Feb 2024 18:12:10 +0000
Subject: [PATCH 31/50] Add junit-bom depedency and use version of junit-bom
 for explicit jupiter and vintage dependencies

---
 core/pom.xml           |  8 ++++----
 pom.xml                | 23 ++++++++++++++---------
 warehouse/core/pom.xml |  4 ++--
 warehouse/pom.xml      |  4 ++--
 4 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/core/pom.xml b/core/pom.xml
index 9c40497bdbf..3e5ab2abe1c 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -57,13 +57,13 @@
         
             org.junit.jupiter
             junit-jupiter-engine
-            ${version.junit.jupiter}
+            ${version.junit.bom}
             test
         
         
             org.junit.vintage
             junit-vintage-engine
-            ${version.junit.vintage}
+            ${version.junit.bom}
             test
         
     
@@ -89,12 +89,12 @@
                         
                             org.junit.jupiter
                             junit-jupiter-engine
-                            ${version.junit.jupiter}
+                            ${version.junit.bom}
                         
                         
                             org.junit.vintage
                             junit-vintage-engine
-                            ${version.junit.vintage}
+                            ${version.junit.bom}
                         
                     
                 
diff --git a/pom.xml b/pom.xml
index a3886846c89..721f34559df 100644
--- a/pom.xml
+++ b/pom.xml
@@ -85,8 +85,6 @@
         20231013
         1.19.0
         4.13.2
-        5.10.2
-        5.10.2
         2.7.2b2
         2.20
         2.20
@@ -1059,6 +1057,13 @@
                 pom
                 import
             
+            
+                org.junit
+                junit-bom
+                ${version.junit.bom}
+                pom
+                import
+            
             
                 org.springframework
                 spring-framework-bom
@@ -1146,19 +1151,19 @@
             
                 org.junit.jupiter
                 junit-jupiter-api
-                ${version.junit.jupiter}
+                ${version.junit.bom}
                 test
             
             
                 org.junit.jupiter
                 junit-jupiter-engine
-                ${version.junit.jupiter}
+                ${version.junit.bom}
                 test
             
             
                 org.junit.vintage
                 junit-vintage-engine
-                ${version.junit.vintage}
+                ${version.junit.bom}
                 test
             
             
@@ -1485,12 +1490,12 @@
                         
                             org.junit.jupiter
                             junit-jupiter-engine
-                            ${version.junit.jupiter}
+                            ${version.junit.bom}
                         
                         
                             org.junit.vintage
                             junit-vintage-engine
-                            ${version.junit.vintage}
+                            ${version.junit.bom}
                         
                     
                     
@@ -1577,12 +1582,12 @@
                         
                             org.junit.jupiter
                             junit-jupiter-engine
-                            ${version.junit.jupiter}
+                            ${version.junit.bom}
                         
                         
                             org.junit.vintage
                             junit-vintage-engine
-                            ${version.junit.vintage}
+                            ${version.junit.bom}
                         
                     
                 
diff --git a/warehouse/core/pom.xml b/warehouse/core/pom.xml
index dde77c0a59a..b73785a289c 100644
--- a/warehouse/core/pom.xml
+++ b/warehouse/core/pom.xml
@@ -251,12 +251,12 @@
                     
                         org.junit.jupiter
                         junit-jupiter-engine
-                        ${version.junit.jupiter}
+                        ${version.junit.bom}
                     
                     
                         org.junit.vintage
                         junit-vintage-engine
-                        ${version.junit.vintage}
+                        ${version.junit.bom}
                     
                 
             
diff --git a/warehouse/pom.xml b/warehouse/pom.xml
index c9115a807b1..4d306af3d99 100644
--- a/warehouse/pom.xml
+++ b/warehouse/pom.xml
@@ -480,12 +480,12 @@
                         
                             org.junit.jupiter
                             junit-jupiter-engine
-                            ${version.junit.jupiter}
+                            ${version.junit.bom}
                         
                         
                             org.junit.vintage
                             junit-vintage-engine
-                            ${version.junit.vintage}
+                            ${version.junit.bom}
                         
                     
                     

From ffb87b314ab6fe6682428969718f906fe31f8440 Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Tue, 13 Feb 2024 14:40:49 +0000
Subject: [PATCH 32/50] Clean up extra jupiter dependency declarations

---
 core/pom.xml                          | 27 ---------------------------
 pom.xml                               | 18 ------------------
 warehouse/accumulo-extensions/pom.xml | 10 ----------
 warehouse/common/pom.xml              | 10 ----------
 warehouse/core/pom.xml                | 10 ----------
 warehouse/ingest-core/pom.xml         | 10 ----------
 warehouse/ingest-csv/pom.xml          | 10 ----------
 warehouse/ingest-wikipedia/pom.xml    | 10 ----------
 warehouse/query-core/pom.xml          | 10 ----------
 web-services/cached-results/pom.xml   | 10 ----------
 web-services/common-util/pom.xml      | 10 ----------
 web-services/common/pom.xml           | 10 ----------
 web-services/map-reduce/pom.xml       | 10 ----------
 web-services/query/pom.xml            | 10 ----------
 web-services/security/pom.xml         | 10 ----------
 15 files changed, 175 deletions(-)

diff --git a/core/pom.xml b/core/pom.xml
index 3e5ab2abe1c..ad428683a8d 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -26,21 +26,6 @@
                 ${version.junit}
                 test
             
-            
-                org.junit.jupiter
-                junit-jupiter-api
-                test
-            
-            
-                org.junit.jupiter
-                junit-jupiter-engine
-                test
-            
-            
-                org.junit.vintage
-                junit-vintage-engine
-                test
-            
         
         
     
@@ -54,18 +39,6 @@
             ${version.junit}
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            ${version.junit.bom}
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            ${version.junit.bom}
-            test
-        
     
     
         
diff --git a/pom.xml b/pom.xml
index 721f34559df..de852769ed6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1148,24 +1148,6 @@
                 ${version.weld-test}
                 test
             
-            
-                org.junit.jupiter
-                junit-jupiter-api
-                ${version.junit.bom}
-                test
-            
-            
-                org.junit.jupiter
-                junit-jupiter-engine
-                ${version.junit.bom}
-                test
-            
-            
-                org.junit.vintage
-                junit-vintage-engine
-                ${version.junit.bom}
-                test
-            
             
                 org.mockito
                 mockito-core
diff --git a/warehouse/accumulo-extensions/pom.xml b/warehouse/accumulo-extensions/pom.xml
index a69ab3678c1..14f445611d2 100644
--- a/warehouse/accumulo-extensions/pom.xml
+++ b/warehouse/accumulo-extensions/pom.xml
@@ -54,16 +54,6 @@
             jmock-junit4
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
     
     
         
diff --git a/warehouse/common/pom.xml b/warehouse/common/pom.xml
index f95de245364..8991e037ffb 100644
--- a/warehouse/common/pom.xml
+++ b/warehouse/common/pom.xml
@@ -55,15 +55,5 @@
             junit
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-api
-            test
-        
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
     
 
diff --git a/warehouse/core/pom.xml b/warehouse/core/pom.xml
index b73785a289c..7a0084b2a9b 100644
--- a/warehouse/core/pom.xml
+++ b/warehouse/core/pom.xml
@@ -198,16 +198,6 @@
             easymock
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-api
-            test
-        
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
     
     
         
diff --git a/warehouse/ingest-core/pom.xml b/warehouse/ingest-core/pom.xml
index 3ac8433edc4..ce13224d0e0 100644
--- a/warehouse/ingest-core/pom.xml
+++ b/warehouse/ingest-core/pom.xml
@@ -202,16 +202,6 @@
             javassist
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
     
     
         
diff --git a/warehouse/ingest-csv/pom.xml b/warehouse/ingest-csv/pom.xml
index 16e12e17776..35b1c2df03c 100644
--- a/warehouse/ingest-csv/pom.xml
+++ b/warehouse/ingest-csv/pom.xml
@@ -120,16 +120,6 @@
             javassist
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
     
     
         
diff --git a/warehouse/ingest-wikipedia/pom.xml b/warehouse/ingest-wikipedia/pom.xml
index 6a7b5db56fa..fea39cd417e 100644
--- a/warehouse/ingest-wikipedia/pom.xml
+++ b/warehouse/ingest-wikipedia/pom.xml
@@ -110,16 +110,6 @@
             javassist
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
     
     
         
diff --git a/warehouse/query-core/pom.xml b/warehouse/query-core/pom.xml
index 20ad75eb873..42a8e1d2c99 100644
--- a/warehouse/query-core/pom.xml
+++ b/warehouse/query-core/pom.xml
@@ -350,16 +350,6 @@
             junit-jupiter-api
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
         
             org.mockito
             mockito-core
diff --git a/web-services/cached-results/pom.xml b/web-services/cached-results/pom.xml
index bf1bc1691e4..de70d9d80cb 100644
--- a/web-services/cached-results/pom.xml
+++ b/web-services/cached-results/pom.xml
@@ -124,16 +124,6 @@
             javassist
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
     
     
         ${project.artifactId}
diff --git a/web-services/common-util/pom.xml b/web-services/common-util/pom.xml
index c23e05e869c..c75e95b7f1d 100644
--- a/web-services/common-util/pom.xml
+++ b/web-services/common-util/pom.xml
@@ -187,16 +187,6 @@
             javassist
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
     
     
         ${project.artifactId}
diff --git a/web-services/common/pom.xml b/web-services/common/pom.xml
index 22ead5adb64..5df8eaa097d 100644
--- a/web-services/common/pom.xml
+++ b/web-services/common/pom.xml
@@ -255,16 +255,6 @@
             javassist
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
         
             org.springframework
             spring-test
diff --git a/web-services/map-reduce/pom.xml b/web-services/map-reduce/pom.xml
index 26e0eddf68f..e61c1bd1304 100644
--- a/web-services/map-reduce/pom.xml
+++ b/web-services/map-reduce/pom.xml
@@ -174,16 +174,6 @@
             junit
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
         
             org.springframework
             spring-expression
diff --git a/web-services/query/pom.xml b/web-services/query/pom.xml
index ba27a7067c4..b91638faf34 100644
--- a/web-services/query/pom.xml
+++ b/web-services/query/pom.xml
@@ -230,16 +230,6 @@
             javassist
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
     
     
         ${project.artifactId}
diff --git a/web-services/security/pom.xml b/web-services/security/pom.xml
index 313d0f389fb..b739d4aa839 100644
--- a/web-services/security/pom.xml
+++ b/web-services/security/pom.xml
@@ -249,16 +249,6 @@
             weld-core-impl
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
         
             org.springframework
             spring-expression

From a8018765f92e097afc3d710ee28f0a552cc8f8ed Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Tue, 13 Feb 2024 14:49:26 +0000
Subject: [PATCH 33/50] Additional module cleanup

---
 warehouse/index-stats/pom.xml                     | 10 ----------
 warehouse/ingest-json/pom.xml                     | 10 ----------
 warehouse/ingest-nyctlc/pom.xml                   | 11 -----------
 warehouse/ops-tools/config-compare/pom.xml        | 10 ----------
 warehouse/ops-tools/index-validation/pom.xml      | 10 ----------
 warehouse/ssdeep-common/pom.xml                   | 10 ----------
 web-services/atom/pom.xml                         | 10 ----------
 web-services/client/pom.xml                       | 15 ---------------
 .../deploy/spring-framework-integration/pom.xml   | 10 ----------
 web-services/model/pom.xml                        | 10 ----------
 web-services/modification/pom.xml                 | 10 ----------
 11 files changed, 116 deletions(-)

diff --git a/warehouse/index-stats/pom.xml b/warehouse/index-stats/pom.xml
index fd123f15304..9d1af44b5fc 100644
--- a/warehouse/index-stats/pom.xml
+++ b/warehouse/index-stats/pom.xml
@@ -59,16 +59,6 @@
             javassist
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
     
     
         
diff --git a/warehouse/ingest-json/pom.xml b/warehouse/ingest-json/pom.xml
index 3cda7ebbe5f..5d7c4f1a8a7 100644
--- a/warehouse/ingest-json/pom.xml
+++ b/warehouse/ingest-json/pom.xml
@@ -125,16 +125,6 @@
             javassist
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
     
     
         
diff --git a/warehouse/ingest-nyctlc/pom.xml b/warehouse/ingest-nyctlc/pom.xml
index ae17424c068..fa661596a31 100644
--- a/warehouse/ingest-nyctlc/pom.xml
+++ b/warehouse/ingest-nyctlc/pom.xml
@@ -36,17 +36,6 @@
             datawave-query-core
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
-
     
     
         
diff --git a/warehouse/ops-tools/config-compare/pom.xml b/warehouse/ops-tools/config-compare/pom.xml
index c49e63faff1..dde7abee524 100644
--- a/warehouse/ops-tools/config-compare/pom.xml
+++ b/warehouse/ops-tools/config-compare/pom.xml
@@ -58,15 +58,5 @@
             1.3
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
     
 
diff --git a/warehouse/ops-tools/index-validation/pom.xml b/warehouse/ops-tools/index-validation/pom.xml
index 65fe126998c..ff10e398236 100644
--- a/warehouse/ops-tools/index-validation/pom.xml
+++ b/warehouse/ops-tools/index-validation/pom.xml
@@ -79,16 +79,6 @@
             1.3
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
     
     
         
diff --git a/warehouse/ssdeep-common/pom.xml b/warehouse/ssdeep-common/pom.xml
index 34299e08780..ee7fd9ef88d 100644
--- a/warehouse/ssdeep-common/pom.xml
+++ b/warehouse/ssdeep-common/pom.xml
@@ -23,15 +23,5 @@
             junit
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
     
 
diff --git a/web-services/atom/pom.xml b/web-services/atom/pom.xml
index d45ec3571ca..4a925cf581b 100644
--- a/web-services/atom/pom.xml
+++ b/web-services/atom/pom.xml
@@ -97,16 +97,6 @@
             jboss-transaction-api_1.2_spec
             provided
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
     
     
         ${project.artifactId}
diff --git a/web-services/client/pom.xml b/web-services/client/pom.xml
index 29cdd50c2a9..49f8ee5ba5a 100644
--- a/web-services/client/pom.xml
+++ b/web-services/client/pom.xml
@@ -171,21 +171,6 @@
             junit
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-api
-            test
-        
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
     
     
         ${project.artifactId}
diff --git a/web-services/deploy/spring-framework-integration/pom.xml b/web-services/deploy/spring-framework-integration/pom.xml
index 6f724e3b4a0..52459f2a6e4 100644
--- a/web-services/deploy/spring-framework-integration/pom.xml
+++ b/web-services/deploy/spring-framework-integration/pom.xml
@@ -100,16 +100,6 @@
             junit
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
         
             org.jboss.arquillian.container
             arquillian-weld-ee-embedded-1.1
diff --git a/web-services/model/pom.xml b/web-services/model/pom.xml
index f9e58f0acb1..d136098c61c 100644
--- a/web-services/model/pom.xml
+++ b/web-services/model/pom.xml
@@ -133,16 +133,6 @@
             javassist
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
     
     
         ${project.artifactId}
diff --git a/web-services/modification/pom.xml b/web-services/modification/pom.xml
index 15bfcd97ea8..138d81fe4ae 100644
--- a/web-services/modification/pom.xml
+++ b/web-services/modification/pom.xml
@@ -82,16 +82,6 @@
             junit
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-            test
-        
     
     
         ${project.artifactId}

From ea86d853fe69831840677646e756b8c71c0c5a4d Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Tue, 13 Feb 2024 15:23:09 +0000
Subject: [PATCH 34/50] Add back some dependencies that were removed..

---
 common-test/pom.xml      |  5 +++++
 warehouse/common/pom.xml |  5 +++++
 warehouse/pom.xml        | 13 +++++++++++++
 3 files changed, 23 insertions(+)

diff --git a/common-test/pom.xml b/common-test/pom.xml
index 7d0c176560a..4d3e3f71919 100644
--- a/common-test/pom.xml
+++ b/common-test/pom.xml
@@ -63,5 +63,10 @@
             provided
             true
         
+        
+            org.junit.jupiter
+            junit-jupiter-api
+            test
+        
     
 
diff --git a/warehouse/common/pom.xml b/warehouse/common/pom.xml
index 8991e037ffb..484fdfa55ec 100644
--- a/warehouse/common/pom.xml
+++ b/warehouse/common/pom.xml
@@ -55,5 +55,10 @@
             junit
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-api
+            test
+        
     
 
diff --git a/warehouse/pom.xml b/warehouse/pom.xml
index 4d306af3d99..71878da7d5c 100644
--- a/warehouse/pom.xml
+++ b/warehouse/pom.xml
@@ -445,6 +445,19 @@
             gov.nsa.datawave
             datawave-common-test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+        
+        
+            org.junit.vintage
+            junit-vintage-engine
+        
+        
+            org.junit.jupiter
+            junit-jupiter-api
+            test
+        
     
     
         

From 23604066e5c00d42732fca7df843099f120a7358 Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Tue, 13 Feb 2024 16:13:16 +0000
Subject: [PATCH 35/50] remove junit version from core pom.xml

---
 core/pom.xml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/core/pom.xml b/core/pom.xml
index ad428683a8d..0617eeaf717 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -36,7 +36,6 @@
         
             junit
             junit
-            ${version.junit}
             test
         
     

From 7d497f1f562d3575cf62b1c430c48eb55a743c6c Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Tue, 13 Feb 2024 16:13:16 +0000
Subject: [PATCH 36/50] add junit-jupiter-engine to common-test

add junit-jupiter-engine to warehouse-common

remove jupiter-api from warehouse pom

restore jupitier-api to ws-client pom
---
 common-test/pom.xml         | 2 +-
 warehouse/common/pom.xml    | 2 +-
 warehouse/pom.xml           | 5 -----
 web-services/client/pom.xml | 5 +++++
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/common-test/pom.xml b/common-test/pom.xml
index 4d3e3f71919..381b91105e1 100644
--- a/common-test/pom.xml
+++ b/common-test/pom.xml
@@ -65,7 +65,7 @@
         
         
             org.junit.jupiter
-            junit-jupiter-api
+            junit-jupiter-engine
             test
         
     
diff --git a/warehouse/common/pom.xml b/warehouse/common/pom.xml
index 484fdfa55ec..ca14424020d 100644
--- a/warehouse/common/pom.xml
+++ b/warehouse/common/pom.xml
@@ -57,7 +57,7 @@
         
         
             org.junit.jupiter
-            junit-jupiter-api
+            junit-jupiter-engine
             test
         
     
diff --git a/warehouse/pom.xml b/warehouse/pom.xml
index 71878da7d5c..dfea0e7c452 100644
--- a/warehouse/pom.xml
+++ b/warehouse/pom.xml
@@ -453,11 +453,6 @@
             org.junit.vintage
             junit-vintage-engine
         
-        
-            org.junit.jupiter
-            junit-jupiter-api
-            test
-        
     
     
         
diff --git a/web-services/client/pom.xml b/web-services/client/pom.xml
index 49f8ee5ba5a..587d5f818f7 100644
--- a/web-services/client/pom.xml
+++ b/web-services/client/pom.xml
@@ -171,6 +171,11 @@
             junit
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-api
+            test
+        
     
     
         ${project.artifactId}

From 684cab92b814a03466a06f5835d42b05e8501340 Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Thu, 15 Feb 2024 13:57:03 +0000
Subject: [PATCH 37/50] fix bad rebase

---
 pom.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pom.xml b/pom.xml
index de852769ed6..f76a4534115 100644
--- a/pom.xml
+++ b/pom.xml
@@ -85,6 +85,7 @@
         20231013
         1.19.0
         4.13.2
+        5.10.2
         2.7.2b2
         2.20
         2.20

From 2c62646a4596b9a18a07dbdef30a8eaee78fc8d0 Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Fri, 16 Feb 2024 11:51:58 +0000
Subject: [PATCH 38/50] remove dependencies from warehouse pom

---
 warehouse/pom.xml | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/warehouse/pom.xml b/warehouse/pom.xml
index dfea0e7c452..e3c96be4bb5 100644
--- a/warehouse/pom.xml
+++ b/warehouse/pom.xml
@@ -440,20 +440,6 @@
             
         
     
-    
-        
-            gov.nsa.datawave
-            datawave-common-test
-        
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-        
-        
-            org.junit.vintage
-            junit-vintage-engine
-        
-    
     
         
             

From 5b9c1bf5f4fdc4e5728eed1e733c58ce61fdde8d Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Fri, 16 Feb 2024 13:54:58 +0000
Subject: [PATCH 39/50] add back dependency on datawave-common-test

---
 warehouse/pom.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/warehouse/pom.xml b/warehouse/pom.xml
index e3c96be4bb5..4d306af3d99 100644
--- a/warehouse/pom.xml
+++ b/warehouse/pom.xml
@@ -440,6 +440,12 @@
             
         
     
+    
+        
+            gov.nsa.datawave
+            datawave-common-test
+        
+    
     
         
             

From e77f54c820e134ff96302c447d3ea7652e005faa Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Fri, 16 Feb 2024 15:23:07 +0000
Subject: [PATCH 40/50] add junit-jupiter-engine to warehouse pom with test
 scope

---
 warehouse/pom.xml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/warehouse/pom.xml b/warehouse/pom.xml
index 4d306af3d99..b04ded1e3ce 100644
--- a/warehouse/pom.xml
+++ b/warehouse/pom.xml
@@ -445,6 +445,11 @@
             gov.nsa.datawave
             datawave-common-test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
     
     
         

From 36956c6a9655ccbe67a7c74b5a02ee0cbb5e6c91 Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Fri, 16 Feb 2024 15:48:25 +0000
Subject: [PATCH 41/50] Add jupiter-engine dependency to webservices pom and
 ssdeep-common pom

---
 warehouse/ssdeep-common/pom.xml | 5 +++++
 web-services/pom.xml            | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/warehouse/ssdeep-common/pom.xml b/warehouse/ssdeep-common/pom.xml
index ee7fd9ef88d..1d355b64dec 100644
--- a/warehouse/ssdeep-common/pom.xml
+++ b/warehouse/ssdeep-common/pom.xml
@@ -23,5 +23,10 @@
             junit
             test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
     
 
diff --git a/web-services/pom.xml b/web-services/pom.xml
index e1b5614829d..767cbdf18d4 100644
--- a/web-services/pom.xml
+++ b/web-services/pom.xml
@@ -506,6 +506,11 @@
             gov.nsa.datawave
             datawave-common-test
         
+        
+            org.junit.jupiter
+            junit-jupiter-engine
+            test
+        
     
     
         

From 7e356d1a1f2df1c9f466edc65adca021b97f8c9b Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Fri, 16 Feb 2024 17:20:12 +0000
Subject: [PATCH 42/50] remove duplicate junit-jupiter-engine dependency from
 warehouse-common pom, add jupiter-engine and vintage-engine to failsafe
 plugins in webservices and warehouse pom

---
 warehouse/common/pom.xml |  5 -----
 warehouse/pom.xml        | 12 ++++++++++++
 web-services/pom.xml     | 12 ++++++++++++
 3 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/warehouse/common/pom.xml b/warehouse/common/pom.xml
index ca14424020d..8991e037ffb 100644
--- a/warehouse/common/pom.xml
+++ b/warehouse/common/pom.xml
@@ -55,10 +55,5 @@
             junit
             test
         
-        
-            org.junit.jupiter
-            junit-jupiter-engine
-            test
-        
     
 
diff --git a/warehouse/pom.xml b/warehouse/pom.xml
index b04ded1e3ce..f222279fef3 100644
--- a/warehouse/pom.xml
+++ b/warehouse/pom.xml
@@ -556,6 +556,18 @@
                         /dev/null
                     
                 
+                
+                    
+                        org.junit.jupiter
+                        junit-jupiter-engine
+                        ${version.junit.bom}
+                    
+                    
+                        org.junit.vintage
+                        junit-vintage-engine
+                        ${version.junit.bom}
+                    
+                
                 
                     
                         integration-test
diff --git a/web-services/pom.xml b/web-services/pom.xml
index 767cbdf18d4..211e7d25df7 100644
--- a/web-services/pom.xml
+++ b/web-services/pom.xml
@@ -629,6 +629,18 @@
                         /dev/null
                     
                 
+                
+                    
+                        org.junit.jupiter
+                        junit-jupiter-engine
+                        ${version.junit.bom}
+                    
+                    
+                        org.junit.vintage
+                        junit-vintage-engine
+                        ${version.junit.bom}
+                    
+                
                 
                     
                         integration-test

From 97c7b60c6a822ef184a426247d678a72ad6d5ae7 Mon Sep 17 00:00:00 2001
From: Moriarty <22225248+apmoriarty@users.noreply.github.com>
Date: Thu, 1 Feb 2024 12:00:27 +0000
Subject: [PATCH 43/50] Optionally limit query execution based on the number of
 indexed equality terms

---
 .../query/config/ShardQueryConfiguration.java | 11 ++++
 .../visitors/IndexedTermCountingVisitor.java  | 55 +++++++++++++++++++
 .../query/planner/DefaultQueryPlanner.java    |  9 +++
 .../query/tables/ShardQueryLogic.java         |  8 +++
 .../config/ShardQueryConfigurationTest.java   |  2 +
 .../ExecutableExpansionVisitorTest.java       |  1 +
 .../IndexedTermCountingVisitorTest.java       | 44 +++++++++++++++
 .../query/planner/CompositeIndexTest.java     |  1 +
 .../query/planner/GeoSortedQueryDataTest.java |  1 +
 .../datawave/query/EventQueryLogicFactory.xml |  1 +
 .../datawave/query/QueryLogicFactory.xml      |  1 +
 11 files changed, 134 insertions(+)
 create mode 100644 warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IndexedTermCountingVisitor.java
 create mode 100644 warehouse/query-core/src/test/java/datawave/query/jexl/visitors/IndexedTermCountingVisitorTest.java

diff --git a/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java b/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java
index 6fbef8a1b4c..03c95651452 100644
--- a/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java
+++ b/warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java
@@ -321,6 +321,8 @@ public class ShardQueryConfiguration extends GenericQueryConfiguration implement
     private int initialMaxTermThreshold = 2500;
     // the intermediate term threshold is used to enforce a term limit prior to the range stream
     private int intermediateMaxTermThreshold = 2500;
+    // the maximum number of indexed terms to lookup in the global index. Practically this is the number of equality nodes in the query.
+    private int indexedMaxTermThreshold = 2500;
     private int finalMaxTermThreshold = 2500;
     private int maxDepthThreshold = 2500;
     private boolean expandFields = true;
@@ -591,6 +593,7 @@ public ShardQueryConfiguration(ShardQueryConfiguration other) {
         this.setShardsPerDayThreshold(other.getShardsPerDayThreshold());
         this.setInitialMaxTermThreshold(other.getInitialMaxTermThreshold());
         this.setIntermediateMaxTermThreshold(other.getIntermediateMaxTermThreshold());
+        this.setIndexedMaxTermThreshold(other.getIndexedMaxTermThreshold());
         this.setFinalMaxTermThreshold(other.getFinalMaxTermThreshold());
         this.setMaxDepthThreshold(other.getMaxDepthThreshold());
         this.setMaxUnfieldedExpansionThreshold(other.getMaxUnfieldedExpansionThreshold());
@@ -1216,6 +1219,14 @@ public void setIntermediateMaxTermThreshold(int intermediateMaxTermThreshold) {
         this.intermediateMaxTermThreshold = intermediateMaxTermThreshold;
     }
 
+    public int getIndexedMaxTermThreshold() {
+        return indexedMaxTermThreshold;
+    }
+
+    public void setIndexedMaxTermThreshold(int indexedMaxTermThreshold) {
+        this.indexedMaxTermThreshold = indexedMaxTermThreshold;
+    }
+
     public int getFinalMaxTermThreshold() {
         return finalMaxTermThreshold;
     }
diff --git a/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IndexedTermCountingVisitor.java b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IndexedTermCountingVisitor.java
new file mode 100644
index 00000000000..e69ee0c18e0
--- /dev/null
+++ b/warehouse/query-core/src/main/java/datawave/query/jexl/visitors/IndexedTermCountingVisitor.java
@@ -0,0 +1,55 @@
+package datawave.query.jexl.visitors;
+
+import java.util.Set;
+
+import org.apache.commons.jexl3.parser.ASTEQNode;
+import org.apache.commons.jexl3.parser.JexlNode;
+
+import datawave.query.jexl.JexlASTHelper;
+
+/**
+ * Helper visitor that counts the number of EQ nodes with an indexed field. Used to limit large queries prior to the range stream lookup.
+ */
+public class IndexedTermCountingVisitor extends ShortCircuitBaseVisitor {
+
+    private final Set indexedFields;
+    private int count = 0;
+
+    /**
+     * Counts the number of EQ nodes with an indexed field
+     *
+     * @param node
+     *            the query tree
+     * @param indexedFields
+     *            the set of indexed fields
+     * @return the number of indexed EQ nodes
+     */
+    public static int countTerms(JexlNode node, Set indexedFields) {
+        IndexedTermCountingVisitor visitor = new IndexedTermCountingVisitor(indexedFields);
+        node.jjtAccept(visitor, null);
+        return visitor.getCount();
+    }
+
+    /**
+     * Constructor that accepts a set of indexed fields
+     *
+     * @param indexedFields
+     *            a set of indexed fields
+     */
+    public IndexedTermCountingVisitor(Set indexedFields) {
+        this.indexedFields = indexedFields;
+    }
+
+    @Override
+    public Object visit(ASTEQNode node, Object data) {
+        String field = JexlASTHelper.getIdentifier(node);
+        if (indexedFields.contains(field)) {
+            count++;
+        }
+        return data;
+    }
+
+    public int getCount() {
+        return count;
+    }
+}
diff --git a/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java b/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java
index ff7aa805081..012427ba370 100644
--- a/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java
+++ b/warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java
@@ -112,6 +112,7 @@
 import datawave.query.jexl.visitors.FixUnindexedNumericTerms;
 import datawave.query.jexl.visitors.FunctionIndexQueryExpansionVisitor;
 import datawave.query.jexl.visitors.GeoWavePruningVisitor;
+import datawave.query.jexl.visitors.IndexedTermCountingVisitor;
 import datawave.query.jexl.visitors.IngestTypePruningVisitor;
 import datawave.query.jexl.visitors.InvertNodeVisitor;
 import datawave.query.jexl.visitors.IsNotNullIntentVisitor;
@@ -2572,6 +2573,14 @@ public Tuple2,Boolean> getQueryRanges(ScannerFactor
                                 "Query with " + termCount + " exceeds the initial max term threshold of " + config.getIntermediateMaxTermThreshold());
             }
 
+            if (config.getIndexedMaxTermThreshold() > 0) {
+                int indexedEqualityTerms = IndexedTermCountingVisitor.countTerms(config.getQueryTree(), config.getIndexedFields());
+                if (indexedEqualityTerms > config.getIndexedMaxTermThreshold()) {
+                    throw new DatawaveQueryException("Query with " + indexedEqualityTerms + " indexed EQ nodes exceeds the indexedMaxTermThreshold of "
+                                    + config.getIndexedMaxTermThreshold());
+                }
+            }
+
             if (termCount >= pushdownThreshold) {
                 if (log.isTraceEnabled()) {
                     log.trace("pushing down query because it has " + termCount + " when our max is " + pushdownThreshold);
diff --git a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java
index 951e85f4cfe..20d9d20fbd6 100644
--- a/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java
+++ b/warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java
@@ -1545,6 +1545,14 @@ public void setIntermediateMaxTermThreshold(int intermediateMaxTermThreshold) {
         getConfig().setIntermediateMaxTermThreshold(intermediateMaxTermThreshold);
     }
 
+    public int getIndexedMaxTermThreshold() {
+        return getConfig().getIndexedMaxTermThreshold();
+    }
+
+    public void setIndexedMaxTermThreshold(int indexedMaxTermThreshold) {
+        getConfig().setIndexedMaxTermThreshold(indexedMaxTermThreshold);
+    }
+
     public int getFinalMaxTermThreshold() {
         return getConfig().getFinalMaxTermThreshold();
     }
diff --git a/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java b/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java
index de9a3a5db96..28db4f7d2cf 100644
--- a/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java
@@ -325,6 +325,8 @@ public void setUp() throws Exception {
         updatedValues.put("initialMaxTermThreshold", 2540);
         defaultValues.put("intermediateMaxTermThreshold", 2500);
         updatedValues.put("intermediateMaxTermThreshold", 5500);
+        defaultValues.put("indexedMaxTermThreshold", 2500);
+        updatedValues.put("indexedMaxTermThreshold", 5500);
         defaultValues.put("finalMaxTermThreshold", 2500);
         updatedValues.put("finalMaxTermThreshold", 2501);
         defaultValues.put("maxDepthThreshold", 2500);
diff --git a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/ExecutableExpansionVisitorTest.java b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/ExecutableExpansionVisitorTest.java
index 4386e4c930d..693b2a753ee 100644
--- a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/ExecutableExpansionVisitorTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/ExecutableExpansionVisitorTest.java
@@ -292,6 +292,7 @@ public void testGeowaveExpansion() throws Exception {
         logic.setMaxDepthThreshold(30);
         logic.setInitialMaxTermThreshold(10000);
         logic.setIntermediateMaxTermThreshold(10000);
+        logic.setIndexedMaxTermThreshold(10000);
         logic.setFinalMaxTermThreshold(10000);
 
         //  @formatter:off
diff --git a/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/IndexedTermCountingVisitorTest.java b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/IndexedTermCountingVisitorTest.java
new file mode 100644
index 00000000000..043ed5686bd
--- /dev/null
+++ b/warehouse/query-core/src/test/java/datawave/query/jexl/visitors/IndexedTermCountingVisitorTest.java
@@ -0,0 +1,44 @@
+package datawave.query.jexl.visitors;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
+
+import java.util.Collections;
+
+import org.apache.commons.jexl3.parser.ASTJexlScript;
+import org.apache.commons.jexl3.parser.ParseException;
+import org.junit.jupiter.api.Test;
+
+import datawave.query.jexl.JexlASTHelper;
+
+class IndexedTermCountingVisitorTest {
+
+    @Test
+    void testSimpleCount() {
+        String query = "F == '1' || F == '2' || (F == '3' && F == '4')";
+        test(query, 4);
+    }
+
+    @Test
+    void testNoCount() {
+        String query = "A == '1' && B == '2'";
+        test(query, 0);
+    }
+
+    @Test
+    void testPartialCount() {
+        String query = "A == '1' && B == '2' && F == 'zee'";
+        test(query, 1);
+    }
+
+    private void test(String query, int expectedCount) {
+        try {
+            ASTJexlScript script = JexlASTHelper.parseAndFlattenJexlQuery(query);
+            int count = IndexedTermCountingVisitor.countTerms(script, Collections.singleton("F"));
+            assertEquals(expectedCount, count);
+        } catch (ParseException e) {
+            fail("Failed to parse query: " + query);
+        }
+    }
+
+}
diff --git a/warehouse/query-core/src/test/java/datawave/query/planner/CompositeIndexTest.java b/warehouse/query-core/src/test/java/datawave/query/planner/CompositeIndexTest.java
index 79f2cc92c28..c0fe6282a1b 100644
--- a/warehouse/query-core/src/test/java/datawave/query/planner/CompositeIndexTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/planner/CompositeIndexTest.java
@@ -350,6 +350,7 @@ public void compositeWithoutIvaratorTest() throws Exception {
 
         ShardQueryLogic logic = getShardQueryLogic(false);
         logic.setIntermediateMaxTermThreshold(50);
+        logic.setIndexedMaxTermThreshold(50);
         List queries = getQueryRanges(logic, query, false);
         Assert.assertEquals(12, queries.size());
 
diff --git a/warehouse/query-core/src/test/java/datawave/query/planner/GeoSortedQueryDataTest.java b/warehouse/query-core/src/test/java/datawave/query/planner/GeoSortedQueryDataTest.java
index 8d02d2f21e7..706b76712ba 100644
--- a/warehouse/query-core/src/test/java/datawave/query/planner/GeoSortedQueryDataTest.java
+++ b/warehouse/query-core/src/test/java/datawave/query/planner/GeoSortedQueryDataTest.java
@@ -277,6 +277,7 @@ public void setupTest() {
         logic.setMaxDepthThreshold(10);
 
         logic.setIntermediateMaxTermThreshold(50);
+        logic.setIndexedMaxTermThreshold(50);
 
         // set the pushdown threshold really high to avoid collapsing uids into shards (overrides setCollapseUids if #terms is greater than this threshold)
         ((DefaultQueryPlanner) (logic.getQueryPlanner())).setPushdownThreshold(1000000);
diff --git a/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml b/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml
index 16e7bbd58d3..bcd5fef1646 100644
--- a/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml
+++ b/warehouse/query-core/src/test/resources/datawave/query/EventQueryLogicFactory.xml
@@ -98,6 +98,7 @@
         
         	
         
+        
         
         	
         
diff --git a/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml b/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml
index 7e2c2bc64cf..43e06468d05 100644
--- a/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml
+++ b/warehouse/query-core/src/test/resources/datawave/query/QueryLogicFactory.xml
@@ -199,6 +199,7 @@
         
         	
         
+        
         
         	
         

From 6470036f0f941c95c6fbb799d5d8516925bceb1b Mon Sep 17 00:00:00 2001
From: Ivan Bella 
Date: Tue, 20 Feb 2024 13:00:59 -0500
Subject: [PATCH 44/50] re #2270: Iimplementation of a record filter for the
 ingest processing (#2271)

* Created an event predicate for discarding events during ingest processing
* Created a discard interval and future discard interval predicates
* Updated tests to test hit and miss on the interval predicates
* Add default for DISCARD_FUTURE_INTERVAL event

Co-authored-by: foster33 
Co-authored-by: Keith Ratcliffe 
Co-authored-by: hlgp 
Co-authored-by: palindrome <31748527+hlgp@users.noreply.github.com>
Co-authored-by: matthpeterson 
---
 .../bin/services/datawave/bootstrap.sh        |   1 +
 properties/dev.properties                     |   3 +
 .../main/resources/config/ingest-config.xml   |   5 +
 ...ataTypeDiscardFutureIntervalPredicate.java |  59 ++++++
 .../DataTypeDiscardIntervalPredicate.java     |  59 ++++++
 .../ingest/mapreduce/EventMapper.java         |  90 ++++++---
 .../ingest/mapreduce/RawRecordPredicate.java  |  54 +++++
 .../datawave/ingest/metric/IngestInput.java   |   2 +-
 .../ingest/mapreduce/EventMapperTest.java     | 191 ++++++++++++++++++
 .../HadoopTestConfiguration.java              |   3 +-
 10 files changed, 438 insertions(+), 29 deletions(-)
 create mode 100644 warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/DataTypeDiscardFutureIntervalPredicate.java
 create mode 100644 warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/DataTypeDiscardIntervalPredicate.java
 create mode 100644 warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/RawRecordPredicate.java

diff --git a/contrib/datawave-quickstart/bin/services/datawave/bootstrap.sh b/contrib/datawave-quickstart/bin/services/datawave/bootstrap.sh
index f66994423f9..898828f357f 100644
--- a/contrib/datawave-quickstart/bin/services/datawave/bootstrap.sh
+++ b/contrib/datawave-quickstart/bin/services/datawave/bootstrap.sh
@@ -184,6 +184,7 @@ function setBuildPropertyOverrides() {
    echo "mapReduce.job.tracker=${DW_HADOOP_RESOURCE_MANAGER_ADDRESS_CLIENT}" >> ${BUILD_PROPERTIES_FILE}
    echo "bulkResults.job.tracker=${DW_HADOOP_RESOURCE_MANAGER_ADDRESS_CLIENT}" >> ${BUILD_PROPERTIES_FILE}
    echo "EVENT_DISCARD_INTERVAL=0" >> ${BUILD_PROPERTIES_FILE}
+   echo "EVENT_DISCARD_FUTURE_INTERVAL=0" >> ${BUILD_PROPERTIES_FILE}
    echo "ingest.data.types=${DW_DATAWAVE_INGEST_LIVE_DATA_TYPES},${DW_DATAWAVE_INGEST_BULK_DATA_TYPES}" >> ${BUILD_PROPERTIES_FILE}
    echo "JOB_CACHE_REPLICATION=1" >> ${BUILD_PROPERTIES_FILE}
    echo "EDGE_DEFINITION_FILE=${DW_DATAWAVE_INGEST_EDGE_DEFINITIONS}" >> ${BUILD_PROPERTIES_FILE}
diff --git a/properties/dev.properties b/properties/dev.properties
index b6c7d8bb355..73ae81eba4c 100644
--- a/properties/dev.properties
+++ b/properties/dev.properties
@@ -88,6 +88,9 @@ PYTHON=/usr/bin/python
 # Setting discard interval to 0 in order to disable auto-ageoff @ ingest time
 EVENT_DISCARD_INTERVAL=0
 
+# Setting discard interval to 0 in order to disable auto-ageoff @ ingest time
+EVENT_DISCARD_FUTURE_INTERVAL=0
+
 DATAWAVE_CACHE_PORT=20444
 
 EDGE_DEFINITION_FILE=config/edge-definitions.xml
diff --git a/warehouse/ingest-configuration/src/main/resources/config/ingest-config.xml b/warehouse/ingest-configuration/src/main/resources/config/ingest-config.xml
index ce5c2e2cb0c..b23a72e2670 100644
--- a/warehouse/ingest-configuration/src/main/resources/config/ingest-config.xml
+++ b/warehouse/ingest-configuration/src/main/resources/config/ingest-config.xml
@@ -28,6 +28,11 @@
   ${EVENT_DISCARD_INTERVAL}
 
 
+    
+        event.discard.future.interval
+        ${EVENT_DISCARD_FUTURE_INTERVAL}
+    
+
 
   snowflake.zookeepers
   ${SNOWFLAKE_ZOOKEEPERS}
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/DataTypeDiscardFutureIntervalPredicate.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/DataTypeDiscardFutureIntervalPredicate.java
new file mode 100644
index 00000000000..71d6bcb7fe0
--- /dev/null
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/DataTypeDiscardFutureIntervalPredicate.java
@@ -0,0 +1,59 @@
+package datawave.ingest.mapreduce;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Logger;
+
+import datawave.ingest.data.RawRecordContainer;
+import datawave.ingest.metric.IngestInput;
+import datawave.ingest.time.Now;
+
+public class DataTypeDiscardFutureIntervalPredicate implements RawRecordPredicate {
+
+    private static final Logger log = Logger.getLogger(DataTypeDiscardFutureIntervalPredicate.class);
+
+    /**
+     * number which will be used to evaluate whether or not an Event should be processed. If the Event.getEventDate() is less than (now + interval) then it will
+     * be processed.
+     */
+    public static final String DISCARD_FUTURE_INTERVAL = "event.discard.future.interval";
+
+    private static final Now now = Now.getInstance();
+
+    private long discardFutureInterval = 0L;
+
+    @Override
+    public void setConfiguration(String type, Configuration conf) {
+        long defaultInterval = conf.getLong(DISCARD_FUTURE_INTERVAL, 0l);
+        this.discardFutureInterval = conf.getLong(type + "." + DISCARD_FUTURE_INTERVAL, defaultInterval);
+        log.info("Setting up type: " + type + " with future interval " + this.discardFutureInterval);
+    }
+
+    @Override
+    public boolean shouldProcess(RawRecordContainer record) {
+        // Determine whether the event date is greater than the interval. Excluding fatal error events.
+        if (discardFutureInterval != 0L && (record.getDate() > (now.get() + discardFutureInterval))) {
+            if (log.isInfoEnabled())
+                log.info("Event with time " + record.getDate() + " newer than specified interval of " + (now.get() + discardFutureInterval) + ", skipping...");
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public String getCounterName() {
+        return IngestInput.FUTURE_EVENT.name();
+    }
+
+    @Override
+    public int hashCode() {
+        return (int) discardFutureInterval;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj instanceof DataTypeDiscardFutureIntervalPredicate) {
+            return discardFutureInterval == (((DataTypeDiscardFutureIntervalPredicate) obj).discardFutureInterval);
+        }
+        return false;
+    }
+}
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/DataTypeDiscardIntervalPredicate.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/DataTypeDiscardIntervalPredicate.java
new file mode 100644
index 00000000000..69a0857cc26
--- /dev/null
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/DataTypeDiscardIntervalPredicate.java
@@ -0,0 +1,59 @@
+package datawave.ingest.mapreduce;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Logger;
+
+import datawave.ingest.data.RawRecordContainer;
+import datawave.ingest.metric.IngestInput;
+import datawave.ingest.time.Now;
+
+public class DataTypeDiscardIntervalPredicate implements RawRecordPredicate {
+
+    private static final Logger log = Logger.getLogger(DataTypeDiscardIntervalPredicate.class);
+
+    /**
+     * number which will be used to evaluate whether or not an Event should be processed. If the Event.getEventDate() is greater than (now - interval) then it
+     * will be processed.
+     */
+    public static final String DISCARD_INTERVAL = "event.discard.interval";
+
+    private static final Now now = Now.getInstance();
+
+    private long discardInterval = 0L;
+
+    @Override
+    public void setConfiguration(String type, Configuration conf) {
+        long defaultInterval = conf.getLong(DISCARD_INTERVAL, 0l);
+        this.discardInterval = conf.getLong(type + "." + DISCARD_INTERVAL, defaultInterval);
+        log.info("Setting up type: " + type + " with interval " + this.discardInterval);
+    }
+
+    @Override
+    public boolean shouldProcess(RawRecordContainer record) {
+        // Determine whether the event date is greater than the interval. Excluding fatal error events.
+        if (discardInterval != 0L && (record.getDate() < (now.get() - discardInterval))) {
+            if (log.isInfoEnabled())
+                log.info("Event with time " + record.getDate() + " older than specified interval of " + (now.get() - discardInterval) + ", skipping...");
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public String getCounterName() {
+        return IngestInput.OLD_EVENT.name();
+    }
+
+    @Override
+    public int hashCode() {
+        return (int) discardInterval;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj instanceof DataTypeDiscardIntervalPredicate) {
+            return discardInterval == (((DataTypeDiscardIntervalPredicate) obj).discardInterval);
+        }
+        return false;
+    }
+}
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/EventMapper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/EventMapper.java
index 86e600060f8..edce34ef644 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/EventMapper.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/EventMapper.java
@@ -5,6 +5,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -16,6 +17,7 @@
 import java.util.Stack;
 import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 import org.apache.accumulo.core.data.Mutation;
 import org.apache.accumulo.core.data.Value;
@@ -100,11 +102,11 @@ public class EventMapper extends StatsDE
 
     private static final Logger log = Logger.getLogger(EventMapper.class);
 
-    /**
-     * number which will be used to evaluate whether or not an Event should be processed. If the Event.getEventDate() is greater than (now - interval) then it
-     * will be processed.
-     */
-    public static final String DISCARD_INTERVAL = "event.discard.interval";
+    // for backward compatibility, these DISCARD constants are maintained here as well
+    public static final String DISCARD_INTERVAL = DataTypeDiscardIntervalPredicate.DISCARD_INTERVAL;
+    public static final String DISCARD_FUTURE_INTERVAL = DataTypeDiscardFutureIntervalPredicate.DISCARD_FUTURE_INTERVAL;
+
+    public static final String RECORD_PREDICATES = "event.predicates";
 
     public static final String CONTEXT_WRITER_CLASS = "ingest.event.mapper.context.writer.class";
     public static final String CONTEXT_WRITER_OUTPUT_TABLE_COUNTERS = "ingest.event.mapper.context.writer.output.table.counters";
@@ -132,14 +134,16 @@ public class EventMapper extends StatsDE
 
     protected Map>> typeMap = new HashMap<>();
 
-    /**
-     * might as well cache the discard interval
-     */
-    protected Map dataTypeDiscardIntervalCache = new HashMap<>();
+    // Predicates are used to filter out events if needed. If predicates exist
+    // for a datatype, then the predicates need to all pass (return true)
+    // inorder to ingest the record. The event is otherwise dropped and a counter is
+    // incremented for the predicate class.
+    protected Map> predicateMap = new HashMap<>();
 
-    private FileSplit split = null;
+    // base set of predicates
+    private Collection predicates = Collections.emptyList();
 
-    private long interval = 0l;
+    private FileSplit split = null;
 
     private static Now now = Now.getInstance();
 
@@ -187,7 +191,11 @@ public void setup(Context context) throws IOException, InterruptedException {
         // Initialize the Type Registry
         TypeRegistry.getInstance(context.getConfiguration());
 
-        interval = context.getConfiguration().getLong(DISCARD_INTERVAL, 0l);
+        // load the predicates applied to all types
+        predicates = new HashSet<>(context.getConfiguration().getTrimmedStringCollection(RECORD_PREDICATES));
+        // always add the discard interval predicates
+        predicates.add(DataTypeDiscardIntervalPredicate.class.getName());
+        predicates.add(DataTypeDiscardFutureIntervalPredicate.class.getName());
 
         // default to true, but it can be disabled
         createSequenceFileName = context.getConfiguration().getBoolean(LOAD_SEQUENCE_FILE_NAME, true);
@@ -286,7 +294,7 @@ public void setup(Context context) throws IOException, InterruptedException {
     }
 
     /**
-     * Get the data type handlers for a given type name. This will also fill the dataTypeDiscardIntervalCache and the validators as a side effect.
+     * Get the data type handlers for a given type name. This will also fill the predicate map and the validators as a side effect.
      *
      * @param typeStr
      *            name of the type
@@ -300,11 +308,7 @@ private List> loadDataType(String typeStr, Context context)
 
             typeMap.put(typeStr, new ArrayList<>());
 
-            long myInterval = context.getConfiguration().getLong(typeStr + "." + DISCARD_INTERVAL, interval);
-
-            dataTypeDiscardIntervalCache.put(typeStr, myInterval);
-
-            log.info("Setting up type: " + typeStr + " with interval " + myInterval);
+            predicateMap.put(typeStr, getPredicates(typeStr, context, predicates));
 
             if (!TypeRegistry.getTypeNames().contains(typeStr)) {
                 log.warn("Attempted to load configuration for a type that does not exist in the registry: " + typeStr);
@@ -364,6 +368,35 @@ private List> loadDataType(String typeStr, Context context)
         return typeMap.get(typeStr);
     }
 
+    private Set getPredicates(final String type, final Context context, final Collection basePredicates) {
+        Collection predicateClasses = new HashSet<>(context.getConfiguration().getTrimmedStringCollection(type + "." + RECORD_PREDICATES));
+        predicateClasses.addAll(basePredicates);
+        if (!predicateClasses.isEmpty()) {
+            return predicateClasses.stream().map(s -> {
+                try {
+                    return Class.forName(s);
+                } catch (ClassNotFoundException e) {
+                    throw new IllegalArgumentException("Cannot load predicate for type " + type + ": " + s, e);
+                }
+            }).filter(c -> {
+                if (!RawRecordPredicate.class.isAssignableFrom(c)) {
+                    throw new IllegalArgumentException("Predicate " + c.getName() + " for type " + type + " is not a RawRecordPredicate.");
+                }
+                return true;
+            }).map(c -> {
+                try {
+                    RawRecordPredicate predicate = (RawRecordPredicate) c.getDeclaredConstructor().newInstance();
+                    predicate.setConfiguration(type, context.getConfiguration());
+                    return predicate;
+                } catch (Exception e) {
+                    throw new IllegalArgumentException("Predicate " + c.getName() + " for type " + type + " could not be constructed.", e);
+                }
+            }).collect(Collectors.toSet());
+        } else {
+            return Collections.EMPTY_SET;
+        }
+    }
+
     private List getDataTypeFilterClassNames() {
 
         SortedMap priorityToFilters = new TreeMap<>();
@@ -394,7 +427,7 @@ public void map(K1 key, V1 value, Context context) throws IOException, Interrupt
             eventMapperTimer.start();
         }
 
-        // ensure this datatype's handlers etc are loaded such that the dataTypeDiscardIntervalCache and validators are filled as well
+        // ensure this datatype's handlers etc are loaded such that the predicates and validators are filled as well
         List> typeHandlers = loadDataType(value.getDataType().typeName(), context);
 
         // This is a little bit fragile, but there is no other way
@@ -402,8 +435,6 @@ public void map(K1 key, V1 value, Context context) throws IOException, Interrupt
         // using this to set some counters that collect stats.
         MultiTableRangePartitioner.setContext(context);
 
-        Long myInterval = dataTypeDiscardIntervalCache.get(value.getDataType().typeName());
-
         // setup the configuration on the event
         // this is automatically done by the sequence reader....
         // value.setConf(context.getConfiguration());
@@ -463,12 +494,17 @@ public void map(K1 key, V1 value, Context context) throws IOException, Interrupt
             value.setAuxProperty(ErrorDataTypeHandler.PROCESSED_COUNT, "1");
         }
 
-        // Determine whether the event date is greater than the interval. Excluding fatal error events.
-        if (!value.fatalError() && null != myInterval && 0L != myInterval && (value.getDate() < (now.get() - myInterval))) {
-            if (log.isInfoEnabled())
-                log.info("Event with time " + value.getDate() + " older than specified interval of " + (now.get() - myInterval) + ", skipping...");
-            getCounter(context, IngestInput.OLD_EVENT).increment(1);
-            return;
+        if (!value.fatalError()) {
+            // Determine whether the event should be filtered for any other reason
+            Set predicates = predicateMap.get(value.getDataType().typeName());
+            if (null != predicates && !predicates.isEmpty()) {
+                for (RawRecordPredicate predicate : predicates) {
+                    if (!predicate.test(value)) {
+                        getCounter(context, IngestInput.FILTER.name(), predicate.getCounterName()).increment(1);
+                        return;
+                    }
+                }
+            }
         }
 
         // Add the list of handlers with the ALL specified handlers
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/RawRecordPredicate.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/RawRecordPredicate.java
new file mode 100644
index 00000000000..a2246ca5338
--- /dev/null
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/RawRecordPredicate.java
@@ -0,0 +1,54 @@
+package datawave.ingest.mapreduce;
+
+import java.util.function.Predicate;
+
+import org.apache.hadoop.conf.Configuration;
+
+import datawave.ingest.data.RawRecordContainer;
+
+/**
+ * An implementation of this interface can be used with the EventMapper to filter out events that should not be processed. It is expected that there is an empty
+ * constructor. The setConfiguration(Configuration) method will be called prior to any filtering such that the class can configure itself appropriately.
+ */
+public interface RawRecordPredicate extends Predicate {
+
+    /**
+     * This is the main method used to filter out records that should not be processed.
+     *
+     * @param record
+     *            The raw record container under review
+     * @return true if the event is ok to ingest
+     */
+    boolean shouldProcess(RawRecordContainer record);
+
+    /**
+     * This method will be called after configuration with the map-reduce configuration.
+     *
+     * @param type
+     *            The datatype for which this predicate is being constructed
+     * @param conf
+     *            The hadoop configuration object
+     */
+    default void setConfiguration(String type, Configuration conf) {}
+
+    /**
+     * The counter name used for records that are dropped. Uses the simple name of the class implementation by default.
+     *
+     * @return The counter name
+     */
+    default String getCounterName() {
+        return this.getClass().getSimpleName();
+    }
+
+    /**
+     * The implementation of the java util Predicate method. This method should not be overridden.
+     *
+     * @param record
+     *            The raw record container under review
+     * @return true if the record should be ingested
+     */
+    @Override
+    default boolean test(RawRecordContainer record) {
+        return shouldProcess(record);
+    }
+}
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/metric/IngestInput.java b/warehouse/ingest-core/src/main/java/datawave/ingest/metric/IngestInput.java
index 4d25dc309ce..170ae64442f 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/metric/IngestInput.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/metric/IngestInput.java
@@ -1,5 +1,5 @@
 package datawave.ingest.metric;
 
 public enum IngestInput {
-    OLD_EVENT, FILE_NAME, EVENT_FATAL_ERROR, EVENT_ERROR_TYPE, EVENT_IGNORABLE_ERROR, LINE_BYTES;
+    OLD_EVENT, FUTURE_EVENT, FILE_NAME, EVENT_FATAL_ERROR, EVENT_ERROR_TYPE, EVENT_IGNORABLE_ERROR, LINE_BYTES, FILTER;
 }
diff --git a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/EventMapperTest.java b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/EventMapperTest.java
index 97dcf8dfb93..b0248906afa 100644
--- a/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/EventMapperTest.java
+++ b/warehouse/ingest-core/src/test/java/datawave/ingest/mapreduce/EventMapperTest.java
@@ -10,7 +10,9 @@
 import static org.junit.Assert.assertNotNull;
 
 import java.io.IOException;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.accumulo.core.data.Value;
 import org.apache.hadoop.conf.Configuration;
@@ -26,7 +28,9 @@
 
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
 
+import datawave.data.hash.UIDConstants;
 import datawave.ingest.data.RawRecordContainer;
 import datawave.ingest.data.Type;
 import datawave.ingest.data.TypeRegistry;
@@ -199,6 +203,127 @@ public void shouldNotWriteRawFile() throws IOException, InterruptedException {
         assertEquals(4, written.size());
     }
 
+    public abstract static class TestPredicate implements RawRecordPredicate {
+        public static ThreadLocal> seenTypes = ThreadLocal.withInitial(() -> new HashSet<>());
+        public static ThreadLocal> allowed = ThreadLocal.withInitial(() -> new HashSet<>());
+        public static ThreadLocal> denied = ThreadLocal.withInitial(() -> new HashSet<>());
+
+        public static void reset() {
+            seenTypes.get().clear();
+            allowed.get().clear();
+            denied.get().clear();
+        }
+
+        @Override
+        public void setConfiguration(String type, Configuration conf) {
+            seenTypes.get().add(type);
+        }
+
+        @Override
+        public boolean test(RawRecordContainer record) {
+            boolean value = RawRecordPredicate.super.test(record);
+            if (value) {
+                allowed.get().add(record);
+            } else {
+                denied.get().add(record);
+            }
+            return value;
+        }
+
+        @Override
+        public int hashCode() {
+            return getClass().hashCode();
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            return getClass().equals(obj.getClass());
+        }
+    }
+
+    public static class AlwaysProcessPredicate extends TestPredicate {
+        @Override
+        public boolean shouldProcess(RawRecordContainer record) {
+            return true;
+        }
+    }
+
+    public static class DroppingAllPredicate extends TestPredicate {
+        @Override
+        public boolean shouldProcess(RawRecordContainer record) {
+            return false;
+        }
+    }
+
+    public static class CantConfigureEventPredicate extends AlwaysProcessPredicate implements RawRecordPredicate {
+        @Override
+        public void setConfiguration(String type, Configuration conf) {
+            throw new RuntimeException();
+        }
+    }
+
+    @Test
+    public void testBaseEventPredicates() throws IOException, InterruptedException {
+        try {
+            conf.set(EventMapper.RECORD_PREDICATES, AlwaysProcessPredicate.class.getName());
+            eventMapper.setup(mapContext);
+            eventMapper.map(new LongWritable(1), record, mapContext);
+            eventMapper.cleanup(mapContext);
+            Multimap written = TestContextWriter.getWritten();
+            assertEquals(5, written.size());
+            assertEquals(Sets.newHashSet("all", "file"), TestPredicate.seenTypes.get());
+            assertEquals(0, TestPredicate.denied.get().size());
+            assertEquals(1, TestPredicate.allowed.get().size());
+        } finally {
+            TestPredicate.reset();
+        }
+    }
+
+    @Test
+    public void testTypePredicates() throws IOException, InterruptedException {
+        try {
+            conf.set("file." + EventMapper.RECORD_PREDICATES, AlwaysProcessPredicate.class.getName());
+            eventMapper.setup(mapContext);
+            eventMapper.map(new LongWritable(1), record, mapContext);
+            eventMapper.cleanup(mapContext);
+            Multimap written = TestContextWriter.getWritten();
+            assertEquals(5, written.size());
+            assertEquals(Sets.newHashSet("file"), TestPredicate.seenTypes.get());
+            assertEquals(0, TestPredicate.denied.get().size());
+            assertEquals(1, TestPredicate.allowed.get().size());
+        } finally {
+            TestPredicate.reset();
+        }
+    }
+
+    @Test
+    public void testMultTypePredicates() throws IOException, InterruptedException {
+        try {
+            conf.set("file." + EventMapper.RECORD_PREDICATES, AlwaysProcessPredicate.class.getName() + "," + DroppingAllPredicate.class.getName());
+            eventMapper.setup(mapContext);
+            eventMapper.map(new LongWritable(1), record, mapContext);
+            eventMapper.cleanup(mapContext);
+            Multimap written = TestContextWriter.getWritten();
+            assertEquals(0, written.size());
+            assertEquals(Sets.newHashSet("file"), TestPredicate.seenTypes.get());
+            assertEquals(1, TestPredicate.denied.get().size());
+            // allowed size may be 1 or 0, depending on the order of the predicates in the set. No need to test allowed size.
+        } finally {
+            TestPredicate.reset();
+        }
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testFailedPredicates() throws IOException, InterruptedException {
+        try {
+            conf.set("file." + EventMapper.RECORD_PREDICATES, CantConfigureEventPredicate.class.getName());
+            eventMapper.setup(mapContext);
+            eventMapper.map(new LongWritable(1), record, mapContext);
+        } finally {
+            TestPredicate.reset();
+        }
+    }
+
     @Test
     public void errorEventWithZeroTimestampNotDropped() throws IOException, InterruptedException {
         eventMapper.setup(mapContext);
@@ -212,6 +337,72 @@ public void errorEventWithZeroTimestampNotDropped() throws IOException, Interrup
         assertEquals(4, written.size());
     }
 
+    @Test
+    public void testHitDiscardInterval() throws IOException, InterruptedException {
+        // event date < now minus discard interval
+        long yesterday = -1L * UIDConstants.MILLISECONDS_PER_DAY;
+        conf.setLong(record.getDataType().typeName() + "." + DataTypeDiscardIntervalPredicate.DISCARD_INTERVAL, yesterday);
+
+        eventMapper.setup(mapContext);
+        eventMapper.map(new LongWritable(1), record, mapContext);
+        eventMapper.cleanup(mapContext);
+
+        Multimap written = TestContextWriter.getWritten();
+
+        // discard interval lower bound is yesterday. data from today should be dropped
+        assertEquals(0, written.size());
+    }
+
+    @Test
+    public void testMissDiscardInterval() throws IOException, InterruptedException {
+        // event date < now minus discard interval
+        long tomorrow = 1L * UIDConstants.MILLISECONDS_PER_DAY;
+        conf.setLong(record.getDataType().typeName() + "." + DataTypeDiscardIntervalPredicate.DISCARD_INTERVAL, tomorrow);
+
+        eventMapper.setup(mapContext);
+        eventMapper.map(new LongWritable(1), record, mapContext);
+        eventMapper.cleanup(mapContext);
+
+        Multimap written = TestContextWriter.getWritten();
+
+        // discard interval lower bound is tomorrow. data from today should be ok
+        assertEquals(5, written.size());
+    }
+
+    @Test
+    public void testHitFutureDiscardInterval() throws IOException, InterruptedException {
+        // event date > now plus future discard interval
+        long yesterday = -1L * UIDConstants.MILLISECONDS_PER_DAY;
+        conf.setLong(record.getDataType().typeName() + "." + DataTypeDiscardFutureIntervalPredicate.DISCARD_FUTURE_INTERVAL, yesterday);
+
+        eventMapper.setup(mapContext);
+        eventMapper.map(new LongWritable(1), record, mapContext);
+        eventMapper.cleanup(mapContext);
+
+        Multimap written = TestContextWriter.getWritten();
+
+        // future discard is yesterday. data from today should be dropped
+        assertEquals(0, written.size());
+
+    }
+
+    @Test
+    public void testMissFutureDiscardInterval() throws IOException, InterruptedException {
+        // event date > now plus future discard interval
+        long tomorrow = 1L * UIDConstants.MILLISECONDS_PER_DAY;
+        conf.setLong(record.getDataType().typeName() + "." + DataTypeDiscardFutureIntervalPredicate.DISCARD_FUTURE_INTERVAL, tomorrow);
+
+        eventMapper.setup(mapContext);
+        eventMapper.map(new LongWritable(1), record, mapContext);
+        eventMapper.cleanup(mapContext);
+
+        Multimap written = TestContextWriter.getWritten();
+
+        // future discard is tomorrow. data from today should be ok
+        assertEquals(5, written.size());
+
+    }
+
     private Map.Entry getMetric(Multimap written) {
         return getFieldEntry(written, Metric.EVENT_COUNT.toString());
     }
diff --git a/warehouse/query-core/src/test/java/datawave/query/testframework/HadoopTestConfiguration.java b/warehouse/query-core/src/test/java/datawave/query/testframework/HadoopTestConfiguration.java
index e002a90a6c6..e71f1423934 100644
--- a/warehouse/query-core/src/test/java/datawave/query/testframework/HadoopTestConfiguration.java
+++ b/warehouse/query-core/src/test/java/datawave/query/testframework/HadoopTestConfiguration.java
@@ -8,6 +8,7 @@
 import org.apache.log4j.Logger;
 
 import datawave.ingest.data.TypeRegistry;
+import datawave.ingest.mapreduce.DataTypeDiscardIntervalPredicate;
 import datawave.ingest.mapreduce.EventMapper;
 import datawave.ingest.mapreduce.handler.dateindex.DateIndexDataTypeHandler;
 import datawave.ingest.mapreduce.handler.shard.ShardedDataTypeHandler;
@@ -58,7 +59,7 @@ public HadoopTestConfiguration(DataTypeHadoopConfig dataType, Map
         this.set("multiple.numshards.enable", "false");
 
         // mapper options
-        this.set(EventMapper.DISCARD_INTERVAL, Long.toString(Long.MAX_VALUE));
+        this.set(DataTypeDiscardIntervalPredicate.DISCARD_INTERVAL, Long.toString(Long.MAX_VALUE));
         this.set(EventMapper.CONTEXT_WRITER_OUTPUT_TABLE_COUNTERS, Boolean.TRUE.toString());
         this.set(EventMapper.CONTEXT_WRITER_OUTPUT_TABLE_COUNTERS, Boolean.FALSE.toString());
         this.set(ShardedDataTypeHandler.NUM_SHARDS, "1");

From dcb6a281079c67e63f4967ada3c5fc2d91937b55 Mon Sep 17 00:00:00 2001
From: hgklohr 
Date: Tue, 20 Feb 2024 18:32:51 +0000
Subject: [PATCH 45/50] Update pom's for 6.8.0-SNAPSHOT

---
 common-test/pom.xml                                      | 2 +-
 contrib/datawave-quickstart/docker/pom.xml               | 2 +-
 core/pom.xml                                             | 2 +-
 core/utils/pom.xml                                       | 2 +-
 docs/pom.xml                                             | 2 +-
 microservices/pom.xml                                    | 2 +-
 microservices/services/pom.xml                           | 2 +-
 microservices/starters/pom.xml                           | 2 +-
 pom.xml                                                  | 2 +-
 warehouse/accumulo-extensions/pom.xml                    | 2 +-
 warehouse/assemble/datawave/pom.xml                      | 2 +-
 warehouse/assemble/pom.xml                               | 2 +-
 warehouse/assemble/webservice/pom.xml                    | 2 +-
 warehouse/common/pom.xml                                 | 2 +-
 warehouse/core/pom.xml                                   | 2 +-
 warehouse/data-dictionary-core/pom.xml                   | 2 +-
 warehouse/edge-dictionary-core/pom.xml                   | 2 +-
 warehouse/edge-model-configuration-core/pom.xml          | 2 +-
 warehouse/index-stats/pom.xml                            | 2 +-
 warehouse/ingest-configuration/pom.xml                   | 2 +-
 warehouse/ingest-core/pom.xml                            | 2 +-
 warehouse/ingest-csv/pom.xml                             | 2 +-
 warehouse/ingest-json/pom.xml                            | 2 +-
 warehouse/ingest-nyctlc/pom.xml                          | 2 +-
 warehouse/ingest-scripts/pom.xml                         | 2 +-
 warehouse/ingest-ssdeep/pom.xml                          | 2 +-
 warehouse/ingest-wikipedia/pom.xml                       | 2 +-
 warehouse/metrics-core/pom.xml                           | 2 +-
 warehouse/ops-tools/config-compare/pom.xml               | 2 +-
 warehouse/ops-tools/index-validation/pom.xml             | 2 +-
 warehouse/ops-tools/pom.xml                              | 2 +-
 warehouse/pom.xml                                        | 2 +-
 warehouse/query-core/pom.xml                             | 2 +-
 warehouse/regression-testing/pom.xml                     | 2 +-
 warehouse/ssdeep-common/pom.xml                          | 2 +-
 web-services/accumulo/pom.xml                            | 2 +-
 web-services/atom/pom.xml                                | 2 +-
 web-services/cached-results/pom.xml                      | 2 +-
 web-services/client/pom.xml                              | 2 +-
 web-services/common-util/pom.xml                         | 2 +-
 web-services/common/pom.xml                              | 2 +-
 web-services/deploy/application/pom.xml                  | 2 +-
 web-services/deploy/configuration/pom.xml                | 2 +-
 web-services/deploy/docs/pom.xml                         | 2 +-
 web-services/deploy/pom.xml                              | 2 +-
 web-services/deploy/spring-framework-integration/pom.xml | 2 +-
 web-services/dictionary/pom.xml                          | 2 +-
 web-services/examples/client-login/pom.xml               | 2 +-
 web-services/examples/http-client/pom.xml                | 2 +-
 web-services/examples/jms-client/pom.xml                 | 2 +-
 web-services/examples/pom.xml                            | 2 +-
 web-services/examples/query-war/pom.xml                  | 2 +-
 web-services/map-reduce-embedded/pom.xml                 | 2 +-
 web-services/map-reduce-status/pom.xml                   | 2 +-
 web-services/map-reduce/pom.xml                          | 2 +-
 web-services/model/pom.xml                               | 2 +-
 web-services/modification/pom.xml                        | 2 +-
 web-services/pom.xml                                     | 2 +-
 web-services/query-websocket/pom.xml                     | 2 +-
 web-services/query/pom.xml                               | 2 +-
 web-services/rest-api/pom.xml                            | 2 +-
 web-services/security/pom.xml                            | 2 +-
 web-services/web-root/pom.xml                            | 2 +-
 63 files changed, 63 insertions(+), 63 deletions(-)

diff --git a/common-test/pom.xml b/common-test/pom.xml
index 381b91105e1..fcf84396e50 100644
--- a/common-test/pom.xml
+++ b/common-test/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-common-test
     ${project.artifactId}
diff --git a/contrib/datawave-quickstart/docker/pom.xml b/contrib/datawave-quickstart/docker/pom.xml
index 5af989617cf..0f09a781fbd 100644
--- a/contrib/datawave-quickstart/docker/pom.xml
+++ b/contrib/datawave-quickstart/docker/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
         ../../../pom.xml
     
     quickstart
diff --git a/core/pom.xml b/core/pom.xml
index 0617eeaf717..1d1fd2a2966 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     gov.nsa.datawave.core
     datawave-core-parent
diff --git a/core/utils/pom.xml b/core/utils/pom.xml
index f412ce40ea8..0fc382626a1 100644
--- a/core/utils/pom.xml
+++ b/core/utils/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.core
         datawave-core-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     gov.nsa.datawave.core
     datawave-utils-parent
diff --git a/docs/pom.xml b/docs/pom.xml
index c57c473f570..ad1f63a0a6b 100644
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-docs
     
diff --git a/microservices/pom.xml b/microservices/pom.xml
index 446236890f8..6145cee9026 100644
--- a/microservices/pom.xml
+++ b/microservices/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     gov.nsa.datawave.microservice
     datawave-microservice-build-parent
diff --git a/microservices/services/pom.xml b/microservices/services/pom.xml
index 42f8ffa058d..5da825d7ed1 100644
--- a/microservices/services/pom.xml
+++ b/microservices/services/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.microservice
         datawave-microservice-build-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-microservice-service-build-parent
     pom
diff --git a/microservices/starters/pom.xml b/microservices/starters/pom.xml
index ed43f1da156..1391a7a20a5 100644
--- a/microservices/starters/pom.xml
+++ b/microservices/starters/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.microservice
         datawave-microservice-build-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-microservice-starter-build-parent
     pom
diff --git a/pom.xml b/pom.xml
index f76a4534115..de5d65f9237 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
     4.0.0
     gov.nsa.datawave
     datawave-parent
-    6.7.0-SNAPSHOT
+    6.8.0-SNAPSHOT
     pom
     DataWave
     DataWave is a Java-based ingest and query framework that leverages Apache Accumulo to provide fast, secure access to your data.
diff --git a/warehouse/accumulo-extensions/pom.xml b/warehouse/accumulo-extensions/pom.xml
index 14f445611d2..cd511f43167 100644
--- a/warehouse/accumulo-extensions/pom.xml
+++ b/warehouse/accumulo-extensions/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-accumulo-extensions
     ${project.artifactId}
diff --git a/warehouse/assemble/datawave/pom.xml b/warehouse/assemble/datawave/pom.xml
index fb24a4345cf..43943c48561 100644
--- a/warehouse/assemble/datawave/pom.xml
+++ b/warehouse/assemble/datawave/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         assemble-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     assemble-datawave
     jar
diff --git a/warehouse/assemble/pom.xml b/warehouse/assemble/pom.xml
index c70a132b21c..56a6c6fb389 100644
--- a/warehouse/assemble/pom.xml
+++ b/warehouse/assemble/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     assemble-parent
     pom
diff --git a/warehouse/assemble/webservice/pom.xml b/warehouse/assemble/webservice/pom.xml
index 5958c14f08b..7ed462a5ce0 100644
--- a/warehouse/assemble/webservice/pom.xml
+++ b/warehouse/assemble/webservice/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         assemble-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     assemble-webservice
     ${project.artifactId}
diff --git a/warehouse/common/pom.xml b/warehouse/common/pom.xml
index 8991e037ffb..4d9434c9499 100644
--- a/warehouse/common/pom.xml
+++ b/warehouse/common/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-common
     ${project.artifactId}
diff --git a/warehouse/core/pom.xml b/warehouse/core/pom.xml
index 7a0084b2a9b..dbb1d396a72 100644
--- a/warehouse/core/pom.xml
+++ b/warehouse/core/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-core
     jar
diff --git a/warehouse/data-dictionary-core/pom.xml b/warehouse/data-dictionary-core/pom.xml
index 4cf6a46adb9..6035b26df8e 100644
--- a/warehouse/data-dictionary-core/pom.xml
+++ b/warehouse/data-dictionary-core/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-data-dictionary-core
     jar
diff --git a/warehouse/edge-dictionary-core/pom.xml b/warehouse/edge-dictionary-core/pom.xml
index dba8bac91a3..cb6d4cbc48b 100644
--- a/warehouse/edge-dictionary-core/pom.xml
+++ b/warehouse/edge-dictionary-core/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-edge-dictionary-core
     jar
diff --git a/warehouse/edge-model-configuration-core/pom.xml b/warehouse/edge-model-configuration-core/pom.xml
index c3aa3a868e8..9fd6f52750a 100644
--- a/warehouse/edge-model-configuration-core/pom.xml
+++ b/warehouse/edge-model-configuration-core/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-edge-model-configuration-core
     jar
diff --git a/warehouse/index-stats/pom.xml b/warehouse/index-stats/pom.xml
index 9d1af44b5fc..51f5a0b7a20 100644
--- a/warehouse/index-stats/pom.xml
+++ b/warehouse/index-stats/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-index-stats
     jar
diff --git a/warehouse/ingest-configuration/pom.xml b/warehouse/ingest-configuration/pom.xml
index 6298cf821b4..cd743d19d13 100644
--- a/warehouse/ingest-configuration/pom.xml
+++ b/warehouse/ingest-configuration/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ingest-configuration
     
diff --git a/warehouse/ingest-core/pom.xml b/warehouse/ingest-core/pom.xml
index ce13224d0e0..a1f25c8ac66 100644
--- a/warehouse/ingest-core/pom.xml
+++ b/warehouse/ingest-core/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ingest-core
     jar
diff --git a/warehouse/ingest-csv/pom.xml b/warehouse/ingest-csv/pom.xml
index 35b1c2df03c..0c6a3b49972 100644
--- a/warehouse/ingest-csv/pom.xml
+++ b/warehouse/ingest-csv/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ingest-csv
     jar
diff --git a/warehouse/ingest-json/pom.xml b/warehouse/ingest-json/pom.xml
index 5d7c4f1a8a7..db1dac3b30a 100644
--- a/warehouse/ingest-json/pom.xml
+++ b/warehouse/ingest-json/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ingest-json
     jar
diff --git a/warehouse/ingest-nyctlc/pom.xml b/warehouse/ingest-nyctlc/pom.xml
index fa661596a31..d7f80d31a5f 100644
--- a/warehouse/ingest-nyctlc/pom.xml
+++ b/warehouse/ingest-nyctlc/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ingest-nyctlc
     jar
diff --git a/warehouse/ingest-scripts/pom.xml b/warehouse/ingest-scripts/pom.xml
index 0824161dbe5..ed2656cd760 100644
--- a/warehouse/ingest-scripts/pom.xml
+++ b/warehouse/ingest-scripts/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ingest-scripts
     ${project.artifactId}
diff --git a/warehouse/ingest-ssdeep/pom.xml b/warehouse/ingest-ssdeep/pom.xml
index c4774015a00..bfd7ab7dfc9 100644
--- a/warehouse/ingest-ssdeep/pom.xml
+++ b/warehouse/ingest-ssdeep/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ingest-ssdeep
     
diff --git a/warehouse/ingest-wikipedia/pom.xml b/warehouse/ingest-wikipedia/pom.xml
index fea39cd417e..0c40d3ede08 100644
--- a/warehouse/ingest-wikipedia/pom.xml
+++ b/warehouse/ingest-wikipedia/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ingest-wikipedia
     jar
diff --git a/warehouse/metrics-core/pom.xml b/warehouse/metrics-core/pom.xml
index 0257b2bb37e..2b4275125e0 100644
--- a/warehouse/metrics-core/pom.xml
+++ b/warehouse/metrics-core/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-metrics-core
     jar
diff --git a/warehouse/ops-tools/config-compare/pom.xml b/warehouse/ops-tools/config-compare/pom.xml
index dde7abee524..8f2f5d31bb2 100644
--- a/warehouse/ops-tools/config-compare/pom.xml
+++ b/warehouse/ops-tools/config-compare/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-ops-tools-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ops-tools-config-compare
     
diff --git a/warehouse/ops-tools/index-validation/pom.xml b/warehouse/ops-tools/index-validation/pom.xml
index ff10e398236..976eb1e60ee 100644
--- a/warehouse/ops-tools/index-validation/pom.xml
+++ b/warehouse/ops-tools/index-validation/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-ops-tools-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ops-tools-index-validation
     jar
diff --git a/warehouse/ops-tools/pom.xml b/warehouse/ops-tools/pom.xml
index db58cb07060..7681395de39 100644
--- a/warehouse/ops-tools/pom.xml
+++ b/warehouse/ops-tools/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ops-tools-parent
     pom
diff --git a/warehouse/pom.xml b/warehouse/pom.xml
index f222279fef3..3f450b6bf11 100644
--- a/warehouse/pom.xml
+++ b/warehouse/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-warehouse-parent
     pom
diff --git a/warehouse/query-core/pom.xml b/warehouse/query-core/pom.xml
index 42a8e1d2c99..1e4d6599dd3 100644
--- a/warehouse/query-core/pom.xml
+++ b/warehouse/query-core/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-query-core
     jar
diff --git a/warehouse/regression-testing/pom.xml b/warehouse/regression-testing/pom.xml
index 125f4ac8cd9..329fa8e2480 100644
--- a/warehouse/regression-testing/pom.xml
+++ b/warehouse/regression-testing/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-regression-testing
     ${project.artifactId}
diff --git a/warehouse/ssdeep-common/pom.xml b/warehouse/ssdeep-common/pom.xml
index 1d355b64dec..9f42ecb8190 100644
--- a/warehouse/ssdeep-common/pom.xml
+++ b/warehouse/ssdeep-common/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-warehouse-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ssdeep-common
     
diff --git a/web-services/accumulo/pom.xml b/web-services/accumulo/pom.xml
index 5ce18817d58..c574002f358 100644
--- a/web-services/accumulo/pom.xml
+++ b/web-services/accumulo/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-accumulo
     ejb
diff --git a/web-services/atom/pom.xml b/web-services/atom/pom.xml
index 4a925cf581b..20a9dd33e81 100644
--- a/web-services/atom/pom.xml
+++ b/web-services/atom/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-atom
     ejb
diff --git a/web-services/cached-results/pom.xml b/web-services/cached-results/pom.xml
index de70d9d80cb..6720219643a 100644
--- a/web-services/cached-results/pom.xml
+++ b/web-services/cached-results/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-cached-results
     ejb
diff --git a/web-services/client/pom.xml b/web-services/client/pom.xml
index 587d5f818f7..f90eab53abb 100644
--- a/web-services/client/pom.xml
+++ b/web-services/client/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-client
     jar
diff --git a/web-services/common-util/pom.xml b/web-services/common-util/pom.xml
index c75e95b7f1d..f1d23b54fe0 100644
--- a/web-services/common-util/pom.xml
+++ b/web-services/common-util/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-common-util
     jar
diff --git a/web-services/common/pom.xml b/web-services/common/pom.xml
index 5df8eaa097d..61f1c87b0ef 100644
--- a/web-services/common/pom.xml
+++ b/web-services/common/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-common
     ejb
diff --git a/web-services/deploy/application/pom.xml b/web-services/deploy/application/pom.xml
index cfd7c0a4b8a..9681c0b4570 100644
--- a/web-services/deploy/application/pom.xml
+++ b/web-services/deploy/application/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-deploy-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-deploy-application
     ear
diff --git a/web-services/deploy/configuration/pom.xml b/web-services/deploy/configuration/pom.xml
index 83357cc8af5..e10710ab482 100644
--- a/web-services/deploy/configuration/pom.xml
+++ b/web-services/deploy/configuration/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-deploy-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-deploy-configuration
     jar
diff --git a/web-services/deploy/docs/pom.xml b/web-services/deploy/docs/pom.xml
index bf3beaf195f..c17cd77a160 100644
--- a/web-services/deploy/docs/pom.xml
+++ b/web-services/deploy/docs/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-deploy-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-deploy-docs
     war
diff --git a/web-services/deploy/pom.xml b/web-services/deploy/pom.xml
index dc67c937f8e..eab38e9cca2 100644
--- a/web-services/deploy/pom.xml
+++ b/web-services/deploy/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     gov.nsa.datawave.webservices
     datawave-ws-deploy-parent
diff --git a/web-services/deploy/spring-framework-integration/pom.xml b/web-services/deploy/spring-framework-integration/pom.xml
index 52459f2a6e4..1e38b66f8b7 100644
--- a/web-services/deploy/spring-framework-integration/pom.xml
+++ b/web-services/deploy/spring-framework-integration/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-deploy-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     spring-framework-integration
     ${project.artifactId}
diff --git a/web-services/dictionary/pom.xml b/web-services/dictionary/pom.xml
index 20dde3cc373..9d029942255 100644
--- a/web-services/dictionary/pom.xml
+++ b/web-services/dictionary/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-dictionary
     ejb
diff --git a/web-services/examples/client-login/pom.xml b/web-services/examples/client-login/pom.xml
index 7191338ed62..8f4f779aa5b 100644
--- a/web-services/examples/client-login/pom.xml
+++ b/web-services/examples/client-login/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-examples-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-examples-client-login
     ejb
diff --git a/web-services/examples/http-client/pom.xml b/web-services/examples/http-client/pom.xml
index bd9c22ef3de..e3b04de4c80 100644
--- a/web-services/examples/http-client/pom.xml
+++ b/web-services/examples/http-client/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-examples-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-examples-http-client
     jar
diff --git a/web-services/examples/jms-client/pom.xml b/web-services/examples/jms-client/pom.xml
index c94d0813100..3a80d23ba88 100644
--- a/web-services/examples/jms-client/pom.xml
+++ b/web-services/examples/jms-client/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-examples-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-examples-jms-client
     jar
diff --git a/web-services/examples/pom.xml b/web-services/examples/pom.xml
index c8108533252..c485de7a31b 100644
--- a/web-services/examples/pom.xml
+++ b/web-services/examples/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-examples-parent
     pom
diff --git a/web-services/examples/query-war/pom.xml b/web-services/examples/query-war/pom.xml
index 3cb70f571a1..a414bf95751 100644
--- a/web-services/examples/query-war/pom.xml
+++ b/web-services/examples/query-war/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-examples-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-examples-query-war
     war
diff --git a/web-services/map-reduce-embedded/pom.xml b/web-services/map-reduce-embedded/pom.xml
index edd123de926..afc14da85a3 100644
--- a/web-services/map-reduce-embedded/pom.xml
+++ b/web-services/map-reduce-embedded/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-map-reduce-embedded
     jar
diff --git a/web-services/map-reduce-status/pom.xml b/web-services/map-reduce-status/pom.xml
index bfdea7622ab..f1dbe970dfa 100644
--- a/web-services/map-reduce-status/pom.xml
+++ b/web-services/map-reduce-status/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-map-reduce-status
     ejb
diff --git a/web-services/map-reduce/pom.xml b/web-services/map-reduce/pom.xml
index e61c1bd1304..7ddb8e89a61 100644
--- a/web-services/map-reduce/pom.xml
+++ b/web-services/map-reduce/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-map-reduce
     ejb
diff --git a/web-services/model/pom.xml b/web-services/model/pom.xml
index d136098c61c..10ece229cf0 100644
--- a/web-services/model/pom.xml
+++ b/web-services/model/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-model
     ejb
diff --git a/web-services/modification/pom.xml b/web-services/modification/pom.xml
index 138d81fe4ae..9290ac43165 100644
--- a/web-services/modification/pom.xml
+++ b/web-services/modification/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-modification
     ejb
diff --git a/web-services/pom.xml b/web-services/pom.xml
index 211e7d25df7..7a85e77f218 100644
--- a/web-services/pom.xml
+++ b/web-services/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave
         datawave-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     gov.nsa.datawave.webservices
     datawave-ws-parent
diff --git a/web-services/query-websocket/pom.xml b/web-services/query-websocket/pom.xml
index ace51e49e85..664981226ce 100644
--- a/web-services/query-websocket/pom.xml
+++ b/web-services/query-websocket/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-query-websocket
     war
diff --git a/web-services/query/pom.xml b/web-services/query/pom.xml
index b91638faf34..a992a1b79e7 100644
--- a/web-services/query/pom.xml
+++ b/web-services/query/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-query
     ejb
diff --git a/web-services/rest-api/pom.xml b/web-services/rest-api/pom.xml
index fa63ee40984..ed7e358ff22 100644
--- a/web-services/rest-api/pom.xml
+++ b/web-services/rest-api/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-rest-api
     war
diff --git a/web-services/security/pom.xml b/web-services/security/pom.xml
index b739d4aa839..de2f0948a41 100644
--- a/web-services/security/pom.xml
+++ b/web-services/security/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-security
     ejb
diff --git a/web-services/web-root/pom.xml b/web-services/web-root/pom.xml
index ee0c71de60d..a378ad72426 100644
--- a/web-services/web-root/pom.xml
+++ b/web-services/web-root/pom.xml
@@ -4,7 +4,7 @@
     
         gov.nsa.datawave.webservices
         datawave-ws-parent
-        6.7.0-SNAPSHOT
+        6.8.0-SNAPSHOT
     
     datawave-ws-web-root
     war

From cbbfa3886967be4d15f9aaed5b16fa49536e6c35 Mon Sep 17 00:00:00 2001
From: Ivan Bella 
Date: Wed, 21 Feb 2024 12:36:30 -0500
Subject: [PATCH 46/50] Make the configcheck module optional as it is a
 separate utility not (#2283)

required in the main datawave builds
---
 microservices/configcheck/pom.xml |  2 +-
 microservices/pom.xml             | 12 +++++++++++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/microservices/configcheck/pom.xml b/microservices/configcheck/pom.xml
index 83cb77aca80..903efa54929 100644
--- a/microservices/configcheck/pom.xml
+++ b/microservices/configcheck/pom.xml
@@ -3,7 +3,7 @@
     4.0.0
     gov.nsa.datawave.microservice
     datawave-microservice-configcheck
-    6.7.0-SNAPSHOT
+    1.0.0-SNAPSHOT
     ${project.artifactId}
     
         
diff --git a/microservices/pom.xml b/microservices/pom.xml
index 6145cee9026..0213a4d1da7 100644
--- a/microservices/pom.xml
+++ b/microservices/pom.xml
@@ -11,7 +11,6 @@
     pom
     ${project.artifactId}
     
-        configcheck
         services
         starters
     
@@ -38,5 +37,16 @@
                 microservice-service-parent
             
         
+        
+            configcheck
+            
+                
+                    configcheck
+                
+            
+            
+                configcheck
+            
+        
     
 

From 81a66c86cfd679fdba9cbcaaa78bf94cda458194 Mon Sep 17 00:00:00 2001
From: Reggie Hayes 
Date: Wed, 21 Feb 2024 13:14:29 -0500
Subject: [PATCH 47/50] Fields with a state of DROP on normalization failure
 are not written to the index but are still preserved in the event.

---
 .../ingest/data/config/ingest/BaseIngestHelper.java    |  9 ++++++++-
 .../handler/shard/ShardedDataTypeHandler.java          | 10 ++++++----
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java
index 6d7d27cd24e..20be8382ee3 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java
@@ -980,10 +980,17 @@ protected void applyNormalizationAndAddToResults(Multimap createForwardIndices(IngestHelperInterfa
 
         String fieldName = value.getIndexedFieldName();
         String fieldValue = value.getIndexedFieldValue();
-        // produce field index.
-        values.putAll(createShardFieldIndexColumn(event, fieldName, fieldValue, visibility, maskedVisibility, maskedFieldHelper, shardId,
+	if (fieldValue != null) {
+        	// produce field index.
+        	values.putAll(createShardFieldIndexColumn(event, fieldName, fieldValue, visibility, maskedVisibility, maskedFieldHelper, shardId,
                         createBloomFilter(event, fields, reporter)));
 
-        // produce index column
-        values.putAll(createTermIndexColumn(event, fieldName, fieldValue, visibility, maskedVisibility, maskedFieldHelper, shardId,
+        	// produce index column
+        	values.putAll(createTermIndexColumn(event, fieldName, fieldValue, visibility, maskedVisibility, maskedFieldHelper, shardId,
                         this.getShardIndexTableName(), indexValue, Direction.FORWARD));
+	}
 
         return values;
     }

From abf29bd006759ad04e7bb3ad774af6599444a7b3 Mon Sep 17 00:00:00 2001
From: Reggie Hayes 
Date: Thu, 22 Feb 2024 12:22:21 +0000
Subject: [PATCH 48/50] Update code format

---
 .../data/config/ingest/BaseIngestHelper.java   |  6 +++---
 .../handler/shard/ShardedDataTypeHandler.java  | 18 +++++++++---------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java
index 20be8382ee3..c2f4c80fd12 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/data/config/ingest/BaseIngestHelper.java
@@ -982,15 +982,15 @@ protected void applyNormalizationAndAddToResults(Multimap createForwardIndices(IngestHelperInterfa
 
         String fieldName = value.getIndexedFieldName();
         String fieldValue = value.getIndexedFieldValue();
-	if (fieldValue != null) {
-        	// produce field index.
-        	values.putAll(createShardFieldIndexColumn(event, fieldName, fieldValue, visibility, maskedVisibility, maskedFieldHelper, shardId,
-                        createBloomFilter(event, fields, reporter)));
-
-        	// produce index column
-        	values.putAll(createTermIndexColumn(event, fieldName, fieldValue, visibility, maskedVisibility, maskedFieldHelper, shardId,
-                        this.getShardIndexTableName(), indexValue, Direction.FORWARD));
-	}
+        if (fieldValue != null) {
+            // produce field index.
+            values.putAll(createShardFieldIndexColumn(event, fieldName, fieldValue, visibility, maskedVisibility, maskedFieldHelper, shardId,
+                            createBloomFilter(event, fields, reporter)));
+
+            // produce index column
+            values.putAll(createTermIndexColumn(event, fieldName, fieldValue, visibility, maskedVisibility, maskedFieldHelper, shardId,
+                            this.getShardIndexTableName(), indexValue, Direction.FORWARD));
+        }
 
         return values;
     }

From caa0632ed63450e96154858b7cb8b0a02f27b0ee Mon Sep 17 00:00:00 2001
From: Reggie Hayes 
Date: Thu, 22 Feb 2024 14:33:31 -0500
Subject: [PATCH 49/50] Handle case with null fieldValue.

---
 .github/workflows/settings.xml                |  4 +-
 core/base-rest-responses                      |  2 +-
 core/in-memory-accumulo                       |  2 +-
 core/metrics-reporter                         |  2 +-
 core/utils/accumulo-utils                     |  2 +-
 core/utils/common-utils                       |  2 +-
 core/utils/metadata-utils                     |  2 +-
 core/utils/type-utils                         |  2 +-
 microservices/microservice-parent             |  2 +-
 microservices/microservice-service-parent     |  2 +-
 microservices/services/accumulo               |  2 +-
 microservices/services/audit                  |  2 +-
 microservices/services/authorization          |  2 +-
 microservices/services/config                 |  2 +-
 microservices/services/dictionary             |  2 +-
 microservices/services/hazelcast              |  2 +-
 microservices/services/query-metric           |  2 +-
 microservices/starters/audit                  |  2 +-
 microservices/starters/cache                  |  2 +-
 microservices/starters/datawave               |  2 +-
 microservices/starters/metadata               |  2 +-
 microservices/starters/query-metric           |  2 +-
 .../handler/shard/ShardedDataTypeHandler.java | 90 +++++++++----------
 23 files changed, 68 insertions(+), 68 deletions(-)

diff --git a/.github/workflows/settings.xml b/.github/workflows/settings.xml
index d8be2eb498d..1bda8ae2469 100644
--- a/.github/workflows/settings.xml
+++ b/.github/workflows/settings.xml
@@ -16,8 +16,8 @@
    
       
          github-datawave
-         ${env.USER_NAME}
-         ${env.ACCESS_TOKEN}
+         rdhayes68
+         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
       
    
 
diff --git a/core/base-rest-responses b/core/base-rest-responses
index 2a8073ce2c9..f09536681ee 160000
--- a/core/base-rest-responses
+++ b/core/base-rest-responses
@@ -1 +1 @@
-Subproject commit 2a8073ce2c9a53b5784bc1ff1d6ba28e4e76c881
+Subproject commit f09536681eeefe27b0f3e5840966656b04a4e476
diff --git a/core/in-memory-accumulo b/core/in-memory-accumulo
index f6a0c10764a..362d6dccd62 160000
--- a/core/in-memory-accumulo
+++ b/core/in-memory-accumulo
@@ -1 +1 @@
-Subproject commit f6a0c10764a7f176e8d7c8e9737854762f825104
+Subproject commit 362d6dccd62d40235ae2cac45f3225ecb0c2a65f
diff --git a/core/metrics-reporter b/core/metrics-reporter
index 30341b1c530..8089d53feb5 160000
--- a/core/metrics-reporter
+++ b/core/metrics-reporter
@@ -1 +1 @@
-Subproject commit 30341b1c530ac256ad898f34c915e1ea542371f1
+Subproject commit 8089d53feb585c4a77a07f26d2a449afa28dca1c
diff --git a/core/utils/accumulo-utils b/core/utils/accumulo-utils
index 5a087bc5bb0..7673278ecf3 160000
--- a/core/utils/accumulo-utils
+++ b/core/utils/accumulo-utils
@@ -1 +1 @@
-Subproject commit 5a087bc5bb08117c44ff70cacc637ad76dfa9528
+Subproject commit 7673278ecf39ed140a2c0b90b06de87625533e6e
diff --git a/core/utils/common-utils b/core/utils/common-utils
index 048fe1228b8..9e73b23a69d 160000
--- a/core/utils/common-utils
+++ b/core/utils/common-utils
@@ -1 +1 @@
-Subproject commit 048fe1228b82e3c43b336ee5458562e1f0d02775
+Subproject commit 9e73b23a69d806185d53b9936a1cd5e3fc388a67
diff --git a/core/utils/metadata-utils b/core/utils/metadata-utils
index d56dbeb3ca9..b2d97900b43 160000
--- a/core/utils/metadata-utils
+++ b/core/utils/metadata-utils
@@ -1 +1 @@
-Subproject commit d56dbeb3ca921450991f44ed93346d55ba8dc3a0
+Subproject commit b2d97900b4341f06e5e4dd631e57b23b6318b485
diff --git a/core/utils/type-utils b/core/utils/type-utils
index f0d976adbe8..958c15f54a6 160000
--- a/core/utils/type-utils
+++ b/core/utils/type-utils
@@ -1 +1 @@
-Subproject commit f0d976adbe8011b48a8887ac1dc2e675dc0d3e1a
+Subproject commit 958c15f54a6b5dbc627e3b2599c7e43f6446276d
diff --git a/microservices/microservice-parent b/microservices/microservice-parent
index a2d4a828059..11435518737 160000
--- a/microservices/microservice-parent
+++ b/microservices/microservice-parent
@@ -1 +1 @@
-Subproject commit a2d4a82805997ca1524bea483eab0a1bc21f8f8f
+Subproject commit 114355187371a069b66cc3d56b7f44eb7cb55ca2
diff --git a/microservices/microservice-service-parent b/microservices/microservice-service-parent
index 84f57af2959..f0644113ab9 160000
--- a/microservices/microservice-service-parent
+++ b/microservices/microservice-service-parent
@@ -1 +1 @@
-Subproject commit 84f57af29590f42072b5c7a09002c950e66cbdb7
+Subproject commit f0644113ab913c77e5b19b1a2210e28d841390d6
diff --git a/microservices/services/accumulo b/microservices/services/accumulo
index 090b5c72129..e8f790d304a 160000
--- a/microservices/services/accumulo
+++ b/microservices/services/accumulo
@@ -1 +1 @@
-Subproject commit 090b5c72129caee39234d007dc999afe99c9ff54
+Subproject commit e8f790d304ac240680a11255924c01837dc0220b
diff --git a/microservices/services/audit b/microservices/services/audit
index 4a158581eb4..ea0f0cefd99 160000
--- a/microservices/services/audit
+++ b/microservices/services/audit
@@ -1 +1 @@
-Subproject commit 4a158581eb477b6fa14740d89920e98410e6ee3f
+Subproject commit ea0f0cefd9974cc7ec6efe359d0005196b3eb797
diff --git a/microservices/services/authorization b/microservices/services/authorization
index 7c54c487c84..a730d64c58b 160000
--- a/microservices/services/authorization
+++ b/microservices/services/authorization
@@ -1 +1 @@
-Subproject commit 7c54c487c841f7529dedc83dbcdad53e9e1ae8a8
+Subproject commit a730d64c58b0495963a1b0e188cb1d8cb3cb9fa5
diff --git a/microservices/services/config b/microservices/services/config
index f474b0d06a4..5155789b277 160000
--- a/microservices/services/config
+++ b/microservices/services/config
@@ -1 +1 @@
-Subproject commit f474b0d06a434ee235d28e8dcf9a657f9355e92d
+Subproject commit 5155789b2775cbf754d4634cc1a00d8c0a7ea8ad
diff --git a/microservices/services/dictionary b/microservices/services/dictionary
index 728ca486c72..1843b7a44ce 160000
--- a/microservices/services/dictionary
+++ b/microservices/services/dictionary
@@ -1 +1 @@
-Subproject commit 728ca486c7229ff7646c612b12e61418529e8131
+Subproject commit 1843b7a44ceb3321cc26bc691c830be31a29183e
diff --git a/microservices/services/hazelcast b/microservices/services/hazelcast
index ec81bc3bb2e..1dc8d74fea8 160000
--- a/microservices/services/hazelcast
+++ b/microservices/services/hazelcast
@@ -1 +1 @@
-Subproject commit ec81bc3bb2eb987796e65a73db3092eb67989a9e
+Subproject commit 1dc8d74fea842123af714cd097403242ddc61d83
diff --git a/microservices/services/query-metric b/microservices/services/query-metric
index 3545e39dfa0..a6ded67a22a 160000
--- a/microservices/services/query-metric
+++ b/microservices/services/query-metric
@@ -1 +1 @@
-Subproject commit 3545e39dfa05be110245a9609b5d68ac020fcb05
+Subproject commit a6ded67a22acb5edadc0502ec75ab9dc77aaf4f8
diff --git a/microservices/starters/audit b/microservices/starters/audit
index bbf3f4e95b1..93756a38505 160000
--- a/microservices/starters/audit
+++ b/microservices/starters/audit
@@ -1 +1 @@
-Subproject commit bbf3f4e95b139c545b71fae98e743bd09bd28fb0
+Subproject commit 93756a385054b5d21836a812fb9c47f81b3aa44d
diff --git a/microservices/starters/cache b/microservices/starters/cache
index 3a1e500e21f..7c6516ef6c1 160000
--- a/microservices/starters/cache
+++ b/microservices/starters/cache
@@ -1 +1 @@
-Subproject commit 3a1e500e21fb011e9906d82e153808dd4a7d5f0b
+Subproject commit 7c6516ef6c18d69d537a3ff675d6e6843a5cf9bd
diff --git a/microservices/starters/datawave b/microservices/starters/datawave
index 5bd8ef0e83a..a92951fd5fb 160000
--- a/microservices/starters/datawave
+++ b/microservices/starters/datawave
@@ -1 +1 @@
-Subproject commit 5bd8ef0e83ab743d43667850beefe3806120528b
+Subproject commit a92951fd5fb691a9450722b6e2d4ff81a8fbc9c5
diff --git a/microservices/starters/metadata b/microservices/starters/metadata
index 630034dec0f..ed6e3557d43 160000
--- a/microservices/starters/metadata
+++ b/microservices/starters/metadata
@@ -1 +1 @@
-Subproject commit 630034dec0ffa78d1688aa6f6551325b25d14e3d
+Subproject commit ed6e3557d43bfc6cd03989a040df3facce39695a
diff --git a/microservices/starters/query-metric b/microservices/starters/query-metric
index fb1bcd66ae2..17db2621f6f 160000
--- a/microservices/starters/query-metric
+++ b/microservices/starters/query-metric
@@ -1 +1 @@
-Subproject commit fb1bcd66ae21b887e78bb1e2ac702ea4bde3d543
+Subproject commit 17db2621f6f62d347cffdbe7b1ed3195b010a0ca
diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java
index 56bfe17ec04..9c33bfa0635 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java
@@ -557,15 +557,13 @@ protected Multimap createForwardIndices(IngestHelperInterfa
 
         String fieldName = value.getIndexedFieldName();
         String fieldValue = value.getIndexedFieldValue();
-	if (fieldValue != null) {
-        	// produce field index.
-        	values.putAll(createShardFieldIndexColumn(event, fieldName, fieldValue, visibility, maskedVisibility, maskedFieldHelper, shardId,
-                        createBloomFilter(event, fields, reporter)));
+        // produce field index.
+        values.putAll(createShardFieldIndexColumn(event, fieldName, fieldValue, visibility, maskedVisibility, maskedFieldHelper, shardId,
+                    createBloomFilter(event, fields, reporter)));
 
-        	// produce index column
-        	values.putAll(createTermIndexColumn(event, fieldName, fieldValue, visibility, maskedVisibility, maskedFieldHelper, shardId,
-                        this.getShardIndexTableName(), indexValue, Direction.FORWARD));
-	}
+        // produce index column
+        values.putAll(createTermIndexColumn(event, fieldName, fieldValue, visibility, maskedVisibility, maskedFieldHelper, shardId,
+                    this.getShardIndexTableName(), indexValue, Direction.FORWARD));
 
         return values;
     }
@@ -1021,50 +1019,52 @@ protected Multimap createShardFieldIndexColumn(RawRecordCon
 
         Multimap values = HashMultimap.create();
 
-        Text colf = new Text("fi");
-        TextUtil.textAppend(colf, fieldName, replaceMalformedUTF8);
-        Text unmaskedColq = new Text(fieldValue);
-        TextUtil.textAppend(unmaskedColq, event.getDataType().outputName(), replaceMalformedUTF8);
-        TextUtil.textAppend(unmaskedColq, event.getId().toString(), replaceMalformedUTF8);
-
-        if (value == null) {
-            value = NULL_VALUE;
-        }
+        if (StringUtils.isNotEmpty(fieldValue)) {
+            Text colf = new Text("fi");
+            TextUtil.textAppend(colf, fieldName, replaceMalformedUTF8);
+            Text unmaskedColq = new Text(fieldValue);
+            TextUtil.textAppend(unmaskedColq, event.getDataType().outputName(), replaceMalformedUTF8);
+            TextUtil.textAppend(unmaskedColq, event.getId().toString(), replaceMalformedUTF8);
 
-        if (null != maskedFieldHelper && maskedFieldHelper.contains(fieldName)) {
-            if (!StringUtils.isEmpty(fieldValue)) {
-                // Put unmasked colq with original visibility
-                Key k = createKey(shardId, colf, unmaskedColq, visibility, event.getDate(), deleteMode);
-                BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
-                values.put(bKey, value);
+            if (value == null) {
+                value = NULL_VALUE;
             }
 
-            // We need to use the normalized masked values
-            final String normalizedMaskedValue = helper.getNormalizedMaskedValue(fieldName);
-            if (!StringUtils.isEmpty(normalizedMaskedValue)) {
-                Text maskedColq = new Text(normalizedMaskedValue);
-                TextUtil.textAppend(maskedColq, event.getDataType().outputName(), replaceMalformedUTF8);
-                TextUtil.textAppend(maskedColq, event.getId().toString(), replaceMalformedUTF8);
+            if (null != maskedFieldHelper && maskedFieldHelper.contains(fieldName)) {
+                if (!StringUtils.isEmpty(fieldValue)) {
+                    // Put unmasked colq with original visibility
+                    Key k = createKey(shardId, colf, unmaskedColq, visibility, event.getDate(), deleteMode);
+                    BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
+                    values.put(bKey, value);
+                }
 
-                // Put masked colq with masked visibility
-                Key k = createKey(shardId, colf, maskedColq, maskedVisibility, event.getDate(), deleteMode);
+                // We need to use the normalized masked values
+                final String normalizedMaskedValue = helper.getNormalizedMaskedValue(fieldName);
+                if (!StringUtils.isEmpty(normalizedMaskedValue)) {
+                    Text maskedColq = new Text(normalizedMaskedValue);
+                    TextUtil.textAppend(maskedColq, event.getDataType().outputName(), replaceMalformedUTF8);
+                    TextUtil.textAppend(maskedColq, event.getId().toString(), replaceMalformedUTF8);
+
+                    // Put masked colq with masked visibility
+                    Key k = createKey(shardId, colf, maskedColq, maskedVisibility, event.getDate(), deleteMode);
+                    BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
+                    values.put(bKey, value);
+                }
+            } else if (!StringUtils.isEmpty(fieldValue)) {
+                /**
+                 * For values that are not being masked, we use the "unmaskedValue" and the masked visibility e.g. release the value as it was in the event at the
+                 * lower visibility
+                 */
+                byte[] refVisibility = visibility;
+
+                if (null != maskedFieldHelper) {
+                    refVisibility = maskedVisibility;
+                }
+
+                Key k = createKey(shardId, colf, unmaskedColq, refVisibility, event.getDate(), deleteMode);
                 BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
                 values.put(bKey, value);
             }
-        } else if (!StringUtils.isEmpty(fieldValue)) {
-            /**
-             * For values that are not being masked, we use the "unmaskedValue" and the masked visibility e.g. release the value as it was in the event at the
-             * lower visibility
-             */
-            byte[] refVisibility = visibility;
-
-            if (null != maskedFieldHelper) {
-                refVisibility = maskedVisibility;
-            }
-
-            Key k = createKey(shardId, colf, unmaskedColq, refVisibility, event.getDate(), deleteMode);
-            BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
-            values.put(bKey, value);
         }
 
         return values;

From 41ef8f38a6ddaf42980bcfe4bc73e220012e4826 Mon Sep 17 00:00:00 2001
From: Reggie Hayes 
Date: Thu, 22 Feb 2024 15:01:19 -0500
Subject: [PATCH 50/50] Handle case with null fieldValue.

---
 .../handler/shard/ShardedDataTypeHandler.java | 80 ++++++++++---------
 1 file changed, 41 insertions(+), 39 deletions(-)

diff --git a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java
index fc29d4bee9c..39cf336957a 100644
--- a/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java
+++ b/warehouse/ingest-core/src/main/java/datawave/ingest/mapreduce/handler/shard/ShardedDataTypeHandler.java
@@ -559,11 +559,11 @@ protected Multimap createForwardIndices(IngestHelperInterfa
         String fieldValue = value.getIndexedFieldValue();
         // produce field index.
         values.putAll(createShardFieldIndexColumn(event, fieldName, fieldValue, visibility, maskedVisibility, maskedFieldHelper, shardId,
-                    createBloomFilter(event, fields, reporter)));
+                        createBloomFilter(event, fields, reporter)));
 
         // produce index column
         values.putAll(createTermIndexColumn(event, fieldName, fieldValue, visibility, maskedVisibility, maskedFieldHelper, shardId,
-                    this.getShardIndexTableName(), indexValue, Direction.FORWARD));
+                        this.getShardIndexTableName(), indexValue, Direction.FORWARD));
 
         return values;
     }
@@ -1019,50 +1019,52 @@ protected Multimap createShardFieldIndexColumn(RawRecordCon
 
         Multimap values = HashMultimap.create();
 
-        Text colf = new Text("fi");
-        TextUtil.textAppend(colf, fieldName, replaceMalformedUTF8);
-        Text unmaskedColq = new Text(fieldValue);
-        TextUtil.textAppend(unmaskedColq, event.getDataType().outputName(), replaceMalformedUTF8);
-        TextUtil.textAppend(unmaskedColq, event.getId().toString(), replaceMalformedUTF8);
+        if (StringUtils.isNotEmpty(fieldValue)) {
+            Text colf = new Text("fi");
+            TextUtil.textAppend(colf, fieldName, replaceMalformedUTF8);
+            Text unmaskedColq = new Text(fieldValue);
+            TextUtil.textAppend(unmaskedColq, event.getDataType().outputName(), replaceMalformedUTF8);
+            TextUtil.textAppend(unmaskedColq, event.getId().toString(), replaceMalformedUTF8);
 
-        if (value == null) {
-            value = NULL_VALUE;
-        }
-
-        if (null != maskedFieldHelper && maskedFieldHelper.contains(fieldName)) {
-            if (!StringUtils.isEmpty(fieldValue)) {
-                // Put unmasked colq with original visibility
-                Key k = createKey(shardId, colf, unmaskedColq, visibility, event.getDate(), deleteMode);
-                BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
-                values.put(bKey, value);
+            if (value == null) {
+                value = NULL_VALUE;
             }
 
-            // We need to use the normalized masked values
-            final String normalizedMaskedValue = helper.getNormalizedMaskedValue(fieldName);
-            if (!StringUtils.isEmpty(normalizedMaskedValue)) {
-                Text maskedColq = new Text(normalizedMaskedValue);
-                TextUtil.textAppend(maskedColq, event.getDataType().outputName(), replaceMalformedUTF8);
-                TextUtil.textAppend(maskedColq, event.getId().toString(), replaceMalformedUTF8);
+            if (null != maskedFieldHelper && maskedFieldHelper.contains(fieldName)) {
+                if (!StringUtils.isEmpty(fieldValue)) {
+                    // Put unmasked colq with original visibility
+                    Key k = createKey(shardId, colf, unmaskedColq, visibility, event.getDate(), deleteMode);
+                    BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
+                    values.put(bKey, value);
+                }
 
-                // Put masked colq with masked visibility
-                Key k = createKey(shardId, colf, maskedColq, maskedVisibility, event.getDate(), deleteMode);
+                // We need to use the normalized masked values
+                final String normalizedMaskedValue = helper.getNormalizedMaskedValue(fieldName);
+                if (!StringUtils.isEmpty(normalizedMaskedValue)) {
+                    Text maskedColq = new Text(normalizedMaskedValue);
+                    TextUtil.textAppend(maskedColq, event.getDataType().outputName(), replaceMalformedUTF8);
+                    TextUtil.textAppend(maskedColq, event.getId().toString(), replaceMalformedUTF8);
+
+                    // Put masked colq with masked visibility
+                    Key k = createKey(shardId, colf, maskedColq, maskedVisibility, event.getDate(), deleteMode);
+                    BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
+                    values.put(bKey, value);
+                }
+            } else if (!StringUtils.isEmpty(fieldValue)) {
+                /**
+                 * For values that are not being masked, we use the "unmaskedValue" and the masked visibility e.g. release the value as it was in the event at the
+                 * lower visibility
+                 */
+                byte[] refVisibility = visibility;
+
+                if (null != maskedFieldHelper) {
+                    refVisibility = maskedVisibility;
+                }
+
+                Key k = createKey(shardId, colf, unmaskedColq, refVisibility, event.getDate(), deleteMode);
                 BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
                 values.put(bKey, value);
             }
-        } else if (!StringUtils.isEmpty(fieldValue)) {
-            /**
-             * For values that are not being masked, we use the "unmaskedValue" and the masked visibility e.g. release the value as it was in the event at the
-             * lower visibility
-             */
-            byte[] refVisibility = visibility;
-
-            if (null != maskedFieldHelper) {
-                refVisibility = maskedVisibility;
-            }
-
-            Key k = createKey(shardId, colf, unmaskedColq, refVisibility, event.getDate(), deleteMode);
-            BulkIngestKey bKey = new BulkIngestKey(this.getShardTableName(), k);
-            values.put(bKey, value);
         }
 
         return values;

    Role to Auth Mapping table
    Column NameColumn Type