From a8b85582d745cf301228861569cb9ba344dcf2aa Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Tue, 12 Jan 2021 11:32:52 -0500 Subject: [PATCH 001/218] DATAWAVE query microservices initial commit. --- query-microservices/query-api/api/pom.xml | 33 ++++++++ query-microservices/query-api/pom.xml | 33 ++++++++ query-microservices/query-api/service/pom.xml | 82 +++++++++++++++++++ 3 files changed, 148 insertions(+) create mode 100644 query-microservices/query-api/api/pom.xml create mode 100644 query-microservices/query-api/pom.xml create mode 100644 query-microservices/query-api/service/pom.xml diff --git a/query-microservices/query-api/api/pom.xml b/query-microservices/query-api/api/pom.xml new file mode 100644 index 00000000..6038f4c7 --- /dev/null +++ b/query-microservices/query-api/api/pom.xml @@ -0,0 +1,33 @@ + + + 4.0.0 + + gov.nsa.datawave.microservice + datawave-microservice-parent + 1.8 + + + query-api + 1.0-SNAPSHOT + + + + + + + + + + true + + + false + + datawave-github-mvn-repo + https://raw.githubusercontent.com/NationalSecurityAgency/datawave/mvn-repo + + + diff --git a/query-microservices/query-api/pom.xml b/query-microservices/query-api/pom.xml new file mode 100644 index 00000000..a1a949e2 --- /dev/null +++ b/query-microservices/query-api/pom.xml @@ -0,0 +1,33 @@ + + + 4.0.0 + + gov.nsa.datawave.microservice + datawave-microservice-parent + 1.8 + + + query-api-service-parent + 1.0-SNAPSHOT + pom + + api + service + + + + + + true + + + false + + datawave-github-mvn-repo + https://raw.githubusercontent.com/NationalSecurityAgency/datawave/mvn-repo + + + \ No newline at end of file diff --git a/query-microservices/query-api/service/pom.xml b/query-microservices/query-api/service/pom.xml new file mode 100644 index 00000000..0c422ccd --- /dev/null +++ b/query-microservices/query-api/service/pom.xml @@ -0,0 +1,82 @@ + + + 4.0.0 + + gov.nsa.datawave.microservice + datawave-microservice-service-parent + 2.0 + + + query-api-service + 1.0-SNAPSHOT + DATAWAVE Query API Microservice + + 1.0-SNAPSHOT + 1.6.5 + + + + + gov.nsa.datawave.microservice + query-api + ${version.microservice.query-api} + + + gov.nsa.datawave.microservice + spring-boot-starter-datawave + ${version.microservice.starter} + + + + + + gov.nsa.datawave.microservice + query-api + + + gov.nsa.datawave.microservice + spring-boot-starter-datawave + + + + + + + true + + + false + + datawave-github-mvn-repo + https://raw.githubusercontent.com/NationalSecurityAgency/datawave/mvn-repo + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + pl.project13.maven + git-commit-id-plugin + + + + + + docker + + + + com.spotify + docker-maven-plugin + + + + + + \ No newline at end of file From 3c27c6203868458307e497dd9da49683e6361326 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 14 Jan 2021 11:40:49 -0500 Subject: [PATCH 002/218] WIP: Added a readme and laid the groundwork for the microservice. --- .../query-api/service/README.md | 61 ++++++++++++++++++ .../service/src/main/docker/Dockerfile | 11 ++++ .../query/api/QueryController.java | 11 ++++ .../src/main/resources/config/bootstrap.yml | 50 ++++++++++++++ .../service/src/main/resources/log4j2.xml | 46 +++++++++++++ .../src/test/resources/config/application.yml | 47 ++++++++++++++ .../src/test/resources/config/bootstrap.yml | 12 ++++ .../src/test/resources/log4j2-test.xml | 17 +++++ .../service/src/test/resources/ssl/host.p12 | Bin 0 -> 2500 bytes .../service/src/test/resources/ssl/rootCA.p12 | Bin 0 -> 2644 bytes 10 files changed, 255 insertions(+) create mode 100644 query-microservices/query-api/service/README.md create mode 100644 query-microservices/query-api/service/src/main/docker/Dockerfile create mode 100644 query-microservices/query-api/service/src/main/java/datawave/microservice/query/api/QueryController.java create mode 100644 query-microservices/query-api/service/src/main/resources/config/bootstrap.yml create mode 100644 query-microservices/query-api/service/src/main/resources/log4j2.xml create mode 100644 query-microservices/query-api/service/src/test/resources/config/application.yml create mode 100644 query-microservices/query-api/service/src/test/resources/config/bootstrap.yml create mode 100644 query-microservices/query-api/service/src/test/resources/log4j2-test.xml create mode 100644 query-microservices/query-api/service/src/test/resources/ssl/host.p12 create mode 100644 query-microservices/query-api/service/src/test/resources/ssl/rootCA.p12 diff --git a/query-microservices/query-api/service/README.md b/query-microservices/query-api/service/README.md new file mode 100644 index 00000000..b0c4d7cd --- /dev/null +++ b/query-microservices/query-api/service/README.md @@ -0,0 +1,61 @@ +# Query API Service + +[![Apache License][li]][ll] ![Build Status](https://github.com/NationalSecurityAgency/datawave/workflows/Tests/badge.svg) + +The query api service is a DATAWAVE microservice that provides +query capabilities. + +### Query API Context + +*https://host:port/query-api/v1/* + +### User API + +| Method | Operation | Description | Path Param | Request Body | Response Body | +|:-------|:----------|:-----------------------|:-----------|:---------------| +| `GET` | /listQueryLogic | List QueryLogic types that are currently available | N/A | N/A | [QueryLogicResponse] | +| `POST` | /{queryLogic}/define | Define a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| `POST` | /{queryLogic}/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| `POST` | /{queryLogic}/plan | Generate a query plan using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| `POST` | /{queryLogic}/predict | Generate a query prediction using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| `POST` | /{queryLogic}/async/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| `PUT` | /{id}/reset | Resets the specified query | [QueryId] | N/A | [VoidResponse] | +| `POST` | /{queryLogic}/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | +| `POST` | /{queryLogic}/async/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | +| `GET` | /lookupContentUUID/{uuidType}/{uuid} | Returns content associated with the given UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] || [StreamingOutput] | +| `POST` | /lookupContentUUID | Returns content associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] || [StreamingOutput] | +| `GET` | /lookupUUID/{uuidType}/{uuid} | Returns event associated with the given batch of UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] || [StreamingOutput] | +| `POST` | /lookupUUID | Returns event(s) associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] || [StreamingOutput] | +| `GET` | /{id}/plan | Returns the plan for the specified query | [QueryId] | N/A | [GenericResponse] | +| `GET` | /{id}/predictions | Returns the predictions for the specified query | [QueryId] | N/A | [GenericResponse] | +| `GET` | /{id}/async/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | +| `GET` | /{id}/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | +| `PUT` `POST` | /{id}/close | Closes the specified query | [QueryId] | N/A | [VoidResponse] | +| `PUT` `POST` | /{id}/adminClose | Closes the specified query | [QueryId] | N/A | [VoidResponse] | +| `PUT` `POST` | /{id}/cancel | Cancels the specified query | [QueryId] | N/A | [VoidResponse] | +| `PUT` `POST` | /{id}/adminCancel | Cancels the specified query | [QueryId] | N/A | [VoidResponse] | +| `GET` | /listAll | Returns a list of queries associated with the current user | N/A | N/A | [QueryImplListResponse] | +| `GET` | /{id} | Returns query info for the specified query | [QueryId] | N/A | [QueryImplListResponse] | +| `GET` | /list | Returns a list of queries associated with the current user with given name | N/A | [Name] | [QueryImplListResponse] | +| `DELETE` | /{id}/remove | Remove (delete) the specified query | [QueryId] | N/A | [VoidResponse] | +| `POST` | /{id}/duplicate | Duplicates the specified query | [QueryId] | [QueryParameters] | [GenericResponse] | +| `PUT` `POST` | /{id}/update | Updates the specified query | [QueryId] | [QueryParameters] | [GenericResponse] | +| `GET` | /{id}/listAll | Returns a list of queries associated with the specified user | [UserId] | N/A | [QueryImplListResponse] | +| `POST` | /purgeQueryCache | Purges the cache of query objects | N/A | N/A | [VoidResponse] | +| `GET` | /enableTracing | Enables tracing for queries which match the given criteria | N/A | [QueryRegex], [User] | [VoidResponse] | +| `GET` | /disableTracing | Disables tracing for queries which match the given criteria | N/A | [QueryRegex], [User] | [VoidResponse] | +| `GET` | /disableAllTracing | Disables tracing for all queries | N/A | N/A | [VoidResponse] | +| `POST` | /{logicName}/execute | Create a query using the specified query logic and params, and stream the results | [QueryLogicName] | [QueryParameters] | [StreamingOutput] | +| `POST` | /{logicName}/async/execute | Create a query using the specified query logic and params, and stream the results | [QueryLogicName] | [QueryParameters] | [StreamingOutput] | + +--- + +### Getting Started + +TBD + +[getting-started]:https://github.com/NationalSecurityAgency/datawave-microservices-root/blob/master/README.md#getting-started +[pki-dir]:https://github.com/NationalSecurityAgency/datawave-spring-boot-starter/blob/master/src/main/resources/pki + +[li]: http://img.shields.io/badge/license-ASL-blue.svg +[ll]: https://www.apache.org/licenses/LICENSE-2.0 \ No newline at end of file diff --git a/query-microservices/query-api/service/src/main/docker/Dockerfile b/query-microservices/query-api/service/src/main/docker/Dockerfile new file mode 100644 index 00000000..49fdae17 --- /dev/null +++ b/query-microservices/query-api/service/src/main/docker/Dockerfile @@ -0,0 +1,11 @@ +FROM azul/zulu-openjdk-alpine:8 + +LABEL version=${project.version} \ + run="docker run ${docker.image.prefix}${project.artifactId}:latest" \ + description="${project.description}" + +ADD ${project.build.finalName}-exec.jar /app.jar +RUN apk add libc6-compat curl + +EXPOSE 8443 8080 +ENTRYPOINT ["java","-jar","app.jar"] \ No newline at end of file diff --git a/query-microservices/query-api/service/src/main/java/datawave/microservice/query/api/QueryController.java b/query-microservices/query-api/service/src/main/java/datawave/microservice/query/api/QueryController.java new file mode 100644 index 00000000..234fba9e --- /dev/null +++ b/query-microservices/query-api/service/src/main/java/datawave/microservice/query/api/QueryController.java @@ -0,0 +1,11 @@ +package datawave.microservice.query.api; + +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping(path = "/v1", produces = MediaType.APPLICATION_JSON_VALUE) +public class QueryController { + +} \ No newline at end of file diff --git a/query-microservices/query-api/service/src/main/resources/config/bootstrap.yml b/query-microservices/query-api/service/src/main/resources/config/bootstrap.yml new file mode 100644 index 00000000..9b33de46 --- /dev/null +++ b/query-microservices/query-api/service/src/main/resources/config/bootstrap.yml @@ -0,0 +1,50 @@ +spring: + application: + name: query-api + cloud: + config: + # Disable consul-first config by default. We'll turn it back on in the consul profile if that profile is enabled. + discovery: + enabled: false + # Always fail fast so we can retry if the config server is not up yet + failFast: true + # Give the config server time to start up if it hasn't already + retry: + max-attempts: 60 + uri: '${CONFIG_SERVER_URL:http://configuration:8888/configserver}' +--- + +# For the dev profile, check localhost for the config server by default +spring: + profiles: 'dev' + cloud: + config: + uri: '${CONFIG_SERVER_URL:http://localhost:8888/configserver}' + +--- + +spring: + profiles: 'consul' + cloud: + config: + # Use Consul to locate the configuration server and bootstrap app config. + discovery: + enabled: true + # Give the config server a long time to come up and register itself in Consul + retry: + max-attempts: 120 + # Allow the default Consul host to be overridden via an environment variable + consul: + host: ${CONSUL_HOST:localhost} + +--- + +# For the "No Messaging" profile, we need to disable the AMQP bus, our custom RabbitMQ discovery, and the RabbitMQ health indicator. +spring: + profiles: 'nomessaging' + cloud: + bus: + enabled: false + rabbitmq: + discovery: + enabled: false \ No newline at end of file diff --git a/query-microservices/query-api/service/src/main/resources/log4j2.xml b/query-microservices/query-api/service/src/main/resources/log4j2.xml new file mode 100644 index 00000000..2f5ddd6c --- /dev/null +++ b/query-microservices/query-api/service/src/main/resources/log4j2.xml @@ -0,0 +1,46 @@ +Configuration: + status: warn + monitorInterval: 60 + + Properties: + Property: + - name: logDir + value: "logs/" + - name: PID + value: "????" + - name: LOG_PATTERN + value: "%clr{%d{yyyy-MM-dd HH:mm:ss.SSS}}{faint} %clr{%5p} %clr{${sys:PID}}{magenta} %clr{---}{faint} %clr{[%15.15t]}{faint} %clr{%-40.40c{1.}}{cyan} %clr{:}{faint} %m%n%wEx" + + Appenders: + Console: + name: Console + target: SYSTEM_OUT + follow: true + PatternLayout: + pattern: "${LOG_PATTERN}" + + RollingFile: + - name: File + fileName: "${sys:logDir}/query-api-service.log" + filePattern: "${sys:logDir}/query-api-service.log.%d{yyyy-MM-dd}-%i.gz" + append: true + bufferedIO: true + bufferSize: 8192 + Policies: + TimeBasedTriggeringPolicy: + interval: 1 + SizeBasedTriggeringPolicy: + size: 250MB + DefaultRolloverStrategy: + max: 10 + PatternLayout: + pattern: "${LOG_PATTERN}" + + Loggers: + Root: + level: info + AppenderRef: + - ref: Console + level: info + - ref: File + level: trace \ No newline at end of file diff --git a/query-microservices/query-api/service/src/test/resources/config/application.yml b/query-microservices/query-api/service/src/test/resources/config/application.yml new file mode 100644 index 00000000..d99c2e19 --- /dev/null +++ b/query-microservices/query-api/service/src/test/resources/config/application.yml @@ -0,0 +1,47 @@ +spring: + + application: + name: query-api-test + + autoconfigure: + exclude: org.springframework.boot.autoconfigure.web.servlet.error.ErrorMvcAutoConfiguration + + security: + user.password: passwordNotUsed + datawave: + jwt.ttl: 3600 + issuers-required: true + allowed-callers: + - "cn=test.testcorp.com, ou=microservices, ou=development, o=testcorp, c=us" + +hazelcast.client.enabled: false + +server: + port: 0 + non-secure-port: 0 + servlet.context-path: /query-api + ssl: + client-auth: NEED + trust-store: classpath:ssl/rootCA.p12 + trust-store-type: PKCS12 + trust-store-password: LetMeIn + key-store: classpath:ssl/host.p12 + key-store-type: PKCS12 + key-store-password: LetMeIn + outbound-ssl: + key-store: ${server.ssl.key-store} + key-store-password: ${server.ssl.key-store-password} + key-store-type: ${server.ssl.key-store-type} + trust-store: ${server.ssl.trust-store} + trust-store-password: ${server.ssl.trust-store-password} + trust-store-type: ${server.ssl.trust-store-type} + +management: + endpoints: + web: + base-path: "/mgmt" + +logging: + level: + datawave.microservice.query-api: DEBUG + io.undertow.request: FATAL diff --git a/query-microservices/query-api/service/src/test/resources/config/bootstrap.yml b/query-microservices/query-api/service/src/test/resources/config/bootstrap.yml new file mode 100644 index 00000000..2eeba2b0 --- /dev/null +++ b/query-microservices/query-api/service/src/test/resources/config/bootstrap.yml @@ -0,0 +1,12 @@ +spring: + cloud: + bus: + enabled: false + consul: + enabled: false + config: + enabled: false + discovery: + enabled: false + main: + banner-mode: "OFF" \ No newline at end of file diff --git a/query-microservices/query-api/service/src/test/resources/log4j2-test.xml b/query-microservices/query-api/service/src/test/resources/log4j2-test.xml new file mode 100644 index 00000000..0600e42c --- /dev/null +++ b/query-microservices/query-api/service/src/test/resources/log4j2-test.xml @@ -0,0 +1,17 @@ + + + + ???? + %clr{%d{yyyy-MM-dd HH:mm:ss.SSS}}{faint} %clr{%5p} %clr{${sys:PID}}{magenta} %clr{---}{faint} %clr{[%15.15t]}{faint} %clr{%-40.40c{1.}}{cyan} %clr{:}{faint} %m%n%wEx + + + + + + + + + + + + \ No newline at end of file diff --git a/query-microservices/query-api/service/src/test/resources/ssl/host.p12 b/query-microservices/query-api/service/src/test/resources/ssl/host.p12 new file mode 100644 index 0000000000000000000000000000000000000000..5c4a2d66175ca22bfd8d12634c885cd3489d6b8b GIT binary patch literal 2500 zcmY+Ec{mh`8pdbF491>jn4G~uifl8MqcB+_*_SjDlED}x+n|tTY(@4VyP2{Rl_jZc zA^RGQWvp2v+mW5C=ehTud;fT!?|a_&d%pkPABqHR009|LBxnR9^isr)h#hty6OclJ z`hrPNuT$F%MPfMlCt{?K7}ig%6%Y_`I_CaKfG8v*%fCOc0U1#oU}n62Qan~2i3EX| z0l_4Ong;A(35RNJavkrqI3=AXRH*`tPYT-K#C!M;g^aYH75{ zfdj}_q$Wf|#Qix15_~)Jfll}?n`C5Jyz6V8KbbvIof^4m&+%Y@o9S7WODov+ zZfyC$W&ws?b3XSlzvuyAiR(vutGn;xJHrHL$fRq2vs4ZTt^T!w_^6O|x^`k@Mw^WD zdaM!SIimSDNq+?jt$-b!$NK`JlGt*eby%%`(-?{4ntWN?c#gelJstv_NY(Nj7_$pJ zYb^k8-u=-{%rrk0p+nC#FOKcKX)$~2ODC#Kxtc= zV&;?0kLT9M(=sJfXoh+$`uarqurjs`v_zn6w&_qklY5qw?l}#6B_o50MQHJ`vOc%> zjImoS`{P$zQ9?e0*o7+w-$AEEhYful)=1-3YVT4^&!1GwDVMLTI%|^isD5vj`3+tR zvjP;Ye?6R4JN{8_G9Ffe+y8uU&ohX!e*0_NtoM2QFla8>na*A?J&-(RW~OJ$LsX^N z&+V<`_t!)#>+UP7C8f*W*Lkn9XeQ`)P`ql zJGX0I9zini2oHDU&m+1-NnFv5OKPHeIm+Qh3YE$)$o-P!_i)PhpN@W03TXWC+q#~hiMB%fO+iS&od~Os7kwDU#|~G1EDt()H3L z^>W4BnIZ-qTUy8x0=IHukzqDZ!;n|~9S*FZUDghu8J20|5QC3PN}lEWIycgn1=mWY&{tfkwa1h1~>Smezr7Nk&Cfr}iKIG;)XHAcOQylwKEcM&f(D|~qzVQjt zJ^QxT4=dj#YWGbpAvGWwogzWSbsW4Dvk2}7&cGIYkq*zJmcsmkg)=4J=lA}=h%w{} ztAufD$r%;%>OY;JSNeWdY{cZCIBgm_Z$HoO?0J(cc;F)tekJDHjMFcY-ja3RTH4eI z=wrf@L)Vr$L%nuW8Qf1|?XhU}`OM(E5*M`*&y)_lEY{NP2)0)q`|*2O-w(}1m^P8k z^7a}!A_ZQ`$jmpnldc0TeOq*+gX_{-jos*Et?XrbMoOp45`WU$Bk$CJ8wa5R~tIXw(=VpFXLNUIQ4Dlzb0+6zCU!wPSt$lhVi(99LLcT2i zq2e!OfvU;oWEw<`JB3TsKUA@~*m{)HE%wMB_IBlGO8NYMkfIP|%E6L_~@?tM3OUF?s;h<{dCDvIWH* z-Gx005Wky!O1eus= zInU+(n7U?<3`{B`j*^_;mAaI}ez?{TYNhG6!4akdT>Tj1xHJ)JE7wL8Dy+N8_HtV! zZ^7k4q;BqTMhWjn?bc(bp8PUuzBl4}viU(+5jr7FR`8ls_*k&*=j>6LzU_&vq(B{o zZVjIRowDNYiy?Mjdp-ChI-+DuR*hp`eq%_`AP~7+Z9EkD=GSmjbyrQPOX}Nt;&xm? z%Z+?_hX)zhJ2r8_Gx1W%rJ*<20*4D4j+xMX7ee1KnFbbtWnw$UQW2rGj77?$@CmB7 z)z!jB{(spjbUtvjeER4Lgw|;u9nUj&6Uos7kJ)F7s5LQc_>kKx&S~P;Iti1t@=Xf5 zm^g=E%dHT{>j_0!Y;8_!I-X6(WSCEK@JSes>Uxp|5D zS+!;}-|rIHwIw(zILe4@t??8;m*h>`cB-lVKv;|RY}c=XUhvqSiZ1NXK9`+AMC+qO z{!2$F9L)oUF$6+ql8dUMu-Fl1Bed-0K?8SopevxJC51l-O;VA@YIQl>PRqi;DvdK_TlcEzhU|> DziXlX literal 0 HcmV?d00001 diff --git a/query-microservices/query-api/service/src/test/resources/ssl/rootCA.p12 b/query-microservices/query-api/service/src/test/resources/ssl/rootCA.p12 new file mode 100644 index 0000000000000000000000000000000000000000..8c071083ddd7f5973571edbb60ef237f3930bec9 GIT binary patch literal 2644 zcmY+^cQhN0762dL zC1LxDk{ts)-N)mQ(xJ)eZRt5`dF7XenBl>85N6za>OGpT_@ky zGcJgfYhN7DcH#ECu$R~<<31o48|Z16nn!)M6n@s-b*hQRXYSDz`K~OnM3fPNYmSI! zZyY_1s!rQ+o!k7IQAk*_gB}CCSSYJNiHQz!y7yJC6J%K|T6vXaoYH2xGlwh{lm0Sc zzFXD5lS*Ils)HW%S=BSJf_CzqCPSCmdC=TgWlCgxr(!P}T5HO0hzpkl&xiCrNY5u| zbv_ZW-*VeH>!UPE8AxgSoP&N*QSN@J)CgJ~{;G4;2*+)`&^&3*o-xVjHK=8<8eu&= ze!xwZbZm9N$7?@e)6^8IM^LC0pA?}f6tKswuIvxjqvfSom80m}i*c*-n7KDQFo9!! zssuj{)}UPI?$2A6E0xyn?KU{OeF4W*mD=RO{uymhRI)EuZ%pi=JD>T z%lKZg+70rZNq3luVzlQ;oORAsSIc2961&i(89Zbl?EG_XNnRMW6R{K9snc7NYXvvk8WT2c&+4*HyiJ|aMm%r+9qcwx!hn^@NFbatz9sGy zB;_l=RQ-6?3f+CIm4LQcEO8kKC#2vqB|>+bg(W&4VKblWPBp>|etj>$AZCbG!Q#_n zyHxCDu8iZx+#&qATVmwE=BLP94mD4<(u-WaZ#-Aaqc0iknZ-XyK6oxDyB@2n=vQYx z`6)Gfa}fcOEe>tK@Od1b@$XOnh>au|h)87l28F$6ca>T@KdW)WL1O@pxUv_seHNtj zZjyr4xJ|!LW12>N)8N9oaa85mUfc3_p>SLtT zl}PXL_SILx{zRjA>~!$hX|+pq3^S)>h7x^3=Uvm-%AnlkwF+*pt_6(0>ZDs?AqCp$ znewHM7OQJ@B(Uu5x-lJasQ%KYQ%I-hngX{{Y|xqK$Ad42LU8)k1~RaSP>G!v0Xx)Q z-oQPUAD`pSZ5>NTU?!%%?~C1kf0Yol)oQ|eRI_Ys$$5pRu)`}b;e$U7Oq~8!#09}a z4u}#QL6jAG{i-nz=!DP|WT>SMo!7Wi7m~zlCV+dyrYNE4yzHpnsq4Hj@pR2ks;x_g z%L?-gs41vCj9)tE?bH7`0P9V3nXe7$lkX}SnqAD6P%MaFH{2F?H=<3HV`K`z_vq8- zQY^Le29*z)P-Avnht4jo{fYr&1b(n@=7Rohw5IyheE2nAqy;lP9KcWHr{QGPZ-qmi4k)ikuz!Z}fPL${R7t;U#DU4GkS$De`GzSsEoVQ1I8X-zy?p zUs>B|ea*pxY(iLaHb1^*kF6pCCy?>93vB9v9c+qh?jKe7@v%cgvf}ediic3(P6)21R-nPiiTYDee1yZ^TUSK+tIhUi# zb?a=b9hajw?52J>2-v_4J4zZ(KLLM`%yv2--Z8K6?$#8!o(4Sp_DVpn=8o@AZ_L%0YFJQMA< zzX?D;aP$viO{mY0anTB)tO&Gkev>?TWd7kPlMOq>ee4agx1z{q8@P5C>F@LO<(d8} zXYo{a2d68S-|ETluZ(1)*DP4RCjTCBg{+QC&{!UOtVwT{$AJ+1=oxa<=kAtaNl0dO z+#@&LOn1C{p{mKq4J)Bg>&W@G+j^PDbh!~)@H44(gNpcdOTJ6@F7Bo1T5`6Uxm8%S z(JPr2D;aIbN`0E~;N)*@G-~any|aP;Eitz3Vw2*Y z^H=5iwLq-0Y8G+N*b4VpG)Zwn=5+;9^^4wm&um_iv;|!rgA)C(j;FZfxj`_Ioy_#W z)!7$9GMJ(L0GZwoTJ29!V*eb$1DnH^B_C=t{tW+rJ4 sMj#&(00b-G+U}h_z0diR>7&!_Yi6t6iw0UCu=k_f4%>W4^&d?C1tW*xVE_OC literal 0 HcmV?d00001 From e348e35b5aca3b6ec177f12b0ea9a60348c0f67a Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 14 Jan 2021 11:46:18 -0500 Subject: [PATCH 003/218] WIP: Readme formatting --- query-microservices/query-api/service/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/query-microservices/query-api/service/README.md b/query-microservices/query-api/service/README.md index b0c4d7cd..5a88de8f 100644 --- a/query-microservices/query-api/service/README.md +++ b/query-microservices/query-api/service/README.md @@ -11,8 +11,8 @@ query capabilities. ### User API -| Method | Operation | Description | Path Param | Request Body | Response Body | -|:-------|:----------|:-----------------------|:-----------|:---------------| +| Method | Operation | Description | Path Param | Request Body | Response Body | +|:--------------|:-------------------------------------|:----------------------------------------------------------------------------------|:-------------------|:---------------------|:-----------------------------------------| | `GET` | /listQueryLogic | List QueryLogic types that are currently available | N/A | N/A | [QueryLogicResponse] | | `POST` | /{queryLogic}/define | Define a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | | `POST` | /{queryLogic}/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | @@ -22,10 +22,10 @@ query capabilities. | `PUT` | /{id}/reset | Resets the specified query | [QueryId] | N/A | [VoidResponse] | | `POST` | /{queryLogic}/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | | `POST` | /{queryLogic}/async/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | -| `GET` | /lookupContentUUID/{uuidType}/{uuid} | Returns content associated with the given UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] || [StreamingOutput] | -| `POST` | /lookupContentUUID | Returns content associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] || [StreamingOutput] | -| `GET` | /lookupUUID/{uuidType}/{uuid} | Returns event associated with the given batch of UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] || [StreamingOutput] | -| `POST` | /lookupUUID | Returns event(s) associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] || [StreamingOutput] | +| `GET` | /lookupContentUUID/{uuidType}/{uuid} | Returns content associated with the given UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] or [StreamingOutput] | +| `POST` | /lookupContentUUID | Returns content associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] or [StreamingOutput] | +| `GET` | /lookupUUID/{uuidType}/{uuid} | Returns event associated with the given batch of UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] or [StreamingOutput] | +| `POST` | /lookupUUID | Returns event(s) associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] or [StreamingOutput] | | `GET` | /{id}/plan | Returns the plan for the specified query | [QueryId] | N/A | [GenericResponse] | | `GET` | /{id}/predictions | Returns the predictions for the specified query | [QueryId] | N/A | [GenericResponse] | | `GET` | /{id}/async/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | From 31d972432ffc1acc938a1d7508a5a20c5dccfad4 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Wed, 20 Jan 2021 14:42:29 +0000 Subject: [PATCH 004/218] More query storage --- .../java/datawave/microservice/query/api/QueryController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/query-microservices/query-api/service/src/main/java/datawave/microservice/query/api/QueryController.java b/query-microservices/query-api/service/src/main/java/datawave/microservice/query/api/QueryController.java index 234fba9e..1332bb44 100644 --- a/query-microservices/query-api/service/src/main/java/datawave/microservice/query/api/QueryController.java +++ b/query-microservices/query-api/service/src/main/java/datawave/microservice/query/api/QueryController.java @@ -7,5 +7,5 @@ @RestController @RequestMapping(path = "/v1", produces = MediaType.APPLICATION_JSON_VALUE) public class QueryController { - -} \ No newline at end of file + +} From fce3a0c4e6a4dff43c26321f21763b1ec04eaa81 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Tue, 19 Jan 2021 11:04:08 -0500 Subject: [PATCH 005/218] WIP: Working through translation of existing method annotations to spring boot-specific annotations. --- .../query/api/QueryController.java | 11 ------ .../{query-api => query}/api/pom.xml | 2 +- .../{query-api => query}/pom.xml | 2 +- .../{query-api => query}/service/README.md | 6 +-- .../{query-api => query}/service/pom.xml | 4 +- .../service/src/main/docker/Dockerfile | 0 .../microservice/query/QueryController.java | 35 ++++++++++++++++++ .../src/main/resources/config/application.yml | 10 +++++ .../src/main/resources/config/bootstrap.yml | 0 .../service/src/main/resources/log4j2.xml | 0 .../src/test/resources/config/application.yml | 0 .../src/test/resources/config/bootstrap.yml | 0 .../src/test/resources/log4j2-test.xml | 0 .../service/src/test/resources/ssl/host.p12 | Bin .../service/src/test/resources/ssl/rootCA.p12 | Bin 15 files changed, 52 insertions(+), 18 deletions(-) delete mode 100644 query-microservices/query-api/service/src/main/java/datawave/microservice/query/api/QueryController.java rename query-microservices/{query-api => query}/api/pom.xml (97%) rename query-microservices/{query-api => query}/pom.xml (95%) rename query-microservices/{query-api => query}/service/README.md (98%) rename query-microservices/{query-api => query}/service/pom.xml (96%) rename query-microservices/{query-api => query}/service/src/main/docker/Dockerfile (100%) create mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java create mode 100644 query-microservices/query/service/src/main/resources/config/application.yml rename query-microservices/{query-api => query}/service/src/main/resources/config/bootstrap.yml (100%) rename query-microservices/{query-api => query}/service/src/main/resources/log4j2.xml (100%) rename query-microservices/{query-api => query}/service/src/test/resources/config/application.yml (100%) rename query-microservices/{query-api => query}/service/src/test/resources/config/bootstrap.yml (100%) rename query-microservices/{query-api => query}/service/src/test/resources/log4j2-test.xml (100%) rename query-microservices/{query-api => query}/service/src/test/resources/ssl/host.p12 (100%) rename query-microservices/{query-api => query}/service/src/test/resources/ssl/rootCA.p12 (100%) diff --git a/query-microservices/query-api/service/src/main/java/datawave/microservice/query/api/QueryController.java b/query-microservices/query-api/service/src/main/java/datawave/microservice/query/api/QueryController.java deleted file mode 100644 index 1332bb44..00000000 --- a/query-microservices/query-api/service/src/main/java/datawave/microservice/query/api/QueryController.java +++ /dev/null @@ -1,11 +0,0 @@ -package datawave.microservice.query.api; - -import org.springframework.http.MediaType; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -@RestController -@RequestMapping(path = "/v1", produces = MediaType.APPLICATION_JSON_VALUE) -public class QueryController { - -} diff --git a/query-microservices/query-api/api/pom.xml b/query-microservices/query/api/pom.xml similarity index 97% rename from query-microservices/query-api/api/pom.xml rename to query-microservices/query/api/pom.xml index 6038f4c7..577e7263 100644 --- a/query-microservices/query-api/api/pom.xml +++ b/query-microservices/query/api/pom.xml @@ -7,7 +7,7 @@ 1.8 - query-api + query 1.0-SNAPSHOT diff --git a/query-microservices/query-api/pom.xml b/query-microservices/query/pom.xml similarity index 95% rename from query-microservices/query-api/pom.xml rename to query-microservices/query/pom.xml index a1a949e2..c1c58b15 100644 --- a/query-microservices/query-api/pom.xml +++ b/query-microservices/query/pom.xml @@ -7,7 +7,7 @@ 1.8 - query-api-service-parent + query-service-parent 1.0-SNAPSHOT pom diff --git a/query-microservices/query-api/service/README.md b/query-microservices/query/service/README.md similarity index 98% rename from query-microservices/query-api/service/README.md rename to query-microservices/query/service/README.md index 5a88de8f..c8241095 100644 --- a/query-microservices/query-api/service/README.md +++ b/query-microservices/query/service/README.md @@ -1,13 +1,13 @@ -# Query API Service +# Query Service [![Apache License][li]][ll] ![Build Status](https://github.com/NationalSecurityAgency/datawave/workflows/Tests/badge.svg) The query api service is a DATAWAVE microservice that provides query capabilities. -### Query API Context +### Query Context -*https://host:port/query-api/v1/* +*https://host:port/query/v1/* ### User API diff --git a/query-microservices/query-api/service/pom.xml b/query-microservices/query/service/pom.xml similarity index 96% rename from query-microservices/query-api/service/pom.xml rename to query-microservices/query/service/pom.xml index 0c422ccd..82b48655 100644 --- a/query-microservices/query-api/service/pom.xml +++ b/query-microservices/query/service/pom.xml @@ -7,9 +7,9 @@ 2.0 - query-api-service + query-service 1.0-SNAPSHOT - DATAWAVE Query API Microservice + DATAWAVE Query Microservice 1.0-SNAPSHOT 1.6.5 diff --git a/query-microservices/query-api/service/src/main/docker/Dockerfile b/query-microservices/query/service/src/main/docker/Dockerfile similarity index 100% rename from query-microservices/query-api/service/src/main/docker/Dockerfile rename to query-microservices/query/service/src/main/docker/Dockerfile diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java new file mode 100644 index 00000000..12db2dc7 --- /dev/null +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java @@ -0,0 +1,35 @@ +package datawave.microservice.query; + +import org.springframework.http.MediaType; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping(path = "/v1", produces = MediaType.APPLICATION_JSON_VALUE) +public class QueryController { + + + +// /** +// * @param queryLogicName +// * @param queryParameters +// * @return +// */ +// @POST +// @Produces({"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", +// "application/x-protostuff"}) +// @Path("/{logicName}/define") +// @GZIP +// @GenerateQuerySessionId(cookieBasePath = "/DataWave/Query/") +// @EnrichQueryMetrics(methodType = MethodType.CREATE) +// @Interceptors({RequiredInterceptor.class, ResponseInterceptor.class}) +// @Timed(name = "dw.query.defineQuery", absolute = true) +// public GenericResponse defineQuery(@Required("logicName") @PathParam("logicName") String queryLogicName, +// MultivaluedMap queryParameters, @Context HttpHeaders httpHeaders) + @RequestMapping(path = "{logicName}/define", method = {RequestMethod.POST}) + public String define() { + return null; + } + +} \ No newline at end of file diff --git a/query-microservices/query/service/src/main/resources/config/application.yml b/query-microservices/query/service/src/main/resources/config/application.yml new file mode 100644 index 00000000..7e2dc968 --- /dev/null +++ b/query-microservices/query/service/src/main/resources/config/application.yml @@ -0,0 +1,10 @@ +server: + compression: + # enable response compression + enabled: true + + # mime-types that should be compressed (wildcards NOT supported) + mime-types: "application/javascript, application/json, application/xml, application/x-yaml, application/x-protobuf, application/x-protostuff, text/css, text/html, text/javascript, text/plain, text/xml, text/x-yaml, text/yaml" + + # response size at which compression kicks in + min-response-size: 4KB diff --git a/query-microservices/query-api/service/src/main/resources/config/bootstrap.yml b/query-microservices/query/service/src/main/resources/config/bootstrap.yml similarity index 100% rename from query-microservices/query-api/service/src/main/resources/config/bootstrap.yml rename to query-microservices/query/service/src/main/resources/config/bootstrap.yml diff --git a/query-microservices/query-api/service/src/main/resources/log4j2.xml b/query-microservices/query/service/src/main/resources/log4j2.xml similarity index 100% rename from query-microservices/query-api/service/src/main/resources/log4j2.xml rename to query-microservices/query/service/src/main/resources/log4j2.xml diff --git a/query-microservices/query-api/service/src/test/resources/config/application.yml b/query-microservices/query/service/src/test/resources/config/application.yml similarity index 100% rename from query-microservices/query-api/service/src/test/resources/config/application.yml rename to query-microservices/query/service/src/test/resources/config/application.yml diff --git a/query-microservices/query-api/service/src/test/resources/config/bootstrap.yml b/query-microservices/query/service/src/test/resources/config/bootstrap.yml similarity index 100% rename from query-microservices/query-api/service/src/test/resources/config/bootstrap.yml rename to query-microservices/query/service/src/test/resources/config/bootstrap.yml diff --git a/query-microservices/query-api/service/src/test/resources/log4j2-test.xml b/query-microservices/query/service/src/test/resources/log4j2-test.xml similarity index 100% rename from query-microservices/query-api/service/src/test/resources/log4j2-test.xml rename to query-microservices/query/service/src/test/resources/log4j2-test.xml diff --git a/query-microservices/query-api/service/src/test/resources/ssl/host.p12 b/query-microservices/query/service/src/test/resources/ssl/host.p12 similarity index 100% rename from query-microservices/query-api/service/src/test/resources/ssl/host.p12 rename to query-microservices/query/service/src/test/resources/ssl/host.p12 diff --git a/query-microservices/query-api/service/src/test/resources/ssl/rootCA.p12 b/query-microservices/query/service/src/test/resources/ssl/rootCA.p12 similarity index 100% rename from query-microservices/query-api/service/src/test/resources/ssl/rootCA.p12 rename to query-microservices/query/service/src/test/resources/ssl/rootCA.p12 From fdba617ede01af55fb7f60a2783e814692088ee2 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Wed, 20 Jan 2021 18:25:53 +0000 Subject: [PATCH 006/218] formatting --- .../microservice/query/QueryController.java | 40 +++++++++---------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java index 12db2dc7..7fc373a3 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java @@ -8,28 +8,26 @@ @RestController @RequestMapping(path = "/v1", produces = MediaType.APPLICATION_JSON_VALUE) public class QueryController { - - - -// /** -// * @param queryLogicName -// * @param queryParameters -// * @return -// */ -// @POST -// @Produces({"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", -// "application/x-protostuff"}) -// @Path("/{logicName}/define") -// @GZIP -// @GenerateQuerySessionId(cookieBasePath = "/DataWave/Query/") -// @EnrichQueryMetrics(methodType = MethodType.CREATE) -// @Interceptors({RequiredInterceptor.class, ResponseInterceptor.class}) -// @Timed(name = "dw.query.defineQuery", absolute = true) -// public GenericResponse defineQuery(@Required("logicName") @PathParam("logicName") String queryLogicName, -// MultivaluedMap queryParameters, @Context HttpHeaders httpHeaders) + + // /** + // * @param queryLogicName + // * @param queryParameters + // * @return + // */ + // @POST + // @Produces({"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", + // "application/x-protostuff"}) + // @Path("/{logicName}/define") + // @GZIP + // @GenerateQuerySessionId(cookieBasePath = "/DataWave/Query/") + // @EnrichQueryMetrics(methodType = MethodType.CREATE) + // @Interceptors({RequiredInterceptor.class, ResponseInterceptor.class}) + // @Timed(name = "dw.query.defineQuery", absolute = true) + // public GenericResponse defineQuery(@Required("logicName") @PathParam("logicName") String queryLogicName, + // MultivaluedMap queryParameters, @Context HttpHeaders httpHeaders) @RequestMapping(path = "{logicName}/define", method = {RequestMethod.POST}) public String define() { return null; } - -} \ No newline at end of file + +} From 55196738e21d6ca4f88945be64263cf1a0d9d3c7 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 22 Jan 2021 16:40:45 +0000 Subject: [PATCH 007/218] Making progress on rest controller annotation translation. --- .../query/{service => }/README.md | 0 query-microservices/query/api/pom.xml | 2 +- query-microservices/query/service/pom.xml | 10 +++ .../microservice/query/QueryController.java | 2 + .../filter/GenerateQuerySessionIdFilter.java | 68 +++++++++++++++++++ .../annotation/GenerateQuerySessionId.java | 21 ++++++ 6 files changed, 102 insertions(+), 1 deletion(-) rename query-microservices/query/{service => }/README.md (100%) create mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/filter/GenerateQuerySessionIdFilter.java create mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/filter/annotation/GenerateQuerySessionId.java diff --git a/query-microservices/query/service/README.md b/query-microservices/query/README.md similarity index 100% rename from query-microservices/query/service/README.md rename to query-microservices/query/README.md diff --git a/query-microservices/query/api/pom.xml b/query-microservices/query/api/pom.xml index 577e7263..6038f4c7 100644 --- a/query-microservices/query/api/pom.xml +++ b/query-microservices/query/api/pom.xml @@ -7,7 +7,7 @@ 1.8 - query + query-api 1.0-SNAPSHOT diff --git a/query-microservices/query/service/pom.xml b/query-microservices/query/service/pom.xml index 82b48655..5743ed75 100644 --- a/query-microservices/query/service/pom.xml +++ b/query-microservices/query/service/pom.xml @@ -11,6 +11,7 @@ 1.0-SNAPSHOT DATAWAVE Query Microservice + 3.1.5-SNAPSHOT 1.0-SNAPSHOT 1.6.5 @@ -26,6 +27,11 @@ spring-boot-starter-datawave ${version.microservice.starter} + + gov.nsa.datawave.webservices + datawave-ws-client + ${version.datawave} + @@ -37,6 +43,10 @@ gov.nsa.datawave.microservice spring-boot-starter-datawave + + gov.nsa.datawave.webservices + datawave-ws-client + diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java index 7fc373a3..19869a46 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java @@ -1,5 +1,6 @@ package datawave.microservice.query; +import datawave.microservice.query.filter.annotation.GenerateQuerySessionId; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; @@ -25,6 +26,7 @@ public class QueryController { // @Timed(name = "dw.query.defineQuery", absolute = true) // public GenericResponse defineQuery(@Required("logicName") @PathParam("logicName") String queryLogicName, // MultivaluedMap queryParameters, @Context HttpHeaders httpHeaders) + @GenerateQuerySessionId(cookieBasePath = "/DataWave/Query/") @RequestMapping(path = "{logicName}/define", method = {RequestMethod.POST}) public String define() { return null; diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/filter/GenerateQuerySessionIdFilter.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/filter/GenerateQuerySessionIdFilter.java new file mode 100644 index 00000000..8e7ac897 --- /dev/null +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/filter/GenerateQuerySessionIdFilter.java @@ -0,0 +1,68 @@ +package datawave.microservice.query.filter; + +import datawave.Constants; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Component; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.UUID; + +@Component +public class GenerateQuerySessionIdFilter implements Filter { + private final Logger log = Logger.getLogger(GenerateQuerySessionIdFilter.class); + public static final ThreadLocal QUERY_ID = new ThreadLocal<>(); + + @Override + public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { + + // before the request is serviced by the rest controller + filterChain.doFilter(servletRequest, servletResponse); + + // after the request is serviced by the rest controller + if (servletResponse instanceof HttpServletResponse) { + HttpServletResponse httpServletResponse = (HttpServletResponse) servletResponse; + + String path = "TODO"; + String id = ""; + String cookieValue = generateCookieValue(); + boolean setCookie = true; + switch (HttpStatus.Series.valueOf(httpServletResponse.getStatus())) { + case SERVER_ERROR: + case CLIENT_ERROR: + // If we're sending an error response, then there's no need to set a cookie since + // there's no query "session" to stick to this server. + setCookie = false; + QUERY_ID.set(null); + break; + + default: + if (StringUtils.isEmpty(QUERY_ID.get())) { + log.error("threadlocal QUERY_ID was not set."); + } else { + id = QUERY_ID.get(); + QUERY_ID.set(null); + } + break; + } + + if (setCookie) { + Cookie cookie = new Cookie(Constants.QUERY_COOKIE_NAME, cookieValue); + cookie.setPath(path + id); + httpServletResponse.addCookie(cookie); + } + } + } + + public static String generateCookieValue() { + return Integer.toString(UUID.randomUUID().hashCode() & Integer.MAX_VALUE); + } +} diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/filter/annotation/GenerateQuerySessionId.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/filter/annotation/GenerateQuerySessionId.java new file mode 100644 index 00000000..3240239e --- /dev/null +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/filter/annotation/GenerateQuerySessionId.java @@ -0,0 +1,21 @@ +package datawave.microservice.query.filter.annotation; + +import java.lang.annotation.*; + +/** + * An annotation which is used to identify methods that are designated to create a new query and therefore set a cookie with an id for the query. Load balancers + * can use this query to stick other requests for the query (e.g., next, close) to the web server that created the query, and therefore the one that has the + * resources needed to evaluate the query. + * + * @see datawave.annotation.ClearQuerySessionId + * @see datawave.resteasy.interceptor.CreateQuerySessionIDFilter + */ +@Target({ElementType.TYPE, ElementType.METHOD, ElementType.PARAMETER}) +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface GenerateQuerySessionId { + /** + * @return The base path for the generated cookie. The base path will be combined with the query id in order to form the cookie domain. + */ + String cookieBasePath(); +} From 3bdcdafe1d6f151d92a5d823108a98e486335912 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Mon, 25 Jan 2021 16:39:17 +0000 Subject: [PATCH 008/218] Working on a query service test to verify functionality. --- query-microservices/query/service/pom.xml | 9 +- .../microservice/query/QueryController.java | 2 +- .../microservice/query/QueryService.java | 17 ++++ .../src/main/resources/config/bootstrap.yml | 2 +- .../microservice/query/QueryServiceTest.java | 82 +++++++++++++++++++ .../src/test/resources/config/application.yml | 4 +- 6 files changed, 111 insertions(+), 5 deletions(-) create mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/QueryService.java create mode 100644 query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java diff --git a/query-microservices/query/service/pom.xml b/query-microservices/query/service/pom.xml index 5743ed75..5fce6cc7 100644 --- a/query-microservices/query/service/pom.xml +++ b/query-microservices/query/service/pom.xml @@ -32,6 +32,13 @@ datawave-ws-client ${version.datawave} + + gov.nsa.datawave.microservice + spring-boot-starter-datawave + ${version.microservice.starter} + pom + import + @@ -89,4 +96,4 @@ - \ No newline at end of file + diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java index 19869a46..d2348be0 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java @@ -29,7 +29,7 @@ public class QueryController { @GenerateQuerySessionId(cookieBasePath = "/DataWave/Query/") @RequestMapping(path = "{logicName}/define", method = {RequestMethod.POST}) public String define() { - return null; + return ""; } } diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryService.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryService.java new file mode 100644 index 00000000..aaf85e33 --- /dev/null +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryService.java @@ -0,0 +1,17 @@ +package datawave.microservice.query; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.web.servlet.error.ErrorMvcAutoConfiguration; +import org.springframework.cloud.client.discovery.EnableDiscoveryClient; + +/** + * Launcher for the query service + */ +@EnableDiscoveryClient +@SpringBootApplication(scanBasePackages = "datawave.microservice", exclude = {ErrorMvcAutoConfiguration.class}) +public class QueryService { + public static void main(String[] args) { + SpringApplication.run(QueryService.class, args); + } +} diff --git a/query-microservices/query/service/src/main/resources/config/bootstrap.yml b/query-microservices/query/service/src/main/resources/config/bootstrap.yml index 9b33de46..f049a2f5 100644 --- a/query-microservices/query/service/src/main/resources/config/bootstrap.yml +++ b/query-microservices/query/service/src/main/resources/config/bootstrap.yml @@ -1,6 +1,6 @@ spring: application: - name: query-api + name: query cloud: config: # Disable consul-first config by default. We'll turn it back on in the consul profile if that profile is enabled. diff --git a/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java b/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java new file mode 100644 index 00000000..c5158ff4 --- /dev/null +++ b/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java @@ -0,0 +1,82 @@ +package datawave.microservice.query; + +import datawave.microservice.authorization.jwt.JWTRestTemplate; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.security.authorization.DatawaveUser; +import datawave.security.authorization.SubjectIssuerDNPair; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.boot.web.server.LocalServerPort; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import org.springframework.http.HttpMethod; +import org.springframework.http.RequestEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; +import org.springframework.web.util.UriComponentsBuilder; + +import java.util.Collection; +import java.util.Collections; + +import static datawave.security.authorization.DatawaveUser.UserType.USER; +import static org.springframework.test.util.AssertionErrors.assertTrue; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ContextConfiguration(classes = QueryServiceTest.QueryServiceTestConfiguration.class) +@ActiveProfiles({"QueryServiceTest"}) +public class QueryServiceTest { + + @LocalServerPort + private int webServicePort; + + @Autowired + private RestTemplateBuilder restTemplateBuilder; + + private JWTRestTemplate jwtRestTemplate; + + private SubjectIssuerDNPair DN; + private String userDN = "userDn"; + + @Before + public void setup() { + jwtRestTemplate = restTemplateBuilder.build(JWTRestTemplate.class); + DN = SubjectIssuerDNPair.of(userDN, "issuerDn"); + } + + @Test + public void testQuery() { + Collection roles = Collections.singleton("AuthorizedUser"); + DatawaveUser uathDWUser = new DatawaveUser(DN, USER, null, roles, null, System.currentTimeMillis()); + ProxiedUserDetails authUser = new ProxiedUserDetails(Collections.singleton(uathDWUser), uathDWUser.getCreationTime()); + + UriComponents uri = UriComponentsBuilder.newInstance().scheme("https").host("localhost").port(webServicePort).path("/query/v1/EventQuery/define") + .build(); + + MultiValueMap map = new LinkedMultiValueMap<>(); + + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + try { + jwtRestTemplate.exchange(requestEntity, String.class); + } finally { + assertTrue("", true); + } + } + + @Configuration + @Profile("QueryServiceTest") + @ComponentScan(basePackages = "datawave.microservice") + public static class QueryServiceTestConfiguration { + + } +} diff --git a/query-microservices/query/service/src/test/resources/config/application.yml b/query-microservices/query/service/src/test/resources/config/application.yml index d99c2e19..d80d73c2 100644 --- a/query-microservices/query/service/src/test/resources/config/application.yml +++ b/query-microservices/query/service/src/test/resources/config/application.yml @@ -1,7 +1,7 @@ spring: application: - name: query-api-test + name: query autoconfigure: exclude: org.springframework.boot.autoconfigure.web.servlet.error.ErrorMvcAutoConfiguration @@ -19,7 +19,7 @@ hazelcast.client.enabled: false server: port: 0 non-secure-port: 0 - servlet.context-path: /query-api + servlet.context-path: /query ssl: client-auth: NEED trust-store: classpath:ssl/rootCA.p12 From fff2c492927f2fb71b81733bc902a0496423c503 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 27 Jan 2021 13:52:31 +0000 Subject: [PATCH 009/218] Created QuerySessionIdFilter which can be used to configure cookies. --- .../microservice/query/QueryController.java | 6 +- ...dFilter.java => QuerySessionIdFilter.java} | 59 ++++++++++++++----- .../annotation/GenerateQuerySessionId.java | 21 ------- 3 files changed, 48 insertions(+), 38 deletions(-) rename query-microservices/query/service/src/main/java/datawave/microservice/query/filter/{GenerateQuerySessionIdFilter.java => QuerySessionIdFilter.java} (52%) delete mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/filter/annotation/GenerateQuerySessionId.java diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java index d2348be0..ba819fb4 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java @@ -1,6 +1,6 @@ package datawave.microservice.query; -import datawave.microservice.query.filter.annotation.GenerateQuerySessionId; +import datawave.microservice.query.filter.QuerySessionIdFilter.QuerySessionIdContext; import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; @@ -26,9 +26,11 @@ public class QueryController { // @Timed(name = "dw.query.defineQuery", absolute = true) // public GenericResponse defineQuery(@Required("logicName") @PathParam("logicName") String queryLogicName, // MultivaluedMap queryParameters, @Context HttpHeaders httpHeaders) - @GenerateQuerySessionId(cookieBasePath = "/DataWave/Query/") @RequestMapping(path = "{logicName}/define", method = {RequestMethod.POST}) public String define() { + QuerySessionIdContext.setCookieBasePath("/DataWave/Query/"); + QuerySessionIdContext.setQueryId("some-query-id"); + return ""; } diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/filter/GenerateQuerySessionIdFilter.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/filter/QuerySessionIdFilter.java similarity index 52% rename from query-microservices/query/service/src/main/java/datawave/microservice/query/filter/GenerateQuerySessionIdFilter.java rename to query-microservices/query/service/src/main/java/datawave/microservice/query/filter/QuerySessionIdFilter.java index 8e7ac897..39b4507a 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/filter/GenerateQuerySessionIdFilter.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/filter/QuerySessionIdFilter.java @@ -17,23 +17,22 @@ import java.util.UUID; @Component -public class GenerateQuerySessionIdFilter implements Filter { - private final Logger log = Logger.getLogger(GenerateQuerySessionIdFilter.class); - public static final ThreadLocal QUERY_ID = new ThreadLocal<>(); +public class QuerySessionIdFilter implements Filter { + private final Logger log = Logger.getLogger(QuerySessionIdFilter.class); @Override public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { - // before the request is serviced by the rest controller + QuerySessionIdContext.servletRequest.set(servletRequest); + filterChain.doFilter(servletRequest, servletResponse); - // after the request is serviced by the rest controller - if (servletResponse instanceof HttpServletResponse) { + String path = QuerySessionIdContext.getCookieBasePath(); + + if (path != null) { HttpServletResponse httpServletResponse = (HttpServletResponse) servletResponse; - String path = "TODO"; String id = ""; - String cookieValue = generateCookieValue(); boolean setCookie = true; switch (HttpStatus.Series.valueOf(httpServletResponse.getStatus())) { case SERVER_ERROR: @@ -41,28 +40,58 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo // If we're sending an error response, then there's no need to set a cookie since // there's no query "session" to stick to this server. setCookie = false; - QUERY_ID.set(null); break; default: - if (StringUtils.isEmpty(QUERY_ID.get())) { - log.error("threadlocal QUERY_ID was not set."); + if (StringUtils.isEmpty(QuerySessionIdContext.getQueryId())) { + log.error("queryId was not set."); } else { - id = QUERY_ID.get(); - QUERY_ID.set(null); + id = QuerySessionIdContext.getQueryId(); } break; } if (setCookie) { - Cookie cookie = new Cookie(Constants.QUERY_COOKIE_NAME, cookieValue); + Cookie cookie = new Cookie(Constants.QUERY_COOKIE_NAME, generateCookieValue()); cookie.setPath(path + id); httpServletResponse.addCookie(cookie); } } } - public static String generateCookieValue() { + private static String generateCookieValue() { return Integer.toString(UUID.randomUUID().hashCode() & Integer.MAX_VALUE); } + + @Override + public void destroy() { + QuerySessionIdContext.removeServletRequest(); + } + + public static class QuerySessionIdContext { + private static final String COOKIE_BASE_PATH = "cookieBasePath"; + private static final String QUERY_ID = "queryId"; + + private static final ThreadLocal servletRequest = new ThreadLocal<>(); + + public static String getQueryId() { + return (String) servletRequest.get().getAttribute(QUERY_ID); + } + + public static void setQueryId(String queryId) { + servletRequest.get().setAttribute(QUERY_ID, queryId); + } + + public static String getCookieBasePath() { + return (String) servletRequest.get().getAttribute(COOKIE_BASE_PATH); + } + + public static void setCookieBasePath(String cookieBasePath) { + servletRequest.get().setAttribute(COOKIE_BASE_PATH, cookieBasePath); + } + + private static void removeServletRequest() { + servletRequest.remove(); + } + } } diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/filter/annotation/GenerateQuerySessionId.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/filter/annotation/GenerateQuerySessionId.java deleted file mode 100644 index 3240239e..00000000 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/filter/annotation/GenerateQuerySessionId.java +++ /dev/null @@ -1,21 +0,0 @@ -package datawave.microservice.query.filter.annotation; - -import java.lang.annotation.*; - -/** - * An annotation which is used to identify methods that are designated to create a new query and therefore set a cookie with an id for the query. Load balancers - * can use this query to stick other requests for the query (e.g., next, close) to the web server that created the query, and therefore the one that has the - * resources needed to evaluate the query. - * - * @see datawave.annotation.ClearQuerySessionId - * @see datawave.resteasy.interceptor.CreateQuerySessionIDFilter - */ -@Target({ElementType.TYPE, ElementType.METHOD, ElementType.PARAMETER}) -@Retention(RetentionPolicy.RUNTIME) -@Documented -public @interface GenerateQuerySessionId { - /** - * @return The base path for the generated cookie. The base path will be combined with the query id in order to form the cookie domain. - */ - String cookieBasePath(); -} From 88dd65135ef0a5e95fa49a5e1ec5bc6ed44ab9d6 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 29 Jan 2021 14:11:44 +0000 Subject: [PATCH 010/218] got tests running for the storage service --- .../java/datawave/microservice/query/QueryServiceTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java b/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java index c5158ff4..8ca5de2b 100644 --- a/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java +++ b/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java @@ -4,6 +4,7 @@ import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.security.authorization.DatawaveUser; import datawave.security.authorization.SubjectIssuerDNPair; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -72,7 +73,7 @@ public void testQuery() { assertTrue("", true); } } - + @Configuration @Profile("QueryServiceTest") @ComponentScan(basePackages = "datawave.microservice") From d90e1202716a7ed2d22038d2c041aa83a6e35c6b Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 29 Jan 2021 16:05:34 +0000 Subject: [PATCH 011/218] Downgraded to previous spring boot version to fix junit build issues. --- query-microservices/query/api/pom.xml | 2 +- query-microservices/query/pom.xml | 2 +- query-microservices/query/service/pom.xml | 2 +- .../test/java/datawave/microservice/query/QueryServiceTest.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/query-microservices/query/api/pom.xml b/query-microservices/query/api/pom.xml index 6038f4c7..97818bef 100644 --- a/query-microservices/query/api/pom.xml +++ b/query-microservices/query/api/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 1.8 + 1.7 query-api diff --git a/query-microservices/query/pom.xml b/query-microservices/query/pom.xml index c1c58b15..12a53e77 100644 --- a/query-microservices/query/pom.xml +++ b/query-microservices/query/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 1.8 + 1.7 query-service-parent diff --git a/query-microservices/query/service/pom.xml b/query-microservices/query/service/pom.xml index 5fce6cc7..db578d5d 100644 --- a/query-microservices/query/service/pom.xml +++ b/query-microservices/query/service/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-service-parent - 2.0 + 1.7.1 query-service diff --git a/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java b/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java index 8ca5de2b..79919d8b 100644 --- a/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java +++ b/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java @@ -73,7 +73,7 @@ public void testQuery() { assertTrue("", true); } } - + @Configuration @Profile("QueryServiceTest") @ComponentScan(basePackages = "datawave.microservice") From 09a781f4949bb31b8b995b09edd94d66754beb14 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Mon, 8 Feb 2021 16:55:25 +0000 Subject: [PATCH 012/218] Added logic for setting query header fields for base query responses. --- .../microservice/query/QueryController.java | 41 ++++---- .../query/config/QueryServiceConfig.java | 6 ++ .../query/filter/QuerySessionIdFilter.java | 97 ------------------ .../query/web/BaseQueryResponseAdvice.java | 36 +++++++ .../query/web/QuerySessionIdAdvice.java | 98 +++++++++++++++++++ .../annotation/GenerateQuerySessionId.java | 15 +++ .../microservice/query/QueryServiceTest.java | 5 +- 7 files changed, 183 insertions(+), 115 deletions(-) create mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java delete mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/filter/QuerySessionIdFilter.java create mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java create mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java create mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java index ba819fb4..9851f77e 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java @@ -1,9 +1,13 @@ package datawave.microservice.query; -import datawave.microservice.query.filter.QuerySessionIdFilter.QuerySessionIdContext; +import com.codahale.metrics.annotation.Timed; +import datawave.webservice.result.GenericResponse; import org.springframework.http.MediaType; +import org.springframework.util.MultiValueMap; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; @RestController @@ -15,23 +19,26 @@ public class QueryController { // * @param queryParameters // * @return // */ - // @POST - // @Produces({"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", + // X @POST + // X @Produces({"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", // "application/x-protostuff"}) - // @Path("/{logicName}/define") - // @GZIP - // @GenerateQuerySessionId(cookieBasePath = "/DataWave/Query/") + // X @Path("/{logicName}/define") + // X @GZIP + // X @GenerateQuerySessionId(cookieBasePath = "/DataWave/Query/") // @EnrichQueryMetrics(methodType = MethodType.CREATE) - // @Interceptors({RequiredInterceptor.class, ResponseInterceptor.class}) - // @Timed(name = "dw.query.defineQuery", absolute = true) - // public GenericResponse defineQuery(@Required("logicName") @PathParam("logicName") String queryLogicName, - // MultivaluedMap queryParameters, @Context HttpHeaders httpHeaders) - @RequestMapping(path = "{logicName}/define", method = {RequestMethod.POST}) - public String define() { - QuerySessionIdContext.setCookieBasePath("/DataWave/Query/"); - QuerySessionIdContext.setQueryId("some-query-id"); - - return ""; + // X @Interceptors({RequiredInterceptor.class, ResponseInterceptor.class}) + // X @Timed(name = "dw.query.defineQuery", absolute = true) + // X public GenericResponse defineQuery(@Required("logicName") @PathParam("logicName") String queryLogicName, + // X MultivaluedMap queryParameters, @Context HttpHeaders httpHeaders) + // NOTE: The goal is to not use this, but it's here if we need it. +// @GenerateQuerySessionId(cookieBasePath = "/DataWave/Query/") + @Timed(name = "dw.query.defineQuery", absolute = true) // TODO: Figure out where this is used + @RequestMapping(path = "{logicName}/define", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse define(@PathVariable(name = "logicName") String logicName, @RequestParam MultiValueMap parameters) { +// QuerySessionIdContext.setQueryId("some-query-id"); + GenericResponse resp = new GenericResponse<>(); + resp.setResult("something something"); + return resp; } - } diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java new file mode 100644 index 00000000..77c62c39 --- /dev/null +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java @@ -0,0 +1,6 @@ +package datawave.microservice.query.config; + +import org.springframework.context.annotation.Configuration; + +@Configuration +public class QueryServiceConfig {} diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/filter/QuerySessionIdFilter.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/filter/QuerySessionIdFilter.java deleted file mode 100644 index 39b4507a..00000000 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/filter/QuerySessionIdFilter.java +++ /dev/null @@ -1,97 +0,0 @@ -package datawave.microservice.query.filter; - -import datawave.Constants; -import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; -import org.springframework.http.HttpStatus; -import org.springframework.stereotype.Component; - -import javax.servlet.Filter; -import javax.servlet.FilterChain; -import javax.servlet.ServletException; -import javax.servlet.ServletRequest; -import javax.servlet.ServletResponse; -import javax.servlet.http.Cookie; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.util.UUID; - -@Component -public class QuerySessionIdFilter implements Filter { - private final Logger log = Logger.getLogger(QuerySessionIdFilter.class); - - @Override - public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { - - QuerySessionIdContext.servletRequest.set(servletRequest); - - filterChain.doFilter(servletRequest, servletResponse); - - String path = QuerySessionIdContext.getCookieBasePath(); - - if (path != null) { - HttpServletResponse httpServletResponse = (HttpServletResponse) servletResponse; - - String id = ""; - boolean setCookie = true; - switch (HttpStatus.Series.valueOf(httpServletResponse.getStatus())) { - case SERVER_ERROR: - case CLIENT_ERROR: - // If we're sending an error response, then there's no need to set a cookie since - // there's no query "session" to stick to this server. - setCookie = false; - break; - - default: - if (StringUtils.isEmpty(QuerySessionIdContext.getQueryId())) { - log.error("queryId was not set."); - } else { - id = QuerySessionIdContext.getQueryId(); - } - break; - } - - if (setCookie) { - Cookie cookie = new Cookie(Constants.QUERY_COOKIE_NAME, generateCookieValue()); - cookie.setPath(path + id); - httpServletResponse.addCookie(cookie); - } - } - } - - private static String generateCookieValue() { - return Integer.toString(UUID.randomUUID().hashCode() & Integer.MAX_VALUE); - } - - @Override - public void destroy() { - QuerySessionIdContext.removeServletRequest(); - } - - public static class QuerySessionIdContext { - private static final String COOKIE_BASE_PATH = "cookieBasePath"; - private static final String QUERY_ID = "queryId"; - - private static final ThreadLocal servletRequest = new ThreadLocal<>(); - - public static String getQueryId() { - return (String) servletRequest.get().getAttribute(QUERY_ID); - } - - public static void setQueryId(String queryId) { - servletRequest.get().setAttribute(QUERY_ID, queryId); - } - - public static String getCookieBasePath() { - return (String) servletRequest.get().getAttribute(COOKIE_BASE_PATH); - } - - public static void setCookieBasePath(String cookieBasePath) { - servletRequest.get().setAttribute(COOKIE_BASE_PATH, cookieBasePath); - } - - private static void removeServletRequest() { - servletRequest.remove(); - } - } -} diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java new file mode 100644 index 00000000..d09e70f0 --- /dev/null +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java @@ -0,0 +1,36 @@ +package datawave.microservice.query.web; + +import datawave.Constants; +import datawave.webservice.result.BaseQueryResponse; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.core.MethodParameter; +import org.springframework.http.MediaType; +import org.springframework.http.server.ServerHttpRequest; +import org.springframework.http.server.ServerHttpResponse; +import org.springframework.lang.NonNull; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyAdvice; + +// TODO: Should this be in the API or in a starter? +/** + * A {@link ControllerAdvice} that implements {@link ResponseBodyAdvice} in order to allow access to {@link BaseQueryResponse} objects before they are written + * out to the response body. This is primarily used to write the page number, is last page, and partial results headers for the response. + */ +@ControllerAdvice +@ConditionalOnClass(BaseQueryResponse.class) +public class BaseQueryResponseAdvice implements ResponseBodyAdvice { + + @Override + public boolean supports(@NonNull MethodParameter returnType, @NonNull Class converterType) { + return BaseQueryResponse.class.isAssignableFrom(returnType.getParameterType()); + } + + @Override + public BaseQueryResponse beforeBodyWrite(BaseQueryResponse baseQueryResponse, @NonNull MethodParameter returnType, @NonNull MediaType selectedContentType, + @NonNull Class selectedConverterType, @NonNull ServerHttpRequest request, ServerHttpResponse response) { + response.getHeaders().add(Constants.PAGE_NUMBER, String.valueOf(baseQueryResponse.getPageNumber())); + response.getHeaders().add(Constants.IS_LAST_PAGE, String.valueOf(!baseQueryResponse.getHasResults())); + response.getHeaders().add(Constants.PARTIAL_RESULTS, String.valueOf(baseQueryResponse.isPartialResults())); + return baseQueryResponse; + } +} diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java new file mode 100644 index 00000000..494151ee --- /dev/null +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java @@ -0,0 +1,98 @@ +package datawave.microservice.query.web; + +import datawave.Constants; +import datawave.microservice.query.web.annotation.GenerateQuerySessionId; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import org.springframework.core.MethodParameter; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.http.server.ServerHttpRequest; +import org.springframework.http.server.ServerHttpResponse; +import org.springframework.http.server.ServletServerHttpResponse; +import org.springframework.lang.NonNull; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyAdvice; + +import javax.servlet.http.Cookie; +import java.util.Arrays; +import java.util.UUID; + +// TODO: Update to enable based on properties +// TODO: Should this be in the API or in a starter? +@ControllerAdvice +public class QuerySessionIdAdvice implements ResponseBodyAdvice { + private final Logger log = Logger.getLogger(QuerySessionIdAdvice.class); + + @Override + public boolean supports(@NonNull MethodParameter returnType, @NonNull Class> converterType) { + return Arrays.stream(returnType.getMethodAnnotations()) + .anyMatch(GenerateQuerySessionId.class::isInstance); + } + + @Override + public Object beforeBodyWrite(Object body, @NonNull MethodParameter returnType, @NonNull MediaType selectedContentType, + @NonNull Class> selectedConverterType, @NonNull ServerHttpRequest request, + @NonNull ServerHttpResponse response) { + try { + GenerateQuerySessionId annotation = (GenerateQuerySessionId) Arrays.stream(returnType.getMethodAnnotations()) + .filter(GenerateQuerySessionId.class::isInstance).findAny().orElse(null); + + if (annotation != null) { + ServletServerHttpResponse httpServletResponse = (ServletServerHttpResponse) response; + String path = annotation.cookieBasePath(); + + String id = ""; + boolean setCookie = true; + switch (HttpStatus.Series.valueOf(httpServletResponse.getServletResponse().getStatus())) { + case SERVER_ERROR: + case CLIENT_ERROR: + // If we're sending an error response, then there's no need to set a cookie since + // there's no query "session" to stick to this server. + setCookie = false; + break; + + default: + if (StringUtils.isEmpty(QuerySessionIdContext.getQueryId())) { + log.error("queryId was not set."); + } else { + id = QuerySessionIdContext.getQueryId(); + } + break; + } + + if (setCookie) { + Cookie cookie = new Cookie(Constants.QUERY_COOKIE_NAME, generateCookieValue()); + cookie.setPath(path + id); + httpServletResponse.getServletResponse().addCookie(cookie); + } + } + + return body; + } finally { + QuerySessionIdContext.removeQueryId(); + } + } + + private static String generateCookieValue() { + return Integer.toString(UUID.randomUUID().hashCode() & Integer.MAX_VALUE); + } + + public static class QuerySessionIdContext { + + private static final ThreadLocal queryId = new ThreadLocal<>(); + + public static String getQueryId() { + return (String) queryId.get(); + } + + public static void setQueryId(String queryId) { + QuerySessionIdContext.queryId.set(queryId); + } + + private static void removeQueryId() { + queryId.remove(); + } + } +} diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java new file mode 100644 index 00000000..4da1849b --- /dev/null +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java @@ -0,0 +1,15 @@ +package datawave.microservice.query.web.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ElementType.TYPE, ElementType.METHOD, ElementType.PARAMETER}) +@Retention(RetentionPolicy.RUNTIME) +public @interface GenerateQuerySessionId { + /** + * @return The base path for the generated cookie. The base path will be combined with the query id in order to form the cookie domain. + */ + String cookieBasePath(); +} diff --git a/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java b/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java index 79919d8b..1ce0db1d 100644 --- a/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java +++ b/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java @@ -17,6 +17,7 @@ import org.springframework.context.annotation.Profile; import org.springframework.http.HttpMethod; import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringRunner; @@ -68,7 +69,9 @@ public void testQuery() { RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); try { - jwtRestTemplate.exchange(requestEntity, String.class); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, String.class); + + System.out.println("done!"); } finally { assertTrue("", true); } From 6bf89ead877c1e46ead3110c1454651d401d9abc Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Mon, 8 Feb 2021 16:57:56 +0000 Subject: [PATCH 013/218] formatting --- .../java/datawave/microservice/query/QueryController.java | 4 ++-- .../datawave/microservice/query/web/QuerySessionIdAdvice.java | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java index 9851f77e..09c7f1ad 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java @@ -31,12 +31,12 @@ public class QueryController { // X public GenericResponse defineQuery(@Required("logicName") @PathParam("logicName") String queryLogicName, // X MultivaluedMap queryParameters, @Context HttpHeaders httpHeaders) // NOTE: The goal is to not use this, but it's here if we need it. -// @GenerateQuerySessionId(cookieBasePath = "/DataWave/Query/") + // @GenerateQuerySessionId(cookieBasePath = "/DataWave/Query/") @Timed(name = "dw.query.defineQuery", absolute = true) // TODO: Figure out where this is used @RequestMapping(path = "{logicName}/define", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public GenericResponse define(@PathVariable(name = "logicName") String logicName, @RequestParam MultiValueMap parameters) { -// QuerySessionIdContext.setQueryId("some-query-id"); + // QuerySessionIdContext.setQueryId("some-query-id"); GenericResponse resp = new GenericResponse<>(); resp.setResult("something something"); return resp; diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java index 494151ee..f8cea1c3 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java @@ -27,8 +27,7 @@ public class QuerySessionIdAdvice implements ResponseBodyAdvice { @Override public boolean supports(@NonNull MethodParameter returnType, @NonNull Class> converterType) { - return Arrays.stream(returnType.getMethodAnnotations()) - .anyMatch(GenerateQuerySessionId.class::isInstance); + return Arrays.stream(returnType.getMethodAnnotations()).anyMatch(GenerateQuerySessionId.class::isInstance); } @Override From 0f250cf6f09ca39f2dc0189334acee7dd02cd524 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Sun, 14 Feb 2021 10:30:06 -0500 Subject: [PATCH 014/218] WIP: making progress on rest controller request stats interface. --- query-microservices/query/service/pom.xml | 2 +- .../microservice/query/QueryController.java | 21 +- .../query/config/QueryServiceConfig.java | 16 +- .../web/filter/BaseMethodStatsFilter.java | 213 ++++++++++++++++++ .../filter/BaseMethodStatsInterceptor.java | 100 ++++++++ .../microservice/query/QueryServiceTest.java | 4 +- 6 files changed, 346 insertions(+), 10 deletions(-) create mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java create mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsInterceptor.java diff --git a/query-microservices/query/service/pom.xml b/query-microservices/query/service/pom.xml index db578d5d..f849c74e 100644 --- a/query-microservices/query/service/pom.xml +++ b/query-microservices/query/service/pom.xml @@ -13,7 +13,7 @@ 3.1.5-SNAPSHOT 1.0-SNAPSHOT - 1.6.5 + 1.6.6-SNAPSHOT diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java index 09c7f1ad..8dcf8a1a 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java @@ -1,7 +1,8 @@ package datawave.microservice.query; import com.codahale.metrics.annotation.Timed; -import datawave.webservice.result.GenericResponse; +import datawave.microservice.query.web.annotation.GenerateQuerySessionId; +import datawave.webservice.result.BaseQueryResponse; import org.springframework.http.MediaType; import org.springframework.util.MultiValueMap; import org.springframework.web.bind.annotation.PathVariable; @@ -10,6 +11,8 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import javax.annotation.security.RolesAllowed; + @RestController @RequestMapping(path = "/v1", produces = MediaType.APPLICATION_JSON_VALUE) public class QueryController { @@ -31,14 +34,20 @@ public class QueryController { // X public GenericResponse defineQuery(@Required("logicName") @PathParam("logicName") String queryLogicName, // X MultivaluedMap queryParameters, @Context HttpHeaders httpHeaders) // NOTE: The goal is to not use this, but it's here if we need it. - // @GenerateQuerySessionId(cookieBasePath = "/DataWave/Query/") + @RolesAllowed({"AuthorizedUser", "AuthorizedServer", "InternalUser", "Administrator"}) + @GenerateQuerySessionId(cookieBasePath = "/DataWave/Query/") @Timed(name = "dw.query.defineQuery", absolute = true) // TODO: Figure out where this is used @RequestMapping(path = "{logicName}/define", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public GenericResponse define(@PathVariable(name = "logicName") String logicName, @RequestParam MultiValueMap parameters) { + public BaseQueryResponse define(@PathVariable(name = "logicName") String logicName, @RequestParam MultiValueMap parameters) { // QuerySessionIdContext.setQueryId("some-query-id"); - GenericResponse resp = new GenericResponse<>(); - resp.setResult("something something"); - return resp; + // GenericResponse resp = new GenericResponse<>(); + // resp.setResult("something something"); + // return resp; + BaseQueryResponse bqr = new BaseQueryResponse() { + + }; + + return bqr; } } diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java index 77c62c39..cda65171 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java @@ -1,6 +1,20 @@ package datawave.microservice.query.config; +import datawave.microservice.query.web.filter.BaseMethodStatsInterceptor; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.web.servlet.config.annotation.InterceptorRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; @Configuration -public class QueryServiceConfig {} +public class QueryServiceConfig { + @Bean + public WebMvcConfigurer BaseMethodStatsInterceptorConfigurer(BaseMethodStatsInterceptor baseMethodStatsInterceptor) { + return new WebMvcConfigurer() { + @Override + public void addInterceptors(InterceptorRegistry registry) { + registry.addInterceptor(baseMethodStatsInterceptor); + } + }; + } +} diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java new file mode 100644 index 00000000..377a41c8 --- /dev/null +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java @@ -0,0 +1,213 @@ +package datawave.microservice.query.web.filter; + +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Component; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.filter.OncePerRequestFilter; + +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.ServletOutputStream; +import javax.servlet.ServletResponse; +import javax.servlet.WriteListener; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpServletResponseWrapper; +import java.io.IOException; +import java.util.Enumeration; + +import static datawave.microservice.config.web.Constants.REQUEST_LOGIN_TIME_ATTRIBUTE; +import static datawave.microservice.config.web.Constants.REQUEST_START_TIME_NS_ATTRIBUTE; + +@Component +public class BaseMethodStatsFilter extends OncePerRequestFilter { + + protected static class RequestMethodStats { + private String uri; + private String method; + private long loginTime = -1; + private long callStartTime; + private MultiValueMap requestHeaders = new LinkedMultiValueMap<>(); + private MultiValueMap formParameters = new LinkedMultiValueMap<>(); + + public String getUri() { + return uri; + } + + public String getMethod() { + return method; + } + + public long getLoginTime() { + return loginTime; + } + + public long getCallStartTime() { + return callStartTime; + } + + public MultiValueMap getRequestHeaders() { + return requestHeaders; + } + + public MultiValueMap getFormParameters() { + return formParameters; + } + } + + protected static class ResponseMethodStats { + private int statusCode = -1; + private long loginTime = -1; + private long callTime = -1; + private long serializationTime = -1; + private long bytesWritten = -1; + private MultiValueMap responseHeaders = new LinkedMultiValueMap<>(); + + public int getStatusCode() { + return statusCode; + } + + public long getLoginTime() { + return loginTime; + } + + public long getCallTime() { + return callTime; + } + + public long getSerializationTime() { + return serializationTime; + } + + public long getBytesWritten() { + return bytesWritten; + } + + public MultiValueMap getResponseHeaders() { + return responseHeaders; + } + + } + + @Override + public void initFilterBean() throws ServletException { + System.out.println("init"); + } + + @Override + public void doFilterInternal(@NonNull HttpServletRequest request, @NonNull HttpServletResponse response, @NonNull FilterChain chain) + throws IOException, ServletException { + RequestMethodStats stats = new RequestMethodStats(); + + stats.uri = request.getRequestURI(); + stats.method = request.getMethod(); + + stats.callStartTime = System.nanoTime(); + try { + stats.callStartTime = (long) request.getAttribute(REQUEST_START_TIME_NS_ATTRIBUTE); + } catch (Exception e) { + // do nothing + } + try { + stats.loginTime = (long) request.getAttribute(REQUEST_LOGIN_TIME_ATTRIBUTE); + } catch (Exception e) { + // do nothing + } + + for (Enumeration headerNames = request.getHeaderNames(); headerNames.hasMoreElements();) { + String header = headerNames.nextElement(); + for (Enumeration headerValues = request.getHeaders(header); headerValues.hasMoreElements();){ + stats.requestHeaders.add(header, headerValues.nextElement()); + } + } + + // TODO: Finish this! +// MediaType mediaType = request.getMediaType(); +// if (mediaType != null && mediaType.isCompatible(MediaType.APPLICATION_FORM_URLENCODED_TYPE)) { +// MultivaluedMap formParameters = request.getHttpRequest().getDecodedFormParameters(); +// if (formParameters != null) +// MultivaluedTreeMap.addAll(formParameters, stats.formParameters); +// } + // request.setProperty(REQUEST_STATS_NAME, stats); + // return stats; + + System.out.println("doFilter before"); + if (response instanceof HttpServletResponse) { + chain.doFilter(request, new CountingHttpServletResponseWrapper((HttpServletResponse) response)); + } else { + chain.doFilter(request, response); + } + System.out.println("doFilter after"); + } + + @Override + public void destroy() { + System.out.println("destroy"); + } + + private static class CountingHttpServletResponseWrapper extends HttpServletResponseWrapper { + private ServletResponse response; + private CountingServletOutputStream cos; + + /** + * Creates a ServletResponse adaptor wrapping the given response object. + * + * @param response + * the {@link ServletResponse} to be wrapped + * @throws IllegalArgumentException + * if the response is null. + */ + public CountingHttpServletResponseWrapper(HttpServletResponse response) { + super(response); + this.response = response; + } + + @Override + public ServletOutputStream getOutputStream() throws IOException { + if (cos == null) { + cos = new CountingServletOutputStream(response.getOutputStream()); + } + return cos; + } + + public long getByteCount() { + return cos.getByteCount(); + } + } + + private static class CountingServletOutputStream extends ServletOutputStream { + private final ServletOutputStream outputStream; + private long count = 0; + + public CountingServletOutputStream(ServletOutputStream outputStream) { + this.outputStream = outputStream; + } + + @Override + public boolean isReady() { + return outputStream.isReady(); + } + + @Override + public void setWriteListener(WriteListener writeListener) { + outputStream.setWriteListener(writeListener); + } + + @Override + public void write(int b) throws IOException { + outputStream.write(b); + count++; + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + outputStream.write(b, off, len); + this.count += (long) len; + } + + public long getByteCount() { + return count; + } + } +} diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsInterceptor.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsInterceptor.java new file mode 100644 index 00000000..c51e0a50 --- /dev/null +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsInterceptor.java @@ -0,0 +1,100 @@ +package datawave.microservice.query.web.filter; + +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.servlet.ModelAndView; +import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +@Component +public class BaseMethodStatsInterceptor extends HandlerInterceptorAdapter { + protected static final Logger log = Logger.getLogger(BaseMethodStatsInterceptor.class); + + protected static class RequestMethodStats { + + private String uri; + private String method; + private long loginTime = -1; + private long callStartTime; + private MultiValueMap requestHeaders = new LinkedMultiValueMap<>(); + private MultiValueMap formParameters = new LinkedMultiValueMap<>(); + + public String getUri() { + return uri; + } + + public String getMethod() { + return method; + } + + public long getLoginTime() { + return loginTime; + } + + public long getCallStartTime() { + return callStartTime; + } + + public MultiValueMap getRequestHeaders() { + return requestHeaders; + } + + public MultiValueMap getFormParameters() { + return formParameters; + } + } + + protected static class ResponseMethodStats { + private int statusCode = -1; + private long loginTime = -1; + private long callTime = -1; + private long serializationTime = -1; + private long bytesWritten = -1; + private MultiValueMap responseHeaders = new LinkedMultiValueMap<>(); + + public int getStatusCode() { + return statusCode; + } + + public long getLoginTime() { + return loginTime; + } + + public long getCallTime() { + return callTime; + } + + public long getSerializationTime() { + return serializationTime; + } + + public long getBytesWritten() { + return bytesWritten; + } + + public MultiValueMap getResponseHeaders() { + return responseHeaders; + } + + } + + @Override + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception { + System.out.println("pre handle"); + return true; + } + + @Override + public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler, ModelAndView modelAndView) throws Exception { + System.out.println("post handle"); + } + + @Override + public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) throws Exception { + System.out.println("after completion"); + } +} diff --git a/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java b/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java index 1ce0db1d..e99d3c89 100644 --- a/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java +++ b/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java @@ -1,10 +1,10 @@ package datawave.microservice.query; import datawave.microservice.authorization.jwt.JWTRestTemplate; +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.security.authorization.DatawaveUser; import datawave.security.authorization.SubjectIssuerDNPair; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -35,7 +35,7 @@ @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ContextConfiguration(classes = QueryServiceTest.QueryServiceTestConfiguration.class) -@ActiveProfiles({"QueryServiceTest"}) +@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceTest { @LocalServerPort From 46654d2a4fc4fd570cfcca8ac9647ed5d8fc8eb9 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 18 Feb 2021 11:35:34 -0500 Subject: [PATCH 015/218] Added stats filter/interceptors which will be used to update datawave query metrics. --- query-microservices/query/service/pom.xml | 10 + .../microservice/query/QueryController.java | 56 +++--- .../query/config/QueryLogicConfig.java | 18 ++ .../query/config/QueryProperties.java | 26 +++ .../query/config/QueryServiceConfig.java | 15 +- .../microservice/query/web/QueryMetrics.java | 13 ++ .../query/web/QuerySessionIdAdvice.java | 4 +- .../web/annotation/EnrichQueryMetrics.java | 18 ++ .../web/filter/BaseMethodStatsFilter.java | 164 +++++++++++++---- .../filter/BaseMethodStatsInterceptor.java | 100 ---------- .../query/web/filter/LoggingStatsFilter.java | 80 ++++++++ .../QueryMetricsEnrichmentFilterAdvice.java | 174 ++++++++++++++++++ 12 files changed, 491 insertions(+), 187 deletions(-) create mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryLogicConfig.java create mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryProperties.java create mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java create mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java delete mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsInterceptor.java create mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java create mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java diff --git a/query-microservices/query/service/pom.xml b/query-microservices/query/service/pom.xml index f849c74e..2ccb889a 100644 --- a/query-microservices/query/service/pom.xml +++ b/query-microservices/query/service/pom.xml @@ -13,6 +13,7 @@ 3.1.5-SNAPSHOT 1.0-SNAPSHOT + 1.0-SNAPSHOT 1.6.6-SNAPSHOT @@ -22,6 +23,11 @@ query-api ${version.microservice.query-api} + + gov.nsa.datawave.microservice + query-cache + ${version.microservice.query-cache} + gov.nsa.datawave.microservice spring-boot-starter-datawave @@ -46,6 +52,10 @@ gov.nsa.datawave.microservice query-api + + gov.nsa.datawave.microservice + query-cache + gov.nsa.datawave.microservice spring-boot-starter-datawave diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java index 8dcf8a1a..8f43a5c4 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java @@ -1,8 +1,8 @@ package datawave.microservice.query; import com.codahale.metrics.annotation.Timed; -import datawave.microservice.query.web.annotation.GenerateQuerySessionId; -import datawave.webservice.result.BaseQueryResponse; +import datawave.microservice.query.web.annotation.EnrichQueryMetrics; +import datawave.webservice.result.GenericResponse; import org.springframework.http.MediaType; import org.springframework.util.MultiValueMap; import org.springframework.web.bind.annotation.PathVariable; @@ -11,43 +11,29 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import javax.annotation.security.RolesAllowed; - @RestController @RequestMapping(path = "/v1", produces = MediaType.APPLICATION_JSON_VALUE) public class QueryController { - - // /** - // * @param queryLogicName - // * @param queryParameters - // * @return - // */ - // X @POST - // X @Produces({"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", - // "application/x-protostuff"}) - // X @Path("/{logicName}/define") - // X @GZIP - // X @GenerateQuerySessionId(cookieBasePath = "/DataWave/Query/") - // @EnrichQueryMetrics(methodType = MethodType.CREATE) - // X @Interceptors({RequiredInterceptor.class, ResponseInterceptor.class}) - // X @Timed(name = "dw.query.defineQuery", absolute = true) - // X public GenericResponse defineQuery(@Required("logicName") @PathParam("logicName") String queryLogicName, - // X MultivaluedMap queryParameters, @Context HttpHeaders httpHeaders) - // NOTE: The goal is to not use this, but it's here if we need it. - @RolesAllowed({"AuthorizedUser", "AuthorizedServer", "InternalUser", "Administrator"}) - @GenerateQuerySessionId(cookieBasePath = "/DataWave/Query/") - @Timed(name = "dw.query.defineQuery", absolute = true) // TODO: Figure out where this is used + + @Timed(name = "dw.query.defineQuery", absolute = true) + @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE) @RequestMapping(path = "{logicName}/define", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public BaseQueryResponse define(@PathVariable(name = "logicName") String logicName, @RequestParam MultiValueMap parameters) { - // QuerySessionIdContext.setQueryId("some-query-id"); - // GenericResponse resp = new GenericResponse<>(); - // resp.setResult("something something"); - // return resp; - BaseQueryResponse bqr = new BaseQueryResponse() { - - }; - - return bqr; + public GenericResponse define(@PathVariable(name = "logicName") String logicName, @RequestParam MultiValueMap parameters) { + // validate query + + // persist the query w/ query id in the query cache + + // query tracing? + + // update query metrics (i.e. DEFINED) + + + + + + GenericResponse resp = new GenericResponse<>(); + resp.setResult("something something"); + return resp; } } diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryLogicConfig.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryLogicConfig.java new file mode 100644 index 00000000..90c36aa5 --- /dev/null +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryLogicConfig.java @@ -0,0 +1,18 @@ +package datawave.microservice.query.config; + +import org.springframework.validation.annotation.Validated; + +// TODO: This will be a common property file used by multiple services, so we will eventually need to move it out +@Validated +public class QueryLogicConfig { + + private boolean metricsEnabled = false; + + public boolean isMetricsEnabled() { + return metricsEnabled; + } + + public void setMetricsEnabled(boolean metricsEnabled) { + this.metricsEnabled = metricsEnabled; + } +} diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryProperties.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryProperties.java new file mode 100644 index 00000000..27e3ecd5 --- /dev/null +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryProperties.java @@ -0,0 +1,26 @@ +package datawave.microservice.query.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +import javax.validation.Valid; +import java.util.Map; + +// TODO: This may be a common property file used by multiple services, so we will eventually need to move it out +@Validated +@EnableConfigurationProperties(QueryProperties.class) +@ConfigurationProperties(prefix = "query") +public class QueryProperties { + + @Valid + private Map logic; + + public Map getLogic() { + return logic; + } + + public void setLogic(Map logic) { + this.logic = logic; + } +} diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java index cda65171..24b5eeef 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java @@ -1,20 +1,9 @@ package datawave.microservice.query.config; -import datawave.microservice.query.web.filter.BaseMethodStatsInterceptor; -import org.springframework.context.annotation.Bean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Configuration; -import org.springframework.web.servlet.config.annotation.InterceptorRegistry; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; +@EnableConfigurationProperties(QueryProperties.class) @Configuration public class QueryServiceConfig { - @Bean - public WebMvcConfigurer BaseMethodStatsInterceptorConfigurer(BaseMethodStatsInterceptor baseMethodStatsInterceptor) { - return new WebMvcConfigurer() { - @Override - public void addInterceptors(InterceptorRegistry registry) { - registry.addInterceptor(baseMethodStatsInterceptor); - } - }; - } } diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java new file mode 100644 index 00000000..12ce29b7 --- /dev/null +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java @@ -0,0 +1,13 @@ +package datawave.microservice.query.web; + +import datawave.webservice.query.metric.BaseQueryMetric; +import org.springframework.stereotype.Component; + +// TODO: Remove this placeholder and replace it with the real thing once it's ready. +@Component +public class QueryMetrics { + + public void updateMetric(BaseQueryMetric baseQueryMetric) { + System.out.println(baseQueryMetric); + } +} diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java index f8cea1c3..707d6452 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java @@ -70,7 +70,7 @@ public Object beforeBodyWrite(Object body, @NonNull MethodParameter returnType, return body; } finally { - QuerySessionIdContext.removeQueryId(); + QuerySessionIdContext.remove(); } } @@ -90,7 +90,7 @@ public static void setQueryId(String queryId) { QuerySessionIdContext.queryId.set(queryId); } - private static void removeQueryId() { + private static void remove() { queryId.remove(); } } diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java new file mode 100644 index 00000000..e188854d --- /dev/null +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java @@ -0,0 +1,18 @@ +package datawave.microservice.query.web.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target({ElementType.TYPE, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface EnrichQueryMetrics { + enum MethodType { + NONE, CREATE, NEXT, CREATE_AND_NEXT + }; + + MethodType methodType(); +} diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java index 377a41c8..25668a65 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java @@ -1,7 +1,7 @@ package datawave.microservice.query.web.filter; +import org.springframework.http.MediaType; import org.springframework.lang.NonNull; -import org.springframework.stereotype.Component; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.filter.OncePerRequestFilter; @@ -15,13 +15,18 @@ import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponseWrapper; import java.io.IOException; +import java.util.Arrays; import java.util.Enumeration; +import java.util.Map; +import java.util.concurrent.TimeUnit; import static datawave.microservice.config.web.Constants.REQUEST_LOGIN_TIME_ATTRIBUTE; import static datawave.microservice.config.web.Constants.REQUEST_START_TIME_NS_ATTRIBUTE; -@Component -public class BaseMethodStatsFilter extends OncePerRequestFilter { +public abstract class BaseMethodStatsFilter extends OncePerRequestFilter { + + private static final String START_NS_ATTRIBUTE = "STATS_START_NS"; + private static final String STOP_NS_ATTRIBUTE = "STATS_STOP_NS"; protected static class RequestMethodStats { private String uri; @@ -90,64 +95,122 @@ public MultiValueMap getResponseHeaders() { } - @Override - public void initFilterBean() throws ServletException { - System.out.println("init"); - } - @Override public void doFilterInternal(@NonNull HttpServletRequest request, @NonNull HttpServletResponse response, @NonNull FilterChain chain) throws IOException, ServletException { - RequestMethodStats stats = new RequestMethodStats(); - stats.uri = request.getRequestURI(); - stats.method = request.getMethod(); + preProcess(request, response); - stats.callStartTime = System.nanoTime(); + if (!(response instanceof CountingHttpServletResponseWrapper)) { + response = new CountingHttpServletResponseWrapper(response); + } + + chain.doFilter(request, response); + postProcess(request, response); + } + + public void preProcess(HttpServletRequest request, HttpServletResponse response) { + if (BaseMethodStatsContext.getRequestStats() == null) { + BaseMethodStatsContext.setRequestStats(createRequestMethodStats(request, response)); + + long start = System.nanoTime(); + request.setAttribute(START_NS_ATTRIBUTE, start); + } + preProcess(BaseMethodStatsContext.getRequestStats()); + } + + public void preProcess(RequestMethodStats requestStats) { + // do nothing + } + + public void postProcess(HttpServletRequest request, HttpServletResponse response) { + if (BaseMethodStatsContext.getResponseStats() == null) { + long stop = System.nanoTime(); + request.setAttribute(STOP_NS_ATTRIBUTE, stop); + + BaseMethodStatsContext.setResponseStats(createResponseMethodStats(request, response)); + } + postProcess(BaseMethodStatsContext.getResponseStats()); + } + + public void postProcess(ResponseMethodStats responseStats) { + // do nothing + } + + protected RequestMethodStats createRequestMethodStats(HttpServletRequest request, HttpServletResponse response) { + RequestMethodStats requestStats = new RequestMethodStats(); + + requestStats.uri = request.getRequestURI(); + requestStats.method = request.getMethod(); + + requestStats.callStartTime = System.nanoTime(); try { - stats.callStartTime = (long) request.getAttribute(REQUEST_START_TIME_NS_ATTRIBUTE); + requestStats.callStartTime = (long) request.getAttribute(REQUEST_START_TIME_NS_ATTRIBUTE); } catch (Exception e) { // do nothing } try { - stats.loginTime = (long) request.getAttribute(REQUEST_LOGIN_TIME_ATTRIBUTE); + requestStats.loginTime = (long) request.getAttribute(REQUEST_LOGIN_TIME_ATTRIBUTE); } catch (Exception e) { // do nothing } - + for (Enumeration headerNames = request.getHeaderNames(); headerNames.hasMoreElements();) { String header = headerNames.nextElement(); - for (Enumeration headerValues = request.getHeaders(header); headerValues.hasMoreElements();){ - stats.requestHeaders.add(header, headerValues.nextElement()); + for (Enumeration headerValues = request.getHeaders(header); headerValues.hasMoreElements();) { + requestStats.requestHeaders.add(header, headerValues.nextElement()); } } - - // TODO: Finish this! -// MediaType mediaType = request.getMediaType(); -// if (mediaType != null && mediaType.isCompatible(MediaType.APPLICATION_FORM_URLENCODED_TYPE)) { -// MultivaluedMap formParameters = request.getHttpRequest().getDecodedFormParameters(); -// if (formParameters != null) -// MultivaluedTreeMap.addAll(formParameters, stats.formParameters); -// } - // request.setProperty(REQUEST_STATS_NAME, stats); - // return stats; - - System.out.println("doFilter before"); - if (response instanceof HttpServletResponse) { - chain.doFilter(request, new CountingHttpServletResponseWrapper((HttpServletResponse) response)); - } else { - chain.doFilter(request, response); - } - System.out.println("doFilter after"); + + if (request.getContentType() != null && MediaType.parseMediaType(request.getContentType()).isCompatibleWith(MediaType.APPLICATION_FORM_URLENCODED)) { + Map formParameters = request.getParameterMap(); + if (formParameters != null) { + formParameters.forEach((k, v) -> requestStats.formParameters.addAll(k, Arrays.asList(v))); + } + } + + return requestStats; + } + + private ResponseMethodStats createResponseMethodStats(HttpServletRequest request, HttpServletResponse response) { + ResponseMethodStats responseStats = new ResponseMethodStats(); + + long start = System.nanoTime(); + try { + start = (long) request.getAttribute(START_NS_ATTRIBUTE); + } catch (Exception e) { + // do nothing + } + + long stop = System.nanoTime(); + try { + stop = (long) request.getAttribute(STOP_NS_ATTRIBUTE); + } catch (Exception e) { + // do nothing + } + + responseStats.serializationTime = TimeUnit.NANOSECONDS.toMillis(stop - start); + responseStats.loginTime = BaseMethodStatsContext.getRequestStats().getLoginTime(); + responseStats.callTime = TimeUnit.NANOSECONDS.toMillis(stop - BaseMethodStatsContext.getRequestStats().getCallStartTime()); + + if (response instanceof CountingHttpServletResponseWrapper) { + responseStats.bytesWritten = ((CountingHttpServletResponseWrapper) response).getByteCount(); + } + + for (String header : response.getHeaderNames()) { + responseStats.responseHeaders.add(header, response.getHeaders(header)); + } + + return responseStats; } @Override public void destroy() { - System.out.println("destroy"); + BaseMethodStatsContext.remove(); } private static class CountingHttpServletResponseWrapper extends HttpServletResponseWrapper { - private ServletResponse response; + private final ServletResponse response; private CountingServletOutputStream cos; /** @@ -210,4 +273,31 @@ public long getByteCount() { return count; } } + + public static class BaseMethodStatsContext { + + private static final ThreadLocal requestStats = new ThreadLocal<>(); + private static final ThreadLocal responseStats = new ThreadLocal<>(); + + public static RequestMethodStats getRequestStats() { + return requestStats.get(); + } + + public static void setRequestStats(RequestMethodStats requestStats) { + BaseMethodStatsContext.requestStats.set(requestStats); + } + + public static ResponseMethodStats getResponseStats() { + return responseStats.get(); + } + + public static void setResponseStats(ResponseMethodStats responseStats) { + BaseMethodStatsContext.responseStats.set(responseStats); + } + + private static void remove() { + requestStats.remove(); + responseStats.remove(); + } + } } diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsInterceptor.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsInterceptor.java deleted file mode 100644 index c51e0a50..00000000 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsInterceptor.java +++ /dev/null @@ -1,100 +0,0 @@ -package datawave.microservice.query.web.filter; - -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; -import org.springframework.web.servlet.ModelAndView; -import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -@Component -public class BaseMethodStatsInterceptor extends HandlerInterceptorAdapter { - protected static final Logger log = Logger.getLogger(BaseMethodStatsInterceptor.class); - - protected static class RequestMethodStats { - - private String uri; - private String method; - private long loginTime = -1; - private long callStartTime; - private MultiValueMap requestHeaders = new LinkedMultiValueMap<>(); - private MultiValueMap formParameters = new LinkedMultiValueMap<>(); - - public String getUri() { - return uri; - } - - public String getMethod() { - return method; - } - - public long getLoginTime() { - return loginTime; - } - - public long getCallStartTime() { - return callStartTime; - } - - public MultiValueMap getRequestHeaders() { - return requestHeaders; - } - - public MultiValueMap getFormParameters() { - return formParameters; - } - } - - protected static class ResponseMethodStats { - private int statusCode = -1; - private long loginTime = -1; - private long callTime = -1; - private long serializationTime = -1; - private long bytesWritten = -1; - private MultiValueMap responseHeaders = new LinkedMultiValueMap<>(); - - public int getStatusCode() { - return statusCode; - } - - public long getLoginTime() { - return loginTime; - } - - public long getCallTime() { - return callTime; - } - - public long getSerializationTime() { - return serializationTime; - } - - public long getBytesWritten() { - return bytesWritten; - } - - public MultiValueMap getResponseHeaders() { - return responseHeaders; - } - - } - - @Override - public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception { - System.out.println("pre handle"); - return true; - } - - @Override - public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler, ModelAndView modelAndView) throws Exception { - System.out.println("post handle"); - } - - @Override - public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) throws Exception { - System.out.println("after completion"); - } -} diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java new file mode 100644 index 00000000..0840d92b --- /dev/null +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java @@ -0,0 +1,80 @@ +package datawave.microservice.query.web.filter; + +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; +import org.springframework.util.MultiValueMap; + +import java.util.List; +import java.util.Map; + +@Component +public class LoggingStatsFilter extends BaseMethodStatsFilter { + private final Logger log = Logger.getLogger(this.getClass()); + + @Override + public void preProcess(RequestMethodStats requestStats) { + if (!log.isTraceEnabled() && requestStats != null) { + StringBuilder message = new StringBuilder(); + message.append(" URI: ").append(requestStats.getUri()); + message.append(" Method: ").append(requestStats.getMethod()); + message.append(" Request Headers {"); + for (Map.Entry> header : requestStats.getRequestHeaders().entrySet()) { + message.append(" ").append(header.getKey()).append(" -> "); + String sep = ""; + for (Object o : header.getValue()) { + message.append(sep).append(o); + sep = ","; + } + } + message.append("}"); + message.append(" Form Parameters {"); + try { + MultiValueMap formParams = requestStats.getFormParameters(); + if (formParams == null || formParams.isEmpty()) { + message.append(" None "); + } else { + for (Map.Entry> header : formParams.entrySet()) { + message.append(" ").append(header.getKey()).append(" -> "); + String sep = ""; + for (Object o : header.getValue()) { + message.append(sep).append(o); + sep = ","; + } + } + } + } catch (NullPointerException npe) { + log.warn("Unable to log request due to NPE"); + } catch (Exception e) { + if (null != e.getMessage()) + log.warn("Unable to log request due to error: " + e.getMessage()); + else + log.warn("Unable to log request due to error", e); + } + message.append("}"); + log.trace(message.toString()); + } + } + + @Override + public void postProcess(ResponseMethodStats responseStats) { + if (!log.isTraceEnabled() && responseStats != null) { + StringBuilder message = new StringBuilder(); + message.append(" Post Process: StatusCode: ").append(responseStats.getStatusCode()); + message.append(" Response Headers {"); + for (Map.Entry> header : responseStats.getResponseHeaders().entrySet()) { + message.append(" ").append(header.getKey()).append(" -> "); + String sep = ""; + for (Object o : header.getValue()) { + message.append(sep).append(o); + sep = ","; + } + } + message.append("} Serialization time: ").append(responseStats.getSerializationTime()).append("ms"); + message.append(" Bytes written: ").append(responseStats.getBytesWritten()); + message.append(" Login Time: ").append(responseStats.getLoginTime()).append("ms"); + message.append(" Call Time: ").append(responseStats.getCallTime()).append("ms"); + + log.trace(message); + } + } +} diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java new file mode 100644 index 00000000..9d1e9839 --- /dev/null +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java @@ -0,0 +1,174 @@ +package datawave.microservice.query.web.filter; + +import datawave.microservice.common.storage.QueryCache; +import datawave.microservice.common.storage.QueryState; +import datawave.microservice.query.config.QueryProperties; +import datawave.microservice.query.web.QueryMetrics; +import datawave.microservice.query.web.annotation.EnrichQueryMetrics; +import datawave.webservice.query.metric.BaseQueryMetric; +import datawave.webservice.query.metric.QueryMetric; +import datawave.webservice.result.BaseQueryResponse; +import datawave.webservice.result.GenericResponse; +import org.apache.log4j.Logger; +import org.springframework.core.MethodParameter; +import org.springframework.http.MediaType; +import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.http.server.ServerHttpRequest; +import org.springframework.http.server.ServerHttpResponse; +import org.springframework.lang.NonNull; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyAdvice; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.UUID; + +@ControllerAdvice +public class QueryMetricsEnrichmentFilterAdvice extends BaseMethodStatsFilter implements ResponseBodyAdvice { + + private final Logger log = Logger.getLogger(this.getClass()); + + private final QueryProperties queryProperties; + + private final QueryCache queryCache; + + private final QueryMetrics queryMetrics; + + public QueryMetricsEnrichmentFilterAdvice(QueryProperties queryProperties, QueryCache queryCache, QueryMetrics queryMetrics) { + this.queryProperties = queryProperties; + this.queryCache = queryCache; + this.queryMetrics = queryMetrics; + } + + @Override + public boolean supports(MethodParameter returnType, @NonNull Class> converterType) { + boolean supports = false; + EnrichQueryMetrics annotation = (EnrichQueryMetrics) Arrays.stream(returnType.getMethodAnnotations()).filter(EnrichQueryMetrics.class::isInstance).findAny().orElse(null); + if (annotation != null) { + try { + Class returnClass = Objects.requireNonNull(returnType.getMethod()).getReturnType(); + if (GenericResponse.class.isAssignableFrom(returnClass)) { + supports = true; + QueryMetricsEnrichmentContext.setMethodType(annotation.methodType()); + } else if (BaseQueryResponse.class.isAssignableFrom(returnClass)) { + supports = true; + QueryMetricsEnrichmentContext.setMethodType(annotation.methodType()); + } else { + log.error("Unexpected response class for metrics annotated query method " + returnType.getMethod().getName() + ". Response class was " + + returnClass.toString()); + } + } catch (NullPointerException e) { + // do nothing + } + } + + return supports; + } + + @Override + public Object beforeBodyWrite(Object body, @NonNull MethodParameter returnType, @NonNull MediaType selectedContentType, + @NonNull Class> selectedConverterType, @NonNull ServerHttpRequest request, + @NonNull ServerHttpResponse response) { + System.out.println("QueryMetricsEnrichmentFilterAdvice writing metrics"); + + if (body instanceof GenericResponse) { + @SuppressWarnings({"unchecked", "rawtypes"}) + GenericResponse genericResponse = (GenericResponse) body; + QueryMetricsEnrichmentContext.setQueryId(genericResponse.getResult()); + } else if (body instanceof BaseQueryResponse) { + BaseQueryResponse baseResponse = (BaseQueryResponse) body; + QueryMetricsEnrichmentContext.setQueryId(baseResponse.getQueryId()); + } + + return body; + } + + @Override + public void postProcess(ResponseMethodStats responseStats) { + String queryId = QueryMetricsEnrichmentContext.getQueryId(); + if (queryId != null) { + if (queryCache != null) { + QueryState queryState = queryCache.getQuery(UUID.fromString(queryId)); + if (queryState != null && queryProperties.getLogic().get(queryState.getQueryLogic()).isMetricsEnabled()) { + try { + // TODO: This probably shouldn't be instantiated here, and we also shouldn't hard code the basequerymetric implementation. + // just using this as a placeholder until we start to bring things together. + BaseQueryMetric metric = new QueryMetric(); + switch (QueryMetricsEnrichmentContext.getMethodType()) { + case CREATE: + metric.setCreateCallTime(responseStats.getCallTime()); + metric.setLoginTime(responseStats.getLoginTime()); + break; + case CREATE_AND_NEXT: + metric.setCreateCallTime(responseStats.getCallTime()); + metric.setLoginTime(responseStats.getLoginTime()); + List pageTimes = metric.getPageTimes(); + if (pageTimes != null && !pageTimes.isEmpty()) { + BaseQueryMetric.PageMetric pm = pageTimes.get(pageTimes.size() - 1); + pm.setCallTime(responseStats.getCallTime()); + pm.setLoginTime(responseStats.getLoginTime()); + pm.setSerializationTime(responseStats.getSerializationTime()); + pm.setBytesWritten(responseStats.getBytesWritten()); + } + break; + case NEXT: + pageTimes = metric.getPageTimes(); + if (pageTimes != null && !pageTimes.isEmpty()) { + BaseQueryMetric.PageMetric pm = pageTimes.get(pageTimes.size() - 1); + pm.setCallTime(responseStats.getCallTime()); + pm.setLoginTime(responseStats.getLoginTime()); + pm.setSerializationTime(responseStats.getSerializationTime()); + pm.setBytesWritten(responseStats.getBytesWritten()); + } + break; + } + + if (queryMetrics != null) + queryMetrics.updateMetric(metric); + else + log.error("QueryMetricsBean JNDI lookup returned null"); + } catch (Exception e) { + log.error("Unable to record metrics for " + QueryMetricsEnrichmentContext.getMethodType() + " method: " + e.getLocalizedMessage(), e); + } + } else { + log.error("RunningQuery instance not in the cache!, queryId: " + queryId); + } + } else { + log.error("Query cache not injected! No metrics will be recorded for serialization times."); + } + } + } + + @Override + public void destroy() { + super.destroy(); + QueryMetricsEnrichmentContext.remove(); + } + + public static class QueryMetricsEnrichmentContext { + + private static final ThreadLocal queryId = new ThreadLocal<>(); + private static final ThreadLocal methodType = new ThreadLocal<>(); + + public static String getQueryId() { + return queryId.get(); + } + + public static void setQueryId(String queryId) { + QueryMetricsEnrichmentContext.queryId.set(queryId); + } + + public static EnrichQueryMetrics.MethodType getMethodType() { + return methodType.get(); + } + + public static void setMethodType(EnrichQueryMetrics.MethodType methodType) { + QueryMetricsEnrichmentContext.methodType.set(methodType); + } + + private static void remove() { + queryId.remove(); + } + } +} From 8dd51312122583c19e54a23bf02fea6a6b0561d3 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Mon, 1 Mar 2021 20:19:37 -0500 Subject: [PATCH 016/218] added initial docker compose with sample configs and reconfigured query-cache to not be a spring boot app. --- .../microservice/query/QueryController.java | 20 ++++---- .../query/config/QueryLogicConfig.java | 6 +-- .../query/config/QueryProperties.java | 12 ++--- .../query/config/QueryServiceConfig.java | 3 +- .../microservice/query/web/QueryMetrics.java | 2 +- .../QueryMetricsEnrichmentFilterAdvice.java | 49 ++++++++++--------- .../src/test/resources/config/application.yml | 9 ++-- 7 files changed, 50 insertions(+), 51 deletions(-) diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java index 8f43a5c4..23b5fc3e 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java @@ -11,29 +11,27 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import java.util.UUID; + @RestController @RequestMapping(path = "/v1", produces = MediaType.APPLICATION_JSON_VALUE) public class QueryController { - + @Timed(name = "dw.query.defineQuery", absolute = true) @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE) @RequestMapping(path = "{logicName}/define", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public GenericResponse define(@PathVariable(name = "logicName") String logicName, @RequestParam MultiValueMap parameters) { + public GenericResponse define(@PathVariable(name = "logicName") String logicName, @RequestParam MultiValueMap parameters) { // validate query - + // persist the query w/ query id in the query cache - + // query tracing? - + // update query metrics (i.e. DEFINED) - - - - - + GenericResponse resp = new GenericResponse<>(); - resp.setResult("something something"); + resp.setResult(UUID.randomUUID().toString()); return resp; } } diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryLogicConfig.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryLogicConfig.java index 90c36aa5..975704b2 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryLogicConfig.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryLogicConfig.java @@ -5,13 +5,13 @@ // TODO: This will be a common property file used by multiple services, so we will eventually need to move it out @Validated public class QueryLogicConfig { - + private boolean metricsEnabled = false; - + public boolean isMetricsEnabled() { return metricsEnabled; } - + public void setMetricsEnabled(boolean metricsEnabled) { this.metricsEnabled = metricsEnabled; } diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryProperties.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryProperties.java index 27e3ecd5..60022938 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryProperties.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryProperties.java @@ -12,15 +12,15 @@ @EnableConfigurationProperties(QueryProperties.class) @ConfigurationProperties(prefix = "query") public class QueryProperties { - + @Valid - private Map logic; - - public Map getLogic() { + private Map logic; + + public Map getLogic() { return logic; } - - public void setLogic(Map logic) { + + public void setLogic(Map logic) { this.logic = logic; } } diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java index 24b5eeef..4b948a0c 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java @@ -5,5 +5,4 @@ @EnableConfigurationProperties(QueryProperties.class) @Configuration -public class QueryServiceConfig { -} +public class QueryServiceConfig {} diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java index 12ce29b7..129998a5 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java @@ -6,7 +6,7 @@ // TODO: Remove this placeholder and replace it with the real thing once it's ready. @Component public class QueryMetrics { - + public void updateMetric(BaseQueryMetric baseQueryMetric) { System.out.println(baseQueryMetric); } diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java index 9d1e9839..af7a0b25 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java @@ -26,25 +26,26 @@ @ControllerAdvice public class QueryMetricsEnrichmentFilterAdvice extends BaseMethodStatsFilter implements ResponseBodyAdvice { - + private final Logger log = Logger.getLogger(this.getClass()); - + private final QueryProperties queryProperties; - + private final QueryCache queryCache; - + private final QueryMetrics queryMetrics; - + public QueryMetricsEnrichmentFilterAdvice(QueryProperties queryProperties, QueryCache queryCache, QueryMetrics queryMetrics) { this.queryProperties = queryProperties; this.queryCache = queryCache; this.queryMetrics = queryMetrics; } - + @Override public boolean supports(MethodParameter returnType, @NonNull Class> converterType) { boolean supports = false; - EnrichQueryMetrics annotation = (EnrichQueryMetrics) Arrays.stream(returnType.getMethodAnnotations()).filter(EnrichQueryMetrics.class::isInstance).findAny().orElse(null); + EnrichQueryMetrics annotation = (EnrichQueryMetrics) Arrays.stream(returnType.getMethodAnnotations()).filter(EnrichQueryMetrics.class::isInstance) + .findAny().orElse(null); if (annotation != null) { try { Class returnClass = Objects.requireNonNull(returnType.getMethod()).getReturnType(); @@ -56,22 +57,22 @@ public boolean supports(MethodParameter returnType, @NonNull Class> selectedConverterType, @NonNull ServerHttpRequest request, - @NonNull ServerHttpResponse response) { + @NonNull Class> selectedConverterType, @NonNull ServerHttpRequest request, + @NonNull ServerHttpResponse response) { System.out.println("QueryMetricsEnrichmentFilterAdvice writing metrics"); - + if (body instanceof GenericResponse) { @SuppressWarnings({"unchecked", "rawtypes"}) GenericResponse genericResponse = (GenericResponse) body; @@ -80,10 +81,10 @@ public Object beforeBodyWrite(Object body, @NonNull MethodParameter returnType, BaseQueryResponse baseResponse = (BaseQueryResponse) body; QueryMetricsEnrichmentContext.setQueryId(baseResponse.getQueryId()); } - + return body; } - + @Override public void postProcess(ResponseMethodStats responseStats) { String queryId = QueryMetricsEnrichmentContext.getQueryId(); @@ -123,7 +124,7 @@ public void postProcess(ResponseMethodStats responseStats) { } break; } - + if (queryMetrics != null) queryMetrics.updateMetric(metric); else @@ -139,34 +140,34 @@ public void postProcess(ResponseMethodStats responseStats) { } } } - + @Override public void destroy() { super.destroy(); QueryMetricsEnrichmentContext.remove(); } - + public static class QueryMetricsEnrichmentContext { - + private static final ThreadLocal queryId = new ThreadLocal<>(); private static final ThreadLocal methodType = new ThreadLocal<>(); - + public static String getQueryId() { return queryId.get(); } - + public static void setQueryId(String queryId) { QueryMetricsEnrichmentContext.queryId.set(queryId); } - + public static EnrichQueryMetrics.MethodType getMethodType() { return methodType.get(); } - + public static void setMethodType(EnrichQueryMetrics.MethodType methodType) { QueryMetricsEnrichmentContext.methodType.set(methodType); } - + private static void remove() { queryId.remove(); } diff --git a/query-microservices/query/service/src/test/resources/config/application.yml b/query-microservices/query/service/src/test/resources/config/application.yml index d80d73c2..abc257d8 100644 --- a/query-microservices/query/service/src/test/resources/config/application.yml +++ b/query-microservices/query/service/src/test/resources/config/application.yml @@ -1,8 +1,4 @@ spring: - - application: - name: query - autoconfigure: exclude: org.springframework.boot.autoconfigure.web.servlet.error.ErrorMvcAutoConfiguration @@ -16,6 +12,11 @@ spring: hazelcast.client.enabled: false +query: + storage: + backend: + LOCAL + server: port: 0 non-secure-port: 0 From 19c6caa822b1c838d5b9df3789af9bc0d37eb3c5 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Tue, 2 Mar 2021 10:58:20 -0500 Subject: [PATCH 017/218] Added sample configs for the query and query-storage microservices. --- .../service/src/test/resources/ssl/host.p12 | Bin 2500 -> 0 bytes .../service/src/test/resources/ssl/rootCA.p12 | Bin 2644 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 query-microservices/query/service/src/test/resources/ssl/host.p12 delete mode 100644 query-microservices/query/service/src/test/resources/ssl/rootCA.p12 diff --git a/query-microservices/query/service/src/test/resources/ssl/host.p12 b/query-microservices/query/service/src/test/resources/ssl/host.p12 deleted file mode 100644 index 5c4a2d66175ca22bfd8d12634c885cd3489d6b8b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2500 zcmY+Ec{mh`8pdbF491>jn4G~uifl8MqcB+_*_SjDlED}x+n|tTY(@4VyP2{Rl_jZc zA^RGQWvp2v+mW5C=ehTud;fT!?|a_&d%pkPABqHR009|LBxnR9^isr)h#hty6OclJ z`hrPNuT$F%MPfMlCt{?K7}ig%6%Y_`I_CaKfG8v*%fCOc0U1#oU}n62Qan~2i3EX| z0l_4Ong;A(35RNJavkrqI3=AXRH*`tPYT-K#C!M;g^aYH75{ zfdj}_q$Wf|#Qix15_~)Jfll}?n`C5Jyz6V8KbbvIof^4m&+%Y@o9S7WODov+ zZfyC$W&ws?b3XSlzvuyAiR(vutGn;xJHrHL$fRq2vs4ZTt^T!w_^6O|x^`k@Mw^WD zdaM!SIimSDNq+?jt$-b!$NK`JlGt*eby%%`(-?{4ntWN?c#gelJstv_NY(Nj7_$pJ zYb^k8-u=-{%rrk0p+nC#FOKcKX)$~2ODC#Kxtc= zV&;?0kLT9M(=sJfXoh+$`uarqurjs`v_zn6w&_qklY5qw?l}#6B_o50MQHJ`vOc%> zjImoS`{P$zQ9?e0*o7+w-$AEEhYful)=1-3YVT4^&!1GwDVMLTI%|^isD5vj`3+tR zvjP;Ye?6R4JN{8_G9Ffe+y8uU&ohX!e*0_NtoM2QFla8>na*A?J&-(RW~OJ$LsX^N z&+V<`_t!)#>+UP7C8f*W*Lkn9XeQ`)P`ql zJGX0I9zini2oHDU&m+1-NnFv5OKPHeIm+Qh3YE$)$o-P!_i)PhpN@W03TXWC+q#~hiMB%fO+iS&od~Os7kwDU#|~G1EDt()H3L z^>W4BnIZ-qTUy8x0=IHukzqDZ!;n|~9S*FZUDghu8J20|5QC3PN}lEWIycgn1=mWY&{tfkwa1h1~>Smezr7Nk&Cfr}iKIG;)XHAcOQylwKEcM&f(D|~qzVQjt zJ^QxT4=dj#YWGbpAvGWwogzWSbsW4Dvk2}7&cGIYkq*zJmcsmkg)=4J=lA}=h%w{} ztAufD$r%;%>OY;JSNeWdY{cZCIBgm_Z$HoO?0J(cc;F)tekJDHjMFcY-ja3RTH4eI z=wrf@L)Vr$L%nuW8Qf1|?XhU}`OM(E5*M`*&y)_lEY{NP2)0)q`|*2O-w(}1m^P8k z^7a}!A_ZQ`$jmpnldc0TeOq*+gX_{-jos*Et?XrbMoOp45`WU$Bk$CJ8wa5R~tIXw(=VpFXLNUIQ4Dlzb0+6zCU!wPSt$lhVi(99LLcT2i zq2e!OfvU;oWEw<`JB3TsKUA@~*m{)HE%wMB_IBlGO8NYMkfIP|%E6L_~@?tM3OUF?s;h<{dCDvIWH* z-Gx005Wky!O1eus= zInU+(n7U?<3`{B`j*^_;mAaI}ez?{TYNhG6!4akdT>Tj1xHJ)JE7wL8Dy+N8_HtV! zZ^7k4q;BqTMhWjn?bc(bp8PUuzBl4}viU(+5jr7FR`8ls_*k&*=j>6LzU_&vq(B{o zZVjIRowDNYiy?Mjdp-ChI-+DuR*hp`eq%_`AP~7+Z9EkD=GSmjbyrQPOX}Nt;&xm? z%Z+?_hX)zhJ2r8_Gx1W%rJ*<20*4D4j+xMX7ee1KnFbbtWnw$UQW2rGj77?$@CmB7 z)z!jB{(spjbUtvjeER4Lgw|;u9nUj&6Uos7kJ)F7s5LQc_>kKx&S~P;Iti1t@=Xf5 zm^g=E%dHT{>j_0!Y;8_!I-X6(WSCEK@JSes>Uxp|5D zS+!;}-|rIHwIw(zILe4@t??8;m*h>`cB-lVKv;|RY}c=XUhvqSiZ1NXK9`+AMC+qO z{!2$F9L)oUF$6+ql8dUMu-Fl1Bed-0K?8SopevxJC51l-O;VA@YIQl>PRqi;DvdK_TlcEzhU|> DziXlX diff --git a/query-microservices/query/service/src/test/resources/ssl/rootCA.p12 b/query-microservices/query/service/src/test/resources/ssl/rootCA.p12 deleted file mode 100644 index 8c071083ddd7f5973571edbb60ef237f3930bec9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2644 zcmY+^cQhN0762dL zC1LxDk{ts)-N)mQ(xJ)eZRt5`dF7XenBl>85N6za>OGpT_@ky zGcJgfYhN7DcH#ECu$R~<<31o48|Z16nn!)M6n@s-b*hQRXYSDz`K~OnM3fPNYmSI! zZyY_1s!rQ+o!k7IQAk*_gB}CCSSYJNiHQz!y7yJC6J%K|T6vXaoYH2xGlwh{lm0Sc zzFXD5lS*Ils)HW%S=BSJf_CzqCPSCmdC=TgWlCgxr(!P}T5HO0hzpkl&xiCrNY5u| zbv_ZW-*VeH>!UPE8AxgSoP&N*QSN@J)CgJ~{;G4;2*+)`&^&3*o-xVjHK=8<8eu&= ze!xwZbZm9N$7?@e)6^8IM^LC0pA?}f6tKswuIvxjqvfSom80m}i*c*-n7KDQFo9!! zssuj{)}UPI?$2A6E0xyn?KU{OeF4W*mD=RO{uymhRI)EuZ%pi=JD>T z%lKZg+70rZNq3luVzlQ;oORAsSIc2961&i(89Zbl?EG_XNnRMW6R{K9snc7NYXvvk8WT2c&+4*HyiJ|aMm%r+9qcwx!hn^@NFbatz9sGy zB;_l=RQ-6?3f+CIm4LQcEO8kKC#2vqB|>+bg(W&4VKblWPBp>|etj>$AZCbG!Q#_n zyHxCDu8iZx+#&qATVmwE=BLP94mD4<(u-WaZ#-Aaqc0iknZ-XyK6oxDyB@2n=vQYx z`6)Gfa}fcOEe>tK@Od1b@$XOnh>au|h)87l28F$6ca>T@KdW)WL1O@pxUv_seHNtj zZjyr4xJ|!LW12>N)8N9oaa85mUfc3_p>SLtT zl}PXL_SILx{zRjA>~!$hX|+pq3^S)>h7x^3=Uvm-%AnlkwF+*pt_6(0>ZDs?AqCp$ znewHM7OQJ@B(Uu5x-lJasQ%KYQ%I-hngX{{Y|xqK$Ad42LU8)k1~RaSP>G!v0Xx)Q z-oQPUAD`pSZ5>NTU?!%%?~C1kf0Yol)oQ|eRI_Ys$$5pRu)`}b;e$U7Oq~8!#09}a z4u}#QL6jAG{i-nz=!DP|WT>SMo!7Wi7m~zlCV+dyrYNE4yzHpnsq4Hj@pR2ks;x_g z%L?-gs41vCj9)tE?bH7`0P9V3nXe7$lkX}SnqAD6P%MaFH{2F?H=<3HV`K`z_vq8- zQY^Le29*z)P-Avnht4jo{fYr&1b(n@=7Rohw5IyheE2nAqy;lP9KcWHr{QGPZ-qmi4k)ikuz!Z}fPL${R7t;U#DU4GkS$De`GzSsEoVQ1I8X-zy?p zUs>B|ea*pxY(iLaHb1^*kF6pCCy?>93vB9v9c+qh?jKe7@v%cgvf}ediic3(P6)21R-nPiiTYDee1yZ^TUSK+tIhUi# zb?a=b9hajw?52J>2-v_4J4zZ(KLLM`%yv2--Z8K6?$#8!o(4Sp_DVpn=8o@AZ_L%0YFJQMA< zzX?D;aP$viO{mY0anTB)tO&Gkev>?TWd7kPlMOq>ee4agx1z{q8@P5C>F@LO<(d8} zXYo{a2d68S-|ETluZ(1)*DP4RCjTCBg{+QC&{!UOtVwT{$AJ+1=oxa<=kAtaNl0dO z+#@&LOn1C{p{mKq4J)Bg>&W@G+j^PDbh!~)@H44(gNpcdOTJ6@F7Bo1T5`6Uxm8%S z(JPr2D;aIbN`0E~;N)*@G-~any|aP;Eitz3Vw2*Y z^H=5iwLq-0Y8G+N*b4VpG)Zwn=5+;9^^4wm&um_iv;|!rgA)C(j;FZfxj`_Ioy_#W z)!7$9GMJ(L0GZwoTJ29!V*eb$1DnH^B_C=t{tW+rJ4 sMj#&(00b-G+U}h_z0diR>7&!_Yi6t6iw0UCu=k_f4%>W4^&d?C1tW*xVE_OC From 10f15b4b044687f1cf5c9e063ef73860447d2527 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 4 Mar 2021 19:51:23 -0500 Subject: [PATCH 018/218] Fixed the query service test error, and removed unnecessary test certs. --- .../service/src/test/resources/config/application.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/query-microservices/query/service/src/test/resources/config/application.yml b/query-microservices/query/service/src/test/resources/config/application.yml index abc257d8..38d7ac87 100644 --- a/query-microservices/query/service/src/test/resources/config/application.yml +++ b/query-microservices/query/service/src/test/resources/config/application.yml @@ -7,8 +7,9 @@ spring: datawave: jwt.ttl: 3600 issuers-required: true + enforce-allowed-callers: false allowed-callers: - - "cn=test.testcorp.com, ou=microservices, ou=development, o=testcorp, c=us" + - "cn=test keystore, ou=my department, o=my company, st=some-state, c=us" hazelcast.client.enabled: false @@ -23,12 +24,12 @@ server: servlet.context-path: /query ssl: client-auth: NEED - trust-store: classpath:ssl/rootCA.p12 + trust-store: 'classpath:testCA.p12' trust-store-type: PKCS12 - trust-store-password: LetMeIn - key-store: classpath:ssl/host.p12 + trust-store-password: 'ChangeIt' + key-store: 'classpath:testServer.p12' key-store-type: PKCS12 - key-store-password: LetMeIn + key-store-password: 'ChangeIt' outbound-ssl: key-store: ${server.ssl.key-store} key-store-password: ${server.ssl.key-store-password} From 88fd2e53c355358e943dccdf0a235fcce1c2821d Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 5 Mar 2021 10:40:07 -0500 Subject: [PATCH 019/218] Fixed a bug where the query storage cache would complain about multiple messageHandlerMethodFactory beans being present. --- .../service/src/main/resources/{log4j2.xml => log4j2.yml} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename query-microservices/query/service/src/main/resources/{log4j2.xml => log4j2.yml} (86%) diff --git a/query-microservices/query/service/src/main/resources/log4j2.xml b/query-microservices/query/service/src/main/resources/log4j2.yml similarity index 86% rename from query-microservices/query/service/src/main/resources/log4j2.xml rename to query-microservices/query/service/src/main/resources/log4j2.yml index 2f5ddd6c..9fdbc6bf 100644 --- a/query-microservices/query/service/src/main/resources/log4j2.xml +++ b/query-microservices/query/service/src/main/resources/log4j2.yml @@ -21,8 +21,8 @@ Configuration: RollingFile: - name: File - fileName: "${sys:logDir}/query-api-service.log" - filePattern: "${sys:logDir}/query-api-service.log.%d{yyyy-MM-dd}-%i.gz" + fileName: "${sys:logDir}/query-service.log" + filePattern: "${sys:logDir}/query-service.log.%d{yyyy-MM-dd}-%i.gz" append: true bufferedIO: true bufferSize: 8192 From 980c3c36b26390fb7f58a2762485bec79cb0f5f1 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 10 Mar 2021 15:41:53 -0500 Subject: [PATCH 020/218] wip --- query-microservices/query/api/pom.xml | 19 ++- .../microservice/query/QueryParameters.java | 91 +++++++++++ .../microservice/query/QueryPersistence.java | 7 + query-microservices/query/service/pom.xml | 10 ++ .../microservice/query/QueryController.java | 14 +- .../microservice/query/QueryManagement.java | 152 ++++++++++++++++++ .../microservice/query/util/QueryUtil.java | 86 ++++++++++ 7 files changed, 372 insertions(+), 7 deletions(-) create mode 100644 query-microservices/query/api/src/main/java/datawave/microservice/query/QueryParameters.java create mode 100644 query-microservices/query/api/src/main/java/datawave/microservice/query/QueryPersistence.java create mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/QueryManagement.java create mode 100644 query-microservices/query/service/src/main/java/datawave/microservice/query/util/QueryUtil.java diff --git a/query-microservices/query/api/pom.xml b/query-microservices/query/api/pom.xml index 97818bef..c0bb65b1 100644 --- a/query-microservices/query/api/pom.xml +++ b/query-microservices/query/api/pom.xml @@ -9,11 +9,24 @@ query-api 1.0-SNAPSHOT - + + 1.2 + - + + + gov.nsa.datawave.microservice + base-rest-responses + ${version.microservice.base-rest-responses} + + - + + + gov.nsa.datawave.microservice + base-rest-responses + + + + true + + + false + + datawave-github-mvn-repo + https://raw.githubusercontent.com/NationalSecurityAgency/datawave/mvn-repo + + + diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/GenericQueryConfiguration.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/GenericQueryConfiguration.java new file mode 100644 index 00000000..ba176589 --- /dev/null +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/GenericQueryConfiguration.java @@ -0,0 +1,211 @@ +package datawave.microservice.query.configuration; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.collect.Iterators; +import datawave.microservice.query.logic.BaseQueryLogic; +import datawave.util.TableName; +import datawave.webservice.query.Query; +import org.apache.accumulo.core.client.BatchScanner; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.security.Authorizations; + +import javax.xml.bind.annotation.XmlTransient; +import java.util.Collections; +import java.util.Date; +import java.util.Iterator; +import java.util.Set; + +/** + *

+ * A basic query configuration object that contains the information needed to run a query. + *

+ * + *

+ * Provides some "expected" default values for parameters. This configuration object also encapsulates iterators and their options that would be set on a + * {@link BatchScanner}. + *

+ * + */ +public abstract class GenericQueryConfiguration { + private Connector connector = null; + private Set authorizations = Collections.singleton(Authorizations.EMPTY); + + private Query query = null; + + // Leave in a top-level query for backwards-compatibility purposes + private String queryString = null; + + private Date beginDate = null; + private Date endDate = null; + + // The max number of next + seek calls made by the underlying iterators + private Long maxWork = -1L; + + protected int baseIteratorPriority = 100; + + // Table name + private String tableName = TableName.SHARD; + + private Iterator queries = Collections.emptyIterator(); + + protected boolean bypassAccumulo; + + /** + * Empty default constructor + */ + public GenericQueryConfiguration() { + + } + + /** + * Pulls the table name, max query results, and max rows to scan from the provided argument + * + * @param configuredLogic + * A pre-configured BaseQueryLogic to initialize the Configuration with + */ + public GenericQueryConfiguration(BaseQueryLogic configuredLogic) { + this(configuredLogic.getConfig()); + } + + public GenericQueryConfiguration(GenericQueryConfiguration genericConfig) { + this.setBaseIteratorPriority(genericConfig.getBaseIteratorPriority()); + this.setBypassAccumulo(genericConfig.getBypassAccumulo()); + this.setAuthorizations(genericConfig.getAuthorizations()); + this.setBeginDate(genericConfig.getBeginDate()); + this.setConnector(genericConfig.getConnector()); + this.setEndDate(genericConfig.getEndDate()); + this.setMaxWork(genericConfig.getMaxWork()); + this.setQueries(genericConfig.getQueries()); + this.setQueryString(genericConfig.getQueryString()); + this.setTableName(genericConfig.getTableName()); + } + + /** + * Return the configured {@code Iterator} + * + * @return + */ + public Iterator getQueries() { + return Iterators.unmodifiableIterator(this.queries); + } + + /** + * Set the queries to be run. + * + * @param queries + */ + public void setQueries(Iterator queries) { + this.queries = queries; + } + + @JsonIgnore + @XmlTransient + public Connector getConnector() { + return connector; + } + + public void setConnector(Connector connector) { + this.connector = connector; + } + + public Query getQuery() { + return query; + } + + public void setQuery(Query query) { + this.query = query; + } + + public void setQueryString(String query) { + this.queryString = query; + } + + public String getQueryString() { + return queryString; + } + + public Set getAuthorizations() { + return authorizations; + } + + public void setAuthorizations(Set auths) { + this.authorizations = auths; + } + + public int getBaseIteratorPriority() { + return baseIteratorPriority; + } + + public void setBaseIteratorPriority(final int baseIteratorPriority) { + this.baseIteratorPriority = baseIteratorPriority; + } + + public Date getBeginDate() { + return beginDate; + } + + public void setBeginDate(Date beginDate) { + this.beginDate = beginDate; + } + + public Date getEndDate() { + return endDate; + } + + public void setEndDate(Date endDate) { + this.endDate = endDate; + } + + public Long getMaxWork() { + return maxWork; + } + + public void setMaxWork(Long maxWork) { + this.maxWork = maxWork; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public boolean getBypassAccumulo() { + return bypassAccumulo; + } + + public void setBypassAccumulo(boolean bypassAccumulo) { + this.bypassAccumulo = bypassAccumulo; + } + + /** + * Checks for non-null, sane values for the configured values + * + * @return True if all of the encapsulated values have legitimate values, otherwise false + */ + public boolean canRunQuery() { + // Ensure we were given connector and authorizations + if (null == this.getConnector() || null == this.getAuthorizations()) { + return false; + } + + // Ensure valid dates + if (null == this.getBeginDate() || null == this.getEndDate() || endDate.before(beginDate)) { + return false; + } + + // A non-empty table was given + if (null == getTableName() || this.getTableName().isEmpty()) { + return false; + } + + // At least one QueryData was provided + if (null == this.queries) { + return false; + } + + return true; + } +} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/QueryData.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/QueryData.java new file mode 100644 index 00000000..75173fb2 --- /dev/null +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/QueryData.java @@ -0,0 +1,84 @@ +package datawave.microservice.query.configuration; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.data.Range; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Class to encapsulate all required information to run a query. + * + */ +public class QueryData { + List settings = Lists.newArrayList(); + String query; + Collection ranges = Sets.newHashSet(); + Collection columnFamilies = Sets.newHashSet(); + + public QueryData() {} + + public QueryData(String query, Collection ranges, List settings) { + setQuery(query); + setRanges(ranges); + setSettings(settings); + } + + public QueryData(QueryData other) { + this(other.getQuery(), other.getRanges(), other.getSettings()); + } + + public QueryData(QueryData other, Collection ranges) { + setQuery(other.getQuery()); + setSettings(other.getSettings()); + setRanges(ranges); + } + + public QueryData(String queryString, ArrayList ranges, List settings, Collection columnFamilies) { + this(queryString, ranges, settings); + this.columnFamilies.addAll(columnFamilies); + } + + public List getSettings() { + return settings; + } + + public void setSettings(List settings) { + this.settings = Lists.newArrayList(settings); + } + + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + + public Collection getRanges() { + return ranges; + } + + public Collection getColumnFamilies() { + return columnFamilies; + } + + public void setRanges(Collection ranges) { + if (null != ranges) + this.ranges.addAll(ranges); + } + + public void addIterator(IteratorSetting cfg) { + this.settings.add(cfg); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(256); + sb.append("Query: '").append(this.query).append("', Ranges: ").append(this.ranges).append(", Settings: ").append(this.settings); + return sb.toString(); + } +} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/exception/EmptyObjectException.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/exception/EmptyObjectException.java new file mode 100644 index 00000000..38633c58 --- /dev/null +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/exception/EmptyObjectException.java @@ -0,0 +1,7 @@ +package datawave.microservice.query.exception; + +// used when a transformer gets a non-null empty object +// and the TransformIterator should call next instead of returning null +public class EmptyObjectException extends RuntimeException { + +} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/iterator/DatawaveTransformIterator.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/iterator/DatawaveTransformIterator.java new file mode 100644 index 00000000..7dafea21 --- /dev/null +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/iterator/DatawaveTransformIterator.java @@ -0,0 +1,76 @@ +package datawave.microservice.query.iterator; + +import datawave.microservice.query.exception.EmptyObjectException; +import datawave.microservice.query.logic.Flushable; +import org.apache.commons.collections4.Transformer; +import org.apache.commons.collections4.iterators.TransformIterator; +import org.apache.log4j.Logger; + +import java.util.Iterator; + +public class DatawaveTransformIterator extends TransformIterator { + + private Logger log = Logger.getLogger(DatawaveTransformIterator.class); + private O next = null; + + public DatawaveTransformIterator() { + super(); + } + + public DatawaveTransformIterator(Iterator iterator) { + super(iterator); + } + + public DatawaveTransformIterator(Iterator iterator, Transformer transformer) { + super(iterator, transformer); + } + + @Override + public boolean hasNext() { + + if (next == null) { + next = getNext(); + } + return (next != null); + } + + @Override + public O next() { + + O o = null; + if (next == null) { + o = getNext(); + } else { + o = next; + next = null; + } + return o; + } + + private O getNext() { + + boolean done = false; + O o = null; + while (super.hasNext() && !done) { + try { + o = super.next(); + done = true; + } catch (EmptyObjectException e) { + // not yet done, so continue fetching next + } + } + // see if there are any results cached by the transformer + if (o == null && getTransformer() instanceof Flushable) { + done = false; + while (!done) { + try { + o = ((Flushable) getTransformer()).flush(); + done = true; + } catch (EmptyObjectException e) { + // not yet done, so continue flushing + } + } + } + return o; + } +} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/BaseQueryLogic.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/BaseQueryLogic.java new file mode 100644 index 00000000..3fe69e05 --- /dev/null +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/BaseQueryLogic.java @@ -0,0 +1,326 @@ +package datawave.microservice.query.logic; + +import datawave.audit.SelectorExtractor; +import datawave.marking.MarkingFunctions; +import datawave.microservice.query.configuration.GenericQueryConfiguration; +import datawave.microservice.query.iterator.DatawaveTransformIterator; +import datawave.webservice.common.audit.Auditor.AuditType; +import datawave.webservice.query.Query; +import datawave.webservice.query.result.event.ResponseObjectFactory; +import org.apache.accumulo.core.client.BatchScanner; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.ScannerBase; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.commons.collections4.iterators.TransformIterator; +import org.springframework.beans.factory.annotation.Required; + +import javax.inject.Inject; +import java.security.Principal; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public abstract class BaseQueryLogic implements QueryLogic { + + private GenericQueryConfiguration baseConfig; + private String logicName = "No logicName was set"; + private String logicDescription = "Not configured"; + private AuditType auditType = null; + protected long maxResults = -1L; + protected ScannerBase scanner; + @SuppressWarnings("unchecked") + protected Iterator iterator = (Iterator) Collections.emptyList().iterator(); + private int maxPageSize = 0; + private long pageByteTrigger = 0; + private boolean collectQueryMetrics = true; + private String _connPoolName; + protected Principal principal; + protected RoleManager roleManager; + protected MarkingFunctions markingFunctions; + @Inject + protected ResponseObjectFactory responseObjectFactory; + protected SelectorExtractor selectorExtractor; + + public static final String BYPASS_ACCUMULO = "rfile.debug"; + + public BaseQueryLogic() { + getConfig().setBaseIteratorPriority(100); + } + + public BaseQueryLogic(BaseQueryLogic other) { + // Generic Query Config variables + setTableName(other.getTableName()); + setMaxWork(other.getMaxWork()); + setMaxResults(other.getMaxResults()); + setBaseIteratorPriority(other.getBaseIteratorPriority()); + setBypassAccumulo(other.getBypassAccumulo()); + + // Other variables + setMaxResults(other.maxResults); + setMarkingFunctions(other.getMarkingFunctions()); + setResponseObjectFactory(other.getResponseObjectFactory()); + setLogicName(other.getLogicName()); + setLogicDescription(other.getLogicDescription()); + setAuditType(other.getAuditType(null)); + this.scanner = other.scanner; + this.iterator = other.iterator; + setMaxPageSize(other.getMaxPageSize()); + setPageByteTrigger(other.getPageByteTrigger()); + setCollectQueryMetrics(other.getCollectQueryMetrics()); + setConnPoolName(other.getConnPoolName()); + setPrincipal(other.getPrincipal()); + setRoleManager(other.getRoleManager()); + setSelectorExtractor(other.getSelectorExtractor()); + } + + public GenericQueryConfiguration getConfig() { + if (baseConfig == null) { + baseConfig = new GenericQueryConfiguration() {}; + } + + return baseConfig; + } + + @Override + public String getPlan(Connector connection, Query settings, Set runtimeQueryAuthorizations, boolean expandFields, boolean expandValues) + throws Exception { + // for many query logics, the query is what it is + return settings.getQuery(); + } + + public MarkingFunctions getMarkingFunctions() { + return markingFunctions; + } + + public void setMarkingFunctions(MarkingFunctions markingFunctions) { + this.markingFunctions = markingFunctions; + } + + public ResponseObjectFactory getResponseObjectFactory() { + return responseObjectFactory; + } + + public void setResponseObjectFactory(ResponseObjectFactory responseObjectFactory) { + this.responseObjectFactory = responseObjectFactory; + } + + public Principal getPrincipal() { + return principal; + } + + public void setPrincipal(Principal principal) { + this.principal = principal; + } + + @Override + public String getTableName() { + return getConfig().getTableName(); + } + + @Override + public long getMaxResults() { + return this.maxResults; + } + + @Override + @Deprecated + public long getMaxRowsToScan() { + return getMaxWork(); + } + + @Override + public long getMaxWork() { + return getConfig().getMaxWork(); + } + + @Override + public void setTableName(String tableName) { + getConfig().setTableName(tableName); + } + + @Override + public void setMaxResults(long maxResults) { + this.maxResults = maxResults; + } + + @Override + @Deprecated + public void setMaxRowsToScan(long maxRowsToScan) { + setMaxWork(maxRowsToScan); + } + + @Override + public void setMaxWork(long maxWork) { + getConfig().setMaxWork(maxWork); + } + + @Override + public int getMaxPageSize() { + return maxPageSize; + } + + @Override + public void setMaxPageSize(int maxPageSize) { + this.maxPageSize = maxPageSize; + } + + @Override + public long getPageByteTrigger() { + return pageByteTrigger; + } + + @Override + public void setPageByteTrigger(long pageByteTrigger) { + this.pageByteTrigger = pageByteTrigger; + } + + @Override + public int getBaseIteratorPriority() { + return getConfig().getBaseIteratorPriority(); + } + + @Override + public void setBaseIteratorPriority(final int baseIteratorPriority) { + getConfig().setBaseIteratorPriority(baseIteratorPriority); + } + + @Override + public Iterator iterator() { + return iterator; + } + + @Override + public TransformIterator getTransformIterator(Query settings) { + return new DatawaveTransformIterator(this.iterator(), this.getTransformer(settings)); + } + + @Override + public String getLogicName() { + return logicName; + } + + @Override + public void setLogicName(String logicName) { + this.logicName = logicName; + } + + public boolean getBypassAccumulo() { + return getConfig().getBypassAccumulo(); + } + + public void setBypassAccumulo(boolean bypassAccumulo) { + getConfig().setBypassAccumulo(bypassAccumulo); + } + + @Override + public abstract Object clone() throws CloneNotSupportedException; + + public void close() { + if (null == scanner) + return; + if (scanner instanceof BatchScanner) { + scanner.close(); + } + } + + /* + * Implementations must override if they want a query-specific value returned + */ + @Override + public AuditType getAuditType(Query query) { + return auditType; + } + + @Override + public AuditType getAuditType() { + return auditType; + } + + @Override + @Required + // enforces that the unit tests will fail and the application will not deploy unless this property is set + public void setAuditType(AuditType auditType) { + this.auditType = auditType; + } + + @Override + public void setLogicDescription(String logicDescription) { + this.logicDescription = logicDescription; + } + + @Override + public String getLogicDescription() { + return logicDescription; + } + + public boolean getCollectQueryMetrics() { + return collectQueryMetrics; + } + + public void setCollectQueryMetrics(boolean collectQueryMetrics) { + this.collectQueryMetrics = collectQueryMetrics; + } + + public RoleManager getRoleManager() { + return roleManager; + } + + public void setRoleManager(RoleManager roleManager) { + this.roleManager = roleManager; + } + + /** {@inheritDoc} */ + @Override + public String getConnPoolName() { + return _connPoolName; + } + + /** {@inheritDoc} */ + @Override + public void setConnPoolName(final String connPoolName) { + _connPoolName = connPoolName; + } + + public boolean canRunQuery() { + return this.canRunQuery(this.getPrincipal()); + } + + /** {@inheritDoc} */ + public boolean canRunQuery(Principal principal) { + return this.roleManager == null || this.roleManager.canRunQuery(this, principal); + } + + @Override + public final void validate(Map> parameters) throws IllegalArgumentException { + Set requiredParams = getRequiredQueryParameters(); + for (String required : requiredParams) { + List values = parameters.get(required); + if (null == values) { + throw new IllegalArgumentException("Required parameter " + required + " not found"); + } + } + } + + @Override + public List getSelectors(Query settings) throws IllegalArgumentException { + List selectorList = null; + if (this.selectorExtractor != null) { + try { + selectorList = this.selectorExtractor.extractSelectors(settings); + } catch (Exception e) { + throw new IllegalArgumentException(e.getMessage(), e); + } + } + return selectorList; + } + + public void setSelectorExtractor(SelectorExtractor selectorExtractor) { + this.selectorExtractor = selectorExtractor; + } + + public SelectorExtractor getSelectorExtractor() { + return selectorExtractor; + } +} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/Flushable.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/Flushable.java new file mode 100644 index 00000000..6ad37ff9 --- /dev/null +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/Flushable.java @@ -0,0 +1,17 @@ +package datawave.microservice.query.logic; + +import datawave.microservice.query.exception.EmptyObjectException; + +public interface Flushable { + + /** + * The flush method is used to return an results that were cached from the calls to transform(Object). If this method will be called multiple times until a + * null is returned. If EmptyObjectException is thrown instead of returning null, then flush will be called again. + * + * @return A cached object or null if no more exist. + * @throws EmptyObjectException + * if the current cached result is empty, and flush should be called again. + */ + T flush() throws EmptyObjectException; + +} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogic.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogic.java new file mode 100644 index 00000000..bbbace46 --- /dev/null +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogic.java @@ -0,0 +1,300 @@ +package datawave.microservice.query.logic; + +import java.security.Principal; +import java.util.List; +import java.util.Set; + +import datawave.audit.SelectorExtractor; +import datawave.marking.MarkingFunctions; +import datawave.validation.ParameterValidator; +import datawave.webservice.common.audit.Auditor.AuditType; +import datawave.webservice.common.connection.AccumuloConnectionFactory; +import datawave.webservice.query.Query; +import datawave.webservice.query.cache.ResultsPage; +import datawave.microservice.query.configuration.GenericQueryConfiguration; +import datawave.webservice.query.exception.DatawaveErrorCode; +import datawave.webservice.query.exception.QueryException; +import datawave.webservice.query.result.event.ResponseObjectFactory; + +import datawave.webservice.result.BaseResponse; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.commons.collections4.iterators.TransformIterator; + +public interface QueryLogic extends Iterable, Cloneable, ParameterValidator { + + /** + * A mechanism to get the normalized query without actually setting up the query. This can be called with having to call initialize. + * + * The default implementation is to return the query string as the normalized query + * + * @param connection + * - Accumulo connector to use for this query + * @param settings + * - query settings (query, begin date, end date, etc.) + * @param runtimeQueryAuthorizations + * - authorizations that have been calculated for this query based on the caller and server. + * @param expandFields + * - should unfielded terms be expanded + * @param expandValues + * - should regex/ranges be expanded into discrete values + */ + String getPlan(Connector connection, Query settings, Set runtimeQueryAuthorizations, boolean expandFields, boolean expandValues) + throws Exception; + + /** + * Implementations create a configuration using the connection, settings, and runtimeQueryAuthorizations. + * + * @param connection + * - Accumulo connector to use for this query + * @param settings + * - query settings (query, begin date, end date, etc.) + * @param runtimeQueryAuthorizations + * - authorizations that have been calculated for this query based on the caller and server. + * @throws Exception + */ + GenericQueryConfiguration initialize(Connector connection, Query settings, Set runtimeQueryAuthorizations) throws Exception; + + /** + * + * @param settings + * - query settings (query, begin date, end date, etc.) + * @return list of selectors used in the Query + */ + List getSelectors(Query settings); + + SelectorExtractor getSelectorExtractor(); + + /** + * Implementations use the configuration to run their query. It is expected that initialize has already been called. + * + * @param configuration + * Encapsulates all information needed to run a query (whether the query is a BatchScanner, a MapReduce job, etc) + */ + void setupQuery(GenericQueryConfiguration configuration) throws Exception; + + /** + * @return a copy of this instance + */ + Object clone() throws CloneNotSupportedException; + + /** + * @return priority from AccumuloConnectionFactory + */ + AccumuloConnectionFactory.Priority getConnectionPriority(); + + /** + * @return Transformer that will convert Key,Value to a Result object + */ + QueryLogicTransformer getTransformer(Query settings); + + default String getResponseClass(Query query) throws QueryException { + try { + QueryLogicTransformer t = this.getTransformer(query); + BaseResponse refResponse = t.createResponse(new ResultsPage()); + return refResponse.getClass().getCanonicalName(); + } catch (RuntimeException e) { + throw new QueryException(DatawaveErrorCode.QUERY_TRANSFORM_ERROR); + } + } + + /** + * Allows for the customization of handling query results, e.g. allows for aggregation of query results before returning to the client. + * + * @param settings + * The query settings object + * @return Return a TransformIterator for the QueryLogic implementation + */ + TransformIterator getTransformIterator(Query settings); + + /** + * release resources + */ + void close(); + + /** @return the tableName */ + String getTableName(); + + /** + * @return max number of results to pass back to the caller + */ + long getMaxResults(); + + /** + * @return the results of getMaxWork + */ + @Deprecated + long getMaxRowsToScan(); + + /** + * @return max number of nexts + seeks performed by the underlying iterators in total + */ + long getMaxWork(); + + /** + * @return max number of records to return in a page (max pagesize allowed) + */ + int getMaxPageSize(); + + /** + * @return the number of bytes at which a page will be returned, even if pagesize has not been reached + */ + long getPageByteTrigger(); + + /** + * Returns the base iterator priority. + * + * @return base iterator priority + */ + int getBaseIteratorPriority(); + + /** + * @param tableName + * the name of the table + */ + void setTableName(String tableName); + + /** + * @param maxResults + * max number of results to pass back to the caller + */ + void setMaxResults(long maxResults); + + /** + * @param maxRowsToScan + * This is now deprecated and setMaxWork should be used instead. This is equivalent to setMaxWork. + */ + @Deprecated + void setMaxRowsToScan(long maxRowsToScan); + + /** + * @param maxWork + * max work which is normally calculated as the number of next + seek calls made by the underlying iterators + */ + void setMaxWork(long maxWork); + + /** + * @param maxPageSize + * max number of records in a page (max pagesize allowed) + */ + void setMaxPageSize(int maxPageSize); + + /** + * @param pageByteTrigger + * the number of bytes at which a page will be returned, even if pagesize has not been reached + */ + void setPageByteTrigger(long pageByteTrigger); + + /** + * Sets the base iterator priority + * + * @param priority + * base iterator priority + */ + void setBaseIteratorPriority(final int priority); + + /** + * @param logicName + * name of the query logic + */ + void setLogicName(String logicName); + + /** + * @return name of the query logic + */ + String getLogicName(); + + /** + * @param logicDescription + * a brief description of this logic type + */ + void setLogicDescription(String logicDescription); + + /** + * @return the audit level for this logic + */ + AuditType getAuditType(Query query); + + /** + * @return the audit level for this logic for a specific query + */ + AuditType getAuditType(); + + /** + * @param auditType + * the audit level for this logic + */ + void setAuditType(AuditType auditType); + + /** + * @return a brief description of this logic type + */ + String getLogicDescription(); + + /** + * @return should query metrics be collected for this query logic + */ + boolean getCollectQueryMetrics(); + + /** + * @param collectQueryMetrics + * whether query metrics be collected for this query logic + */ + void setCollectQueryMetrics(boolean collectQueryMetrics); + + void setRoleManager(RoleManager roleManager); + + RoleManager getRoleManager(); + + /** + * List of parameters that can be used in the 'params' parameter to Query/create + * + * @return the supported parameters + */ + Set getOptionalQueryParameters(); + + /** + * @param connPoolName + * The name of the connection pool to set. + */ + void setConnPoolName(String connPoolName); + + /** @return the connPoolName */ + String getConnPoolName(); + + /** + * Check that the user has one of the required roles principal my be null when there is no intent to control access to QueryLogic + * + * @param principal + * @return true/false + */ + boolean canRunQuery(Principal principal); + + boolean canRunQuery(); // uses member Principal + + void setPrincipal(Principal principal); + + Principal getPrincipal(); + + MarkingFunctions getMarkingFunctions(); + + void setMarkingFunctions(MarkingFunctions markingFunctions); + + ResponseObjectFactory getResponseObjectFactory(); + + void setResponseObjectFactory(ResponseObjectFactory responseObjectFactory); + + /** + * List of parameters that must be passed from the client for this query logic to work + * + * @return the required parameters + */ + Set getRequiredQueryParameters(); + + /** + * + * @return set of example queries + */ + Set getExampleQueries(); + +} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogicTransformer.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogicTransformer.java new file mode 100644 index 00000000..1b896b73 --- /dev/null +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogicTransformer.java @@ -0,0 +1,26 @@ +package datawave.microservice.query.logic; + +import datawave.microservice.query.exception.EmptyObjectException; +import datawave.webservice.query.cache.ResultsPage; +import datawave.webservice.result.BaseQueryResponse; +import org.apache.commons.collections4.Transformer; + +public interface QueryLogicTransformer extends Transformer { + + /* + * @return a jaxb response object that is specific to this QueryLogic + */ + BaseQueryResponse createResponse(ResultsPage resultList); + + /** + * Transforms the input object (leaving it unchanged) into some output object. + * + * @param input + * the object to be transformed, should be left unchanged + * @return a transformed object + * @throws EmptyObjectException + * if the result is empty + */ + @Override + O transform(I input) throws EmptyObjectException; +} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/RoleManager.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/RoleManager.java new file mode 100644 index 00000000..2606df41 --- /dev/null +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/RoleManager.java @@ -0,0 +1,14 @@ +package datawave.microservice.query.logic; + +import java.security.Principal; +import java.util.Set; + +public interface RoleManager { + + boolean canRunQuery(QueryLogic queryLogic, Principal principal); + + void setRequiredRoles(Set requiredRoles); + + Set getRequiredRoles(); + +} diff --git a/query-microservices/query/service/pom.xml b/query-microservices/query/service/pom.xml index 9d4fe7f9..471fbc59 100644 --- a/query-microservices/query/service/pom.xml +++ b/query-microservices/query/service/pom.xml @@ -12,6 +12,7 @@ DATAWAVE Query Microservice 3.1.5-SNAPSHOT + 1.0-SNAPSHOT 1.0-SNAPSHOT 1.0-SNAPSHOT 1.6.6-SNAPSHOT @@ -19,6 +20,16 @@ + + gov.nsa.datawave + datawave-query-core + ${version.datawave} + + + gov.nsa.datawave.microservice + query + ${version.microservice.query} + gov.nsa.datawave.microservice query-api @@ -54,6 +65,14 @@ + + gov.nsa.datawave + datawave-query-core + + + gov.nsa.datawave.microservice + query + gov.nsa.datawave.microservice query-api @@ -70,10 +89,6 @@ gov.nsa.datawave.microservice spring-boot-starter-datawave-audit - - gov.nsa.datawave.webservices - datawave-ws-client - diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java index 47490328..c5f9392e 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java @@ -1,9 +1,9 @@ package datawave.microservice.query; import com.codahale.metrics.annotation.Timed; +import datawave.microservice.query.config.QueryProperties; import datawave.microservice.query.web.annotation.EnrichQueryMetrics; import datawave.webservice.result.GenericResponse; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.MediaType; import org.springframework.util.MultiValueMap; import org.springframework.web.bind.annotation.PathVariable; @@ -17,10 +17,17 @@ @RestController @RequestMapping(path = "/v1", produces = MediaType.APPLICATION_JSON_VALUE) public class QueryController { - - @Autowired - private QueryManagement queryManagement; - + + private QueryProperties queryProperties; + private QueryParameters queryParameters; + private QueryManagementService queryManagementService; + + public QueryController(QueryProperties queryProperties, QueryParameters queryParameters, QueryManagementService queryManagementService) { + this.queryProperties = queryProperties; + this.queryParameters = queryParameters; + this.queryManagementService = queryManagementService; + } + @Timed(name = "dw.query.defineQuery", absolute = true) @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE) @RequestMapping(path = "{queryLogic}/define", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", @@ -38,6 +45,5 @@ public GenericResponse define(@PathVariable(name = "queryLogic") String resp.setResult(UUID.randomUUID().toString()); return resp; } - - + } diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryManagement.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryManagement.java deleted file mode 100644 index 3fb0b580..00000000 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryManagement.java +++ /dev/null @@ -1,152 +0,0 @@ -package datawave.microservice.query; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import datawave.microservice.query.util.QueryUtil; -import datawave.webservice.common.audit.AuditParameters; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; -import org.springframework.util.MultiValueMap; - -import javax.ws.rs.core.HttpHeaders; -import java.text.MessageFormat; -import java.util.Arrays; -import java.util.Collections; - -@Component -public class QueryManagement { - private final Logger log = LoggerFactory.getLogger(this.getClass()); - - private static final ObjectMapper mapper = new ObjectMapper(); - - /** - * This method will provide some initial query validation for the define and create query calls. - */ - private void validateQuery(String logicName, MultiValueMap parameters, HttpHeaders httpHeaders) { - - // add query logic name to parameters - parameters.add(QueryParameters.QUERY_LOGIC_NAME, logicName); - - log.debug(writeValueAsString(parameters)); - - // Pull "params" values into individual query parameters for validation on the query logic. - // This supports the deprecated "params" value (both on the old and new API). Once we remove the deprecated - // parameter, this code block can go away. - parameters.get(QueryParameters.QUERY_PARAMS).stream().map(QueryUtil::parseParameters).forEach(parameters::addAll); - - parameters.remove(AuditParameters.QUERY_SECURITY_MARKING_COLVIZ); - parameters.remove(AuditParameters.USER_DN); - parameters.remove(AuditParameters.QUERY_AUDIT_TYPE); - - // Ensure that all required parameters exist prior to validating the values. - qp.validate(parameters); - - // The pagesize and expirationDate checks will always be false when called from the RemoteQueryExecutor. - // Leaving for now until we can test to ensure that is always the case. - if (qp.getPagesize() <= 0) { - log.error("Invalid page size: " + qp.getPagesize()); - GenericResponse response = new GenericResponse<>(); - throwBadRequest(DatawaveErrorCode.INVALID_PAGE_SIZE, response); - } - - if (qp.getPageTimeout() != -1 && (qp.getPageTimeout() < PAGE_TIMEOUT_MIN || qp.getPageTimeout() > PAGE_TIMEOUT_MAX)) { - log.error("Invalid page timeout: " + qp.getPageTimeout()); - GenericResponse response = new GenericResponse<>(); - throwBadRequest(DatawaveErrorCode.INVALID_PAGE_TIMEOUT, response); - } - - if (System.currentTimeMillis() >= qp.getExpirationDate().getTime()) { - log.error("Invalid expiration date: " + qp.getExpirationDate()); - GenericResponse response = new GenericResponse<>(); - throwBadRequest(DatawaveErrorCode.INVALID_EXPIRATION_DATE, response); - } - - // Ensure begin date does not occur after the end date (if dates are not null) - if ((qp.getBeginDate() != null && qp.getEndDate() != null) && qp.getBeginDate().after(qp.getEndDate())) { - log.error("Invalid begin and/or end date: " + qp.getBeginDate() + " - " + qp.getEndDate()); - GenericResponse response = new GenericResponse<>(); - throwBadRequest(DatawaveErrorCode.BEGIN_DATE_AFTER_END_DATE, response); - } - - // will throw IllegalArgumentException if not defined - try { - qd.logic = queryLogicFactory.getQueryLogic(queryLogicName, ctx.getCallerPrincipal()); - } catch (Exception e) { - log.error("Failed to get query logic for " + queryLogicName, e); - BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.QUERY_LOGIC_ERROR, e); - GenericResponse response = new GenericResponse<>(); - response.addException(qe.getBottomQueryException()); - throw new BadRequestException(qe, response); - } - qd.logic.validate(parameters); - - try { - marking.clear(); - marking.validate(parameters); - } catch (IllegalArgumentException e) { - log.error("Failed security markings validation", e); - BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.SECURITY_MARKING_CHECK_ERROR, e); - GenericResponse response = new GenericResponse<>(); - response.addException(qe); - throw new BadRequestException(qe, response); - } - // Find out who/what called this method - qd.proxyServers = null; - qd.p = ctx.getCallerPrincipal(); - qd.userDn = qd.p.getName(); - qd.userid = qd.userDn; - qd.dnList = Collections.singletonList(qd.userid); - if (qd.p instanceof DatawavePrincipal) { - DatawavePrincipal dp = (DatawavePrincipal) qd.p; - qd.userid = dp.getShortName(); - qd.userDn = dp.getUserDN().subjectDN(); - String[] dns = dp.getDNs(); - Arrays.sort(dns); - qd.dnList = Arrays.asList(dns); - qd.proxyServers = dp.getProxyServers(); - } - log.trace(qd.userid + " has authorizations " + ((qd.p instanceof DatawavePrincipal) ? ((DatawavePrincipal) qd.p).getAuthorizations() : "")); - - // always check against the max - if (qd.logic.getMaxPageSize() > 0 && qp.getPagesize() > qd.logic.getMaxPageSize()) { - log.error("Invalid page size: " + qp.getPagesize() + " vs " + qd.logic.getMaxPageSize()); - BadRequestQueryException qe = new BadRequestQueryException(DatawaveErrorCode.PAGE_SIZE_TOO_LARGE, MessageFormat.format("Max = {0}.", - qd.logic.getMaxPageSize())); - GenericResponse response = new GenericResponse<>(); - response.addException(qe); - throw new BadRequestException(qe, response); - } - - // validate the max results override relative to the max results on a query logic - // privileged users however can set whatever they want - if (qp.isMaxResultsOverridden() && qd.logic.getMaxResults() >= 0) { - if (!ctx.isCallerInRole(PRIVILEGED_USER)) { - if (qp.getMaxResultsOverride() < 0 || (qd.logic.getMaxResults() < qp.getMaxResultsOverride())) { - log.error("Invalid max results override: " + qp.getMaxResultsOverride() + " vs " + qd.logic.getMaxResults()); - GenericResponse response = new GenericResponse<>(); - throwBadRequest(DatawaveErrorCode.INVALID_MAX_RESULTS_OVERRIDE, response); - } - } - } - - // Set private audit-related parameters, stripping off any that the user might have passed in first. - // These are parameters that aren't passed in by the user, but rather are computed from other sources. - PrivateAuditConstants.stripPrivateParameters(parameters); - parameters.add(PrivateAuditConstants.LOGIC_CLASS, queryLogicName); - parameters.putSingle(PrivateAuditConstants.COLUMN_VISIBILITY, marking.toColumnVisibilityString()); - parameters.add(PrivateAuditConstants.USER_DN, qd.userDn); - - return qd; - } - - private String writeValueAsString(Object object) { - String stringValue = ""; - try { - stringValue = mapper.writeValueAsString(object); - } catch (JsonProcessingException e) { - stringValue = String.valueOf(object); - } - return stringValue; - } -} diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java new file mode 100644 index 00000000..04a5e451 --- /dev/null +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -0,0 +1,174 @@ +package datawave.microservice.query; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import datawave.marking.SecurityMarking; +import datawave.microservice.query.config.QueryExpirationProperties; +import datawave.microservice.query.config.QueryProperties; +import datawave.microservice.query.logic.QueryLogic; +import datawave.microservice.query.util.QueryUtil; +import datawave.webservice.common.audit.AuditParameters; +import datawave.webservice.query.exception.BadRequestQueryException; +import datawave.webservice.query.exception.DatawaveErrorCode; +import datawave.webservice.query.exception.QueryException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.util.MultiValueMap; + +import javax.ws.rs.core.HttpHeaders; +import java.security.Principal; +import java.text.MessageFormat; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +@Service +public class QueryManagementService { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + + private static final ObjectMapper mapper = new ObjectMapper(); + + private QueryProperties queryProperties; + private QueryParameters queryParameters; + private SecurityMarking securityMarking; + + // TODO: Pull these from configuration instead + private final int PAGE_TIMEOUT_MIN = 1; + private final int PAGE_TIMEOUT_MAX = QueryExpirationProperties.PAGE_TIMEOUT_MIN_DEFAULT; + + public QueryManagementService(QueryProperties queryProperties, QueryParameters queryParameters, SecurityMarking securityMarking) { + this.queryProperties = queryProperties; + this.queryParameters = queryParameters; + this.securityMarking = securityMarking; + } + + // A few items that are cached by the validateQuery call + private static class QueryData { + QueryLogic logic = null; + Principal p = null; + Set proxyServers = null; + String userDn = null; + String userid = null; + List dnList = null; + } + + /** + * This method will provide some initial query validation for the define and create query calls. + */ + private void validateQuery(String queryLogicName, MultiValueMap parameters, HttpHeaders httpHeaders) throws QueryException { + + // add query logic name to parameters + parameters.add(QueryParameters.QUERY_LOGIC_NAME, queryLogicName); + + log.debug(writeValueAsString(parameters)); + + // Pull "params" values into individual query parameters for validation on the query logic. + // This supports the deprecated "params" value (both on the old and new API). Once we remove the deprecated + // parameter, this code block can go away. + parameters.get(QueryParameters.QUERY_PARAMS).stream().map(QueryUtil::parseParameters).forEach(parameters::addAll); + + parameters.remove(AuditParameters.QUERY_SECURITY_MARKING_COLVIZ); + parameters.remove(AuditParameters.USER_DN); + parameters.remove(AuditParameters.QUERY_AUDIT_TYPE); + + // Ensure that all required parameters exist prior to validating the values. + queryParameters.validate(parameters); + + // The pagesize and expirationDate checks will always be false when called from the RemoteQueryExecutor. + // Leaving for now until we can test to ensure that is always the case. + if (queryParameters.getPagesize() <= 0) { + log.error("Invalid page size: " + queryParameters.getPagesize()); + throw new BadRequestQueryException(DatawaveErrorCode.INVALID_PAGE_SIZE); + } + + if (queryParameters.getPageTimeout() != -1 + && (queryParameters.getPageTimeout() < PAGE_TIMEOUT_MIN || queryParameters.getPageTimeout() > PAGE_TIMEOUT_MAX)) { + log.error("Invalid page timeout: " + queryParameters.getPageTimeout()); + throw new BadRequestQueryException(DatawaveErrorCode.INVALID_PAGE_TIMEOUT); + } + + if (System.currentTimeMillis() >= queryParameters.getExpirationDate().getTime()) { + log.error("Invalid expiration date: " + queryParameters.getExpirationDate()); + throw new BadRequestQueryException(DatawaveErrorCode.INVALID_EXPIRATION_DATE); + } + + // Ensure begin date does not occur after the end date (if dates are not null) + if ((queryParameters.getBeginDate() != null && queryParameters.getEndDate() != null) + && queryParameters.getBeginDate().after(queryParameters.getEndDate())) { + log.error("Invalid begin and/or end date: " + queryParameters.getBeginDate() + " - " + queryParameters.getEndDate()); + throw new BadRequestQueryException(DatawaveErrorCode.BEGIN_DATE_AFTER_END_DATE); + } + + // // TODO: Get this working! + // // will throw IllegalArgumentException if not defined + // try { + // qd.logic = queryLogicFactory.getQueryLogic(queryLogicName, ctx.getCallerPrincipal()); + // } catch (Exception e) { + // log.error("Failed to get query logic for " + queryLogicName, e); + // throw new BadRequestQueryException(DatawaveErrorCode.QUERY_LOGIC_ERROR, e); + // } + // qd.logic.validate(parameters); + // + // try { + // securityMarking.clear(); + // securityMarking.validate(parameters); + // } catch (IllegalArgumentException e) { + // log.error("Failed security markings validation", e); + // throw new BadRequestQueryException(DatawaveErrorCode.SECURITY_MARKING_CHECK_ERROR, e); + // } + // // Find out who/what called this method + // qd.proxyServers = null; + // qd.p = ctx.getCallerPrincipal(); + // qd.userDn = qd.p.getName(); + // qd.userid = qd.userDn; + // qd.dnList = Collections.singletonList(qd.userid); + // if (qd.p instanceof DatawavePrincipal) { + // DatawavePrincipal dp = (DatawavePrincipal) qd.p; + // qd.userid = dp.getShortName(); + // qd.userDn = dp.getUserDN().subjectDN(); + // String[] dns = dp.getDNs(); + // Arrays.sort(dns); + // qd.dnList = Arrays.asList(dns); + // qd.proxyServers = dp.getProxyServers(); + // } + // log.trace(qd.userid + " has authorizations " + ((qd.p instanceof DatawavePrincipal) ? ((DatawavePrincipal) qd.p).getAuthorizations() : "")); + // + // // always check against the max + // if (qd.logic.getMaxPageSize() > 0 && queryParameters.getPagesize() > qd.logic.getMaxPageSize()) { + // log.error("Invalid page size: " + queryParameters.getPagesize() + " vs " + qd.logic.getMaxPageSize()); + // throw new BadRequestQueryException(DatawaveErrorCode.PAGE_SIZE_TOO_LARGE, MessageFormat.format("Max = {0}.", qd.logic.getMaxPageSize())); + // } + // + // // validate the max results override relative to the max results on a query logic + // // privileged users however can set whatever they want + // if (queryParameters.isMaxResultsOverridden() && qd.logic.getMaxResults() >= 0) { + // if (!ctx.isCallerInRole(PRIVILEGED_USER)) { + // if (queryParameters.getMaxResultsOverride() < 0 || (qd.logic.getMaxResults() < queryParameters.getMaxResultsOverride())) { + // log.error("Invalid max results override: " + queryParameters.getMaxResultsOverride() + " vs " + qd.logic.getMaxResults()); + // throw new BadRequestQueryException(DatawaveErrorCode.INVALID_MAX_RESULTS_OVERRIDE); + // } + // } + // } + // + // // Set private audit-related parameters, stripping off any that the user might have passed in first. + // // These are parameters that aren't passed in by the user, but rather are computed from other sources. + // PrivateAuditConstants.stripPrivateParameters(parameters); + // parameters.add(PrivateAuditConstants.LOGIC_CLASS, queryLogicName); + // parameters.putSingle(PrivateAuditConstants.COLUMN_VISIBILITY, marking.toColumnVisibilityString()); + // parameters.add(PrivateAuditConstants.USER_DN, qd.userDn); + // + // return qd; + } + + private String writeValueAsString(Object object) { + String stringValue = ""; + try { + stringValue = mapper.writeValueAsString(object); + } catch (JsonProcessingException e) { + stringValue = String.valueOf(object); + } + return stringValue; + } +} diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryService.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryService.java index aaf85e33..f2c3ae30 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryService.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryService.java @@ -4,13 +4,25 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.web.servlet.error.ErrorMvcAutoConfiguration; import org.springframework.cloud.client.discovery.EnableDiscoveryClient; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ImportResource; +import org.springframework.web.context.annotation.RequestScope; /** * Launcher for the query service */ @EnableDiscoveryClient @SpringBootApplication(scanBasePackages = "datawave.microservice", exclude = {ErrorMvcAutoConfiguration.class}) +// TODO: import the other query logics +// @ImportResource("classpath:QueryLogicFactory.xml") public class QueryService { + + @Bean + @RequestScope + public QueryParameters queryParameters() { + return new QueryParametersImpl(); + } + public static void main(String[] args) { SpringApplication.run(QueryService.class, args); } diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryProperties.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryProperties.java deleted file mode 100644 index 60022938..00000000 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryProperties.java +++ /dev/null @@ -1,26 +0,0 @@ -package datawave.microservice.query.config; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -import javax.validation.Valid; -import java.util.Map; - -// TODO: This may be a common property file used by multiple services, so we will eventually need to move it out -@Validated -@EnableConfigurationProperties(QueryProperties.class) -@ConfigurationProperties(prefix = "query") -public class QueryProperties { - - @Valid - private Map logic; - - public Map getLogic() { - return logic; - } - - public void setLogic(Map logic) { - this.logic = logic; - } -} diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java index 4b948a0c..9513dce1 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java @@ -1,8 +1,18 @@ package datawave.microservice.query.config; +import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -@EnableConfigurationProperties(QueryProperties.class) +import javax.validation.Valid; + @Configuration -public class QueryServiceConfig {} +public class QueryServiceConfig { + + @Bean + @ConfigurationProperties("query") + public QueryProperties queryProperties() { + return new QueryProperties(); + } +} diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/util/QueryUtil.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/util/QueryUtil.java index 2a3d653c..710b8e51 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/util/QueryUtil.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/util/QueryUtil.java @@ -22,7 +22,7 @@ public class QueryUtil { public static final String PARAMETER_SEPARATOR = ";"; public static final String PARAMETER_NAME_VALUE_SEPARATOR = ":"; private static final String NULL_BYTE = "\u0000"; - + public static String getQueryImplClassName(Key key) { String colq = key.getColumnQualifier().toString(); String[] parts = colq.split(NULL_BYTE); @@ -33,9 +33,9 @@ public static String getQueryImplClassName(Key key) { throw new RuntimeException("Query impl class name not found in colq: " + colq); } } - - public static T deserialize(String queryImplClassName, Text columnVisibility, Value value) throws InvalidProtocolBufferException, - ClassNotFoundException { + + public static T deserialize(String queryImplClassName, Text columnVisibility, Value value) + throws InvalidProtocolBufferException, ClassNotFoundException { @SuppressWarnings("unchecked") Class queryClass = (Class) Class.forName(queryImplClassName); byte[] b = value.get(); @@ -45,15 +45,16 @@ public static T deserialize(String queryImplClassName, Text co queryImpl.setColumnVisibility(columnVisibility.toString()); return queryImpl; } - + public static MultiValueMap parseParameters(final String paramsString) { final MultiValueMap parameters = new LinkedMultiValueMap<>(); - Arrays.stream(paramsString.split(PARAMETER_SEPARATOR)).map(x -> x.split(PARAMETER_NAME_VALUE_SEPARATOR)).filter(x -> x.length == 2).forEach(x -> parameters.add(x[0], x[1])); + Arrays.stream(paramsString.split(PARAMETER_SEPARATOR)).map(x -> x.split(PARAMETER_NAME_VALUE_SEPARATOR)).filter(x -> x.length == 2) + .forEach(x -> parameters.add(x[0], x[1])); return parameters; } - + private static ThreadLocal BUFFER = ThreadLocal.withInitial(() -> LinkedBuffer.allocate(1024)); - + public static Mutation toMutation(T query, ColumnVisibility vis) { // Store by sid for backwards compatibility Mutation m = new Mutation(query.getOwner()); @@ -67,7 +68,7 @@ public static Mutation toMutation(T query, ColumnVisibility vi BUFFER.get().clear(); } } - + public static String toParametersString(final Set parameters) { final StringBuilder params = new StringBuilder(); if (null != parameters) { @@ -75,7 +76,7 @@ public static String toParametersString(final Set parameter if (params.length() > 0) { params.append(PARAMETER_SEPARATOR); } - + params.append(param.getParameterName()); params.append(PARAMETER_NAME_VALUE_SEPARATOR); params.append(param.getParameterValue()); diff --git a/query-microservices/query/service/src/test/resources/config/application.yml b/query-microservices/query/service/src/test/resources/config/application.yml index 38d7ac87..de0b51b3 100644 --- a/query-microservices/query/service/src/test/resources/config/application.yml +++ b/query-microservices/query/service/src/test/resources/config/application.yml @@ -17,6 +17,82 @@ query: storage: backend: LOCAL + logic: + baseEventQuery: + accumuloPassword: "value" + tableName: "value" + dateIndexTableName: "value" + dateIndexHelperFactory: "someRef" + defaultDateTypeName: "value" + metadataTableName: "value" + metadataHelperFactory: "someRef" + indexTableName: "value" + reverseIndexTableName: "value" + maxResults: "value" + queryThreads: "value" + indexLookupThreads: "value" + dateIndexThreads: "value" + fullTableScanEnabled: "value" + includeDataTypeAsField: "value" + disableIndexOnlyDocuments: "value" + indexOnlyFilterFunctionsEnabled: "value" + includeHierarchyFields: "value" + hierarchyFieldOptions: "someRef" + baseIteratorPriority: "value" + maxIndexScanTimeMillis: "value" + collapseUids: "value" + collapseUidsThreshold: "value" + useEnrichers: "true" + contentFieldNames: + - "CONTENT" + realmSuffixExclusionPatterns: + - "<.*>$" + minimumSelectivity: ".2" + enricherClassNames: + - "datawave.query.enrich.DatawaveTermFrequencyEnricher" + useFilters: "value" + filterClassNames: + - "value" + filterOptions: "someRef" + auditType: "NONE" + logicDescription: "Retrieve sharded events/documents, leveraging the global index tables as needed" + eventPerDayThreshold: "40000" + shardsPerDayThreshold: "value" + maxTermThreshold: "value" + maxDepthThreshold: "value" + maxUnfieldedExpansionThreshold: "value" + maxValueExpansionThreshold: "value" + maxOrExpansionThreshold: "value" + maxOrRangeThreshold: "value" + maxOrExpansionFstThreshold: "value" + maxFieldIndexRangeSplit: "value" + maxIvaratorSources: "value" + maxEvaluationPipelines: "value" + maxPipelineCachedResults: "value" + hdfsSiteConfigURLs: "value" + zookeeperConfig: "value" + ivaratorCacheDirConfigs: "someRef" + ivaratorFstHdfsBaseURIs: "value" + ivaratorCacheBufferSize: "10000" + ivaratorMaxOpenFiles: "value" + ivaratorCacheScanPersistThreshold: "100000" + ivaratorCacheScanTimeoutMinutes: "value" + eventQueryDatqeDecoratorTransformer: "someRef" + modelTableName: "value" + modelName: "DATAWAVE" + querySyntaxParsers: + "JEXL": null + "LUCENE": "value-ref" + "LUCENE-UUID": "value-ref" + "TOKENIZED-LUCENE": "value-ref" + queryPlanner: "someRef" + sendTimingToStatsd: "false" + collectQueryMetrics: "true" + logTimingDetails: "false" + statsdHost: "localhost" + statsdPort: "8125" + selectorExtractor: "someRef" + evaluationOnlyFields: "value" server: port: 0 @@ -47,3 +123,8 @@ logging: level: datawave.microservice.query-api: DEBUG io.undertow.request: FATAL + +audit-client: + discovery: + enabled: false + uri: '${AUDIT_SERVER_URL:http://localhost:11111/audit}' From eeefb2d7e13200b489ca24afe7bf1580a8d2a7f3 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Wed, 31 Mar 2021 22:30:03 +0000 Subject: [PATCH 022/218] missing property for query locking manager --- .../query/service/src/test/resources/config/application.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/query-microservices/query/service/src/test/resources/config/application.yml b/query-microservices/query/service/src/test/resources/config/application.yml index de0b51b3..2f2bcb56 100644 --- a/query-microservices/query/service/src/test/resources/config/application.yml +++ b/query-microservices/query/service/src/test/resources/config/application.yml @@ -17,6 +17,8 @@ query: storage: backend: LOCAL + lockManager: + LOCAL logic: baseEventQuery: accumuloPassword: "value" From 39f3cf0cdaf390a5416611ce1f4fe27009702ed3 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Fri, 2 Apr 2021 19:34:13 +0000 Subject: [PATCH 023/218] Updated to spring-boot 2.4 dependencies --- query-microservices/query/api/pom.xml | 4 ++-- query-microservices/query/pom.xml | 4 ++-- query-microservices/query/service/pom.xml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/query-microservices/query/api/pom.xml b/query-microservices/query/api/pom.xml index 8fba2067..b904fb4c 100644 --- a/query-microservices/query/api/pom.xml +++ b/query-microservices/query/api/pom.xml @@ -4,13 +4,13 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 1.7 + 1.8 query-api 1.0-SNAPSHOT - 1.2 + 1.3-SNAPSHOT 2.0.1.Final diff --git a/query-microservices/query/pom.xml b/query-microservices/query/pom.xml index a6dd3ae1..321f066b 100644 --- a/query-microservices/query/pom.xml +++ b/query-microservices/query/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 1.7 + 1.8 query-service-parent @@ -31,4 +31,4 @@ https://raw.githubusercontent.com/NationalSecurityAgency/datawave/mvn-repo - \ No newline at end of file + diff --git a/query-microservices/query/service/pom.xml b/query-microservices/query/service/pom.xml index 471fbc59..6cbb77f4 100644 --- a/query-microservices/query/service/pom.xml +++ b/query-microservices/query/service/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-service-parent - 1.7.1 + 2.1-SNAPSHOT query-service From e9ceb58a4acc561e407bee2a4370309eff0faabd Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 8 Apr 2021 22:35:15 -0400 Subject: [PATCH 024/218] Made some updates for spring boot 2.4.4 --- query-microservices/query/api/pom.xml | 15 +++++++++++++++ query-microservices/query/query/pom.xml | 17 ++++++++++++++++- query-microservices/query/service/pom.xml | 17 ++++++++++++++++- .../src/test/resources/config/application.yml | 5 +++++ 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/query-microservices/query/api/pom.xml b/query-microservices/query/api/pom.xml index b904fb4c..e5a635d5 100644 --- a/query-microservices/query/api/pom.xml +++ b/query-microservices/query/api/pom.xml @@ -36,6 +36,21 @@ javax.validation validation-api + + junit + junit + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.vintage + junit-vintage-engine + test + diff --git a/query-microservices/query/query/pom.xml b/query-microservices/query/query/pom.xml index 7316bd37..7ea1ea24 100644 --- a/query-microservices/query/query/pom.xml +++ b/query-microservices/query/query/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 1.7 + 1.8 query @@ -45,6 +45,21 @@ gov.nsa.datawave.webservices datawave-ws-common + + junit + junit + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.vintage + junit-vintage-engine + test + diff --git a/query-microservices/query/service/pom.xml b/query-microservices/query/service/pom.xml index 6cbb77f4..575dabd5 100644 --- a/query-microservices/query/service/pom.xml +++ b/query-microservices/query/service/pom.xml @@ -16,7 +16,7 @@ 1.0-SNAPSHOT 1.0-SNAPSHOT 1.6.6-SNAPSHOT - 1.2 + 1.3-SNAPSHOT @@ -89,6 +89,21 @@ gov.nsa.datawave.microservice spring-boot-starter-datawave-audit + + junit + junit + test + + + org.junit.jupiter + junit-jupiter-engine + test + + + org.junit.vintage + junit-vintage-engine + test + diff --git a/query-microservices/query/service/src/test/resources/config/application.yml b/query-microservices/query/service/src/test/resources/config/application.yml index 2f2bcb56..344f0ae1 100644 --- a/query-microservices/query/service/src/test/resources/config/application.yml +++ b/query-microservices/query/service/src/test/resources/config/application.yml @@ -19,6 +19,11 @@ query: LOCAL lockManager: LOCAL + syncStorage: + true + sendNotifications: + true + logic: baseEventQuery: accumuloPassword: "value" From 89d7e21965be6c72197db58f3f44cf4b344317ea Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Wed, 14 Apr 2021 15:24:14 +0000 Subject: [PATCH 025/218] Merge branch 'release/version3.2' into feature/queryMicroservices --- .../query/logic/BaseQueryLogic.java | 22 ++++++ .../microservice/query/logic/QueryLogic.java | 75 +++++++++++++++++-- 2 files changed, 92 insertions(+), 5 deletions(-) diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/BaseQueryLogic.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/BaseQueryLogic.java index 3fe69e05..68607e62 100644 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/BaseQueryLogic.java +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/BaseQueryLogic.java @@ -28,6 +28,7 @@ public abstract class BaseQueryLogic implements QueryLogic { private String logicName = "No logicName was set"; private String logicDescription = "Not configured"; private AuditType auditType = null; + private Map dnResultLimits = null; protected long maxResults = -1L; protected ScannerBase scanner; @SuppressWarnings("unchecked") @@ -36,6 +37,7 @@ public abstract class BaseQueryLogic implements QueryLogic { private long pageByteTrigger = 0; private boolean collectQueryMetrics = true; private String _connPoolName; + private Set authorizedDNs; protected Principal principal; protected RoleManager roleManager; protected MarkingFunctions markingFunctions; @@ -323,4 +325,24 @@ public void setSelectorExtractor(SelectorExtractor selectorExtractor) { public SelectorExtractor getSelectorExtractor() { return selectorExtractor; } + + @Override + public Set getAuthorizedDNs() { + return authorizedDNs; + } + + @Override + public void setAuthorizedDNs(Set authorizedDNs) { + this.authorizedDNs = authorizedDNs; + } + + @Override + public void setDnResultLimits(Map dnResultLimits) { + this.dnResultLimits = dnResultLimits; + } + + @Override + public Map getDnResultLimits() { + return dnResultLimits; + } } diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogic.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogic.java index bbbace46..4598129a 100644 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogic.java +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogic.java @@ -1,9 +1,5 @@ package datawave.microservice.query.logic; -import java.security.Principal; -import java.util.List; -import java.util.Set; - import datawave.audit.SelectorExtractor; import datawave.marking.MarkingFunctions; import datawave.validation.ParameterValidator; @@ -15,12 +11,17 @@ import datawave.webservice.query.exception.DatawaveErrorCode; import datawave.webservice.query.exception.QueryException; import datawave.webservice.query.result.event.ResponseObjectFactory; - import datawave.webservice.result.BaseResponse; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.security.Authorizations; import org.apache.commons.collections4.iterators.TransformIterator; +import java.security.Principal; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + public interface QueryLogic extends Iterable, Cloneable, ParameterValidator { /** @@ -297,4 +298,68 @@ default String getResponseClass(Query query) throws QueryException { */ Set getExampleQueries(); + /** + * Return the DNs authorized access to this query logic. + * + * @return the set of DNs authorized access to this query logic, possibly null or empty + */ + Set getAuthorizedDNs(); + + /** + * Set the DNs authorized access to this query logic. + * + * @param allowedDNs + * the DNs authorized access + */ + void setAuthorizedDNs(Set allowedDNs); + + /** + * Return whether or not the provided collection of DNs contains at least oneDN that is authorized for access to this query logic. This will return true in + * the following cases: + *
    + *
  • The set of authorized DNs for this query logic is null or empty.
    • + *
  • The set of authorized DNs is not empty, and the provided collection contains a DN that is also found within the set of authorized DNs.
    • + *
+ * + * @param dns + * the DNs to determine access rights for + * @return true if the collection contains at least one DN that has access to this query logic, or false otherwise + */ + default boolean containsDNWithAccess(Collection dns) { + Set authorizedDNs = getAuthorizedDNs(); + return authorizedDNs == null || authorizedDNs.isEmpty() || (dns != null && dns.stream().anyMatch(authorizedDNs::contains)); + } + + /** + * Set the map of DNs to query result limits. This should override the default limit returned by {@link #getMaxResults()} for any included DNs. + * + * @param dnResultLimits + * the map of DNs to query result limits + */ + void setDnResultLimits(Map dnResultLimits); + + /** + * Return the map of DNs to result limits. + * + * @return the map of DNs to query result limits + */ + Map getDnResultLimits(); + + /** + * Return the maximum number of results to include for the query for any DN present in the specified collection. If limits are found for multiple DNs in the + * collection, the smallest value will be returned. If the provided collection is null or empty, or if no limits are found for any DN, the value of + * {@link #getMaxResults()} will be returned. + * + * @param dns + * the DNs to determine the maximum number of results to include for the query. It's expected that this list represents all the DNs in the DN + * chain for an individual user. + * @return the maximum number of results to include + */ + default long getResultLimit(Collection dns) { + Map dnResultLimits = getDnResultLimits(); + if (dnResultLimits == null || dns == null) { + return getMaxResults(); + } + return dns.stream().filter(dnResultLimits::containsKey).map(dnResultLimits::get).min(Long::compareTo).orElseGet(this::getMaxResults); + } } From 34addec8be9040db44c1f7e9a4d72adb22427fc5 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Wed, 14 Apr 2021 23:06:13 +0000 Subject: [PATCH 026/218] Changed the the iterators to return Result objects instead of Entry This allows registering the last result returned on a context object (e.g. QueryData) Hence facilitating checkpointing the query logic --- .../GenericQueryConfiguration.java | 11 +++ .../query/configuration/QueryData.java | 14 ++- .../query/configuration/Result.java | 95 +++++++++++++++++++ .../query/configuration/ResultContext.java | 10 ++ 4 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/Result.java create mode 100644 query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/ResultContext.java diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/GenericQueryConfiguration.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/GenericQueryConfiguration.java index ba176589..13c8424c 100644 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/GenericQueryConfiguration.java +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/GenericQueryConfiguration.java @@ -27,6 +27,9 @@ * */ public abstract class GenericQueryConfiguration { + // is this execution expected to be checkpointable (changes how we allocate ranges to scanners) + private boolean checkpointable = false; + private Connector connector = null; private Set authorizations = Collections.singleton(Authorizations.EMPTY); @@ -98,6 +101,14 @@ public void setQueries(Iterator queries) { this.queries = queries; } + public boolean isCheckpointable() { + return checkpointable; + } + + public void setCheckpointable(boolean checkpointable) { + this.checkpointable = checkpointable; + } + @JsonIgnore @XmlTransient public Connector getConnector() { diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/QueryData.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/QueryData.java index 75173fb2..d0178d65 100644 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/QueryData.java +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/QueryData.java @@ -3,21 +3,25 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Map; /** * Class to encapsulate all required information to run a query. * */ -public class QueryData { +public class QueryData implements ResultContext { List settings = Lists.newArrayList(); String query; Collection ranges = Sets.newHashSet(); Collection columnFamilies = Sets.newHashSet(); + Map.Entry lastResult; public QueryData() {} @@ -75,6 +79,14 @@ public void addIterator(IteratorSetting cfg) { this.settings.add(cfg); } + public void setLastResult(Map.Entry result) { + this.lastResult = result; + } + + public Map.Entry getLastResult() { + return lastResult; + } + @Override public String toString() { StringBuilder sb = new StringBuilder(256); diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/Result.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/Result.java new file mode 100644 index 00000000..ff9c4571 --- /dev/null +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/Result.java @@ -0,0 +1,95 @@ +package datawave.microservice.query.configuration; + +import com.google.common.base.Function; +import com.google.common.collect.Iterators; +import com.google.common.collect.Maps; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; + +import javax.annotation.Nullable; +import java.util.Iterator; +import java.util.Map; + +public class Result implements Map.Entry { + private final T context; + private final Key key; + private Value value; + + public Result(Key k, Value v) { + this(null, k, v); + } + + public Result(T context, Key k, Value v) { + this.context = context; + this.key = k; + this.value = v; + } + + public T getContext() { + return context; + } + + @Override + public Key getKey() { + return key; + } + + @Override + public Value getValue() { + return value; + } + + @Override + public Value setValue(Value value) { + throw new UnsupportedOperationException("This value is immutable"); + } + + @Override + public boolean equals(Object o) { + if (o instanceof Result) { + Result other = (Result) o; + return new EqualsBuilder().append(context, other.context).append(key, other.key).append(value, other.value).isEquals(); + } + return false; + } + + @Override + public int hashCode() { + return new HashCodeBuilder().append(context).append(key).append(value).toHashCode(); + } + + public Map.Entry returnKeyValue() { + Map.Entry entry = Maps.immutableEntry(key, value); + if (context != null) { + context.setLastResult(entry); + } + return entry; + } + + public static Iterator> keyValueIterator(Iterator it) { + return Iterators.transform(it, new Function>() { + @Override + public Map.Entry apply(@Nullable Result input) { + if (input == null) { + return null; + } + return input.returnKeyValue(); + } + }); + } + + public static Iterator resultIterator(final ResultContext context, Iterator> it) { + return Iterators.transform(it, new Function,Result>() { + @Nullable + @Override + public Result apply(@Nullable Map.Entry keyValueEntry) { + if (keyValueEntry == null) { + return null; + } + return new Result(context, keyValueEntry.getKey(), keyValueEntry.getValue()); + } + }); + } +} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/ResultContext.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/ResultContext.java new file mode 100644 index 00000000..962eb092 --- /dev/null +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/ResultContext.java @@ -0,0 +1,10 @@ +package datawave.microservice.query.configuration; + +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; + +import java.util.Map; + +public interface ResultContext { + void setLastResult(Map.Entry result); +} From 54046afe3e23b92dce6784767e366fb2ce3f5ee3 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 15 Apr 2021 23:25:32 -0400 Subject: [PATCH 027/218] Updated spring boot dependencies to 2.4.4 --- query-microservices/query/query/pom.xml | 14 ++++++++++- query-microservices/query/service/pom.xml | 23 ++++++++++--------- .../microservice/query/QueryServiceTest.java | 1 - .../src/test/resources/config/application.yml | 2 +- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/query-microservices/query/query/pom.xml b/query-microservices/query/query/pom.xml index 7ea1ea24..88e94aad 100644 --- a/query-microservices/query/query/pom.xml +++ b/query-microservices/query/query/pom.xml @@ -11,7 +11,7 @@ 1.0-SNAPSHOT 3.1.5-SNAPSHOT - 1.9 + 1.10-SNAPSHOT @@ -24,11 +24,23 @@ gov.nsa.datawave.webservices datawave-ws-client ${version.datawave} + + + * + org.slf4j + + gov.nsa.datawave.webservices datawave-ws-common ${version.datawave} + + + * + org.slf4j + + diff --git a/query-microservices/query/service/pom.xml b/query-microservices/query/service/pom.xml index 575dabd5..f5c64542 100644 --- a/query-microservices/query/service/pom.xml +++ b/query-microservices/query/service/pom.xml @@ -11,6 +11,7 @@ 1.0-SNAPSHOT DATAWAVE Query Microservice + datawave.microservice.query.QueryService 3.1.5-SNAPSHOT 1.0-SNAPSHOT 1.0-SNAPSHOT @@ -24,6 +25,12 @@ gov.nsa.datawave datawave-query-core ${version.datawave} + + + * + org.slf4j + + gov.nsa.datawave.microservice @@ -55,20 +62,9 @@ datawave-ws-client ${version.datawave} - - gov.nsa.datawave.microservice - spring-boot-starter-datawave - ${version.microservice.starter} - pom - import - - - gov.nsa.datawave - datawave-query-core - gov.nsa.datawave.microservice query @@ -104,6 +100,11 @@ junit-vintage-engine test + + org.springframework.cloud + spring-cloud-stream-test-support + test + diff --git a/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java b/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java index e99d3c89..d43d266a 100644 --- a/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java +++ b/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java @@ -34,7 +34,6 @@ @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@ContextConfiguration(classes = QueryServiceTest.QueryServiceTestConfiguration.class) @ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceTest { diff --git a/query-microservices/query/service/src/test/resources/config/application.yml b/query-microservices/query/service/src/test/resources/config/application.yml index 344f0ae1..9f138ff5 100644 --- a/query-microservices/query/service/src/test/resources/config/application.yml +++ b/query-microservices/query/service/src/test/resources/config/application.yml @@ -128,7 +128,7 @@ management: logging: level: - datawave.microservice.query-api: DEBUG + datawave.microservice: DEBUG io.undertow.request: FATAL audit-client: From 9ab7332e533bcd444924343282ea5689721b96d6 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Fri, 16 Apr 2021 18:05:43 +0000 Subject: [PATCH 028/218] Made a little more headway into determining when a range's results have been completely returned. This is required to appropriately return checkpoints when needed. --- .../query/configuration/QueryData.java | 8 ++++++++ .../query/configuration/Result.java | 18 +++++++++++++++--- .../query/configuration/ResultContext.java | 7 +++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/QueryData.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/QueryData.java index d0178d65..f17d9e4f 100644 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/QueryData.java +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/QueryData.java @@ -22,6 +22,7 @@ public class QueryData implements ResultContext { Collection ranges = Sets.newHashSet(); Collection columnFamilies = Sets.newHashSet(); Map.Entry lastResult; + boolean finished = false; public QueryData() {} @@ -81,6 +82,13 @@ public void addIterator(IteratorSetting cfg) { public void setLastResult(Map.Entry result) { this.lastResult = result; + if (this.lastResult == null) { + this.finished = true; + } + } + + public boolean isFinished() { + return this.finished; } public Map.Entry getLastResult() { diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/Result.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/Result.java index ff9c4571..7ae4b197 100644 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/Result.java +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/Result.java @@ -1,6 +1,7 @@ package datawave.microservice.query.configuration; import com.google.common.base.Function; +import com.google.common.base.Predicate; import com.google.common.collect.Iterators; import com.google.common.collect.Maps; import org.apache.accumulo.core.data.Key; @@ -61,7 +62,7 @@ public int hashCode() { } public Map.Entry returnKeyValue() { - Map.Entry entry = Maps.immutableEntry(key, value); + Map.Entry entry = (key == null ? null : Maps.immutableEntry(key, value)); if (context != null) { context.setLastResult(entry); } @@ -69,7 +70,7 @@ public Map.Entry returnKeyValue() { } public static Iterator> keyValueIterator(Iterator it) { - return Iterators.transform(it, new Function>() { + return Iterators.filter(Iterators.transform(it, new Function>() { @Override public Map.Entry apply(@Nullable Result input) { if (input == null) { @@ -77,11 +78,16 @@ public Map.Entry apply(@Nullable Result input) { } return input.returnKeyValue(); } + }), new Predicate>() { + @Override + public boolean apply(Map.@org.checkerframework.checker.nullness.qual.Nullable Entry keyValueEntry) { + return keyValueEntry != null; + } }); } public static Iterator resultIterator(final ResultContext context, Iterator> it) { - return Iterators.transform(it, new Function,Result>() { + return Iterators.filter(Iterators.transform(it, new Function,Result>() { @Nullable @Override public Result apply(@Nullable Map.Entry keyValueEntry) { @@ -90,6 +96,12 @@ public Result apply(@Nullable Map.Entry keyValueEntry) { } return new Result(context, keyValueEntry.getKey(), keyValueEntry.getValue()); } + }), new Predicate() { + + @Override + public boolean apply(@org.checkerframework.checker.nullness.qual.Nullable Result result) { + return result != null; + } }); } } diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/ResultContext.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/ResultContext.java index 962eb092..22efba4b 100644 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/ResultContext.java +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/ResultContext.java @@ -6,5 +6,12 @@ import java.util.Map; public interface ResultContext { + /** + * Set the last result returned. Setting a result of null denotes this scan is finished. + * + * @param result + */ void setLastResult(Map.Entry result); + + boolean isFinished(); } From bdfa8db57da370e69ac915f0d750ce629cf5da26 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 16 Apr 2021 16:12:31 -0400 Subject: [PATCH 029/218] Added the ability to load the default QueryLogicFactory.xml or setup an override. Added the ability to import configuration overrides from a local file. --- .../microservice/query/QueryService.java | 3 +-- .../src/main/resources/config/bootstrap.yml | 3 +++ .../src/test/resources/QueryLogicFactory.xml | 26 +++++++++++++++++++ .../test/resources/QueryLogicFactoryAlt.xml | 26 +++++++++++++++++++ .../src/test/resources/config/application.yml | 8 ++++++ 5 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 query-microservices/query/service/src/test/resources/QueryLogicFactory.xml create mode 100644 query-microservices/query/service/src/test/resources/QueryLogicFactoryAlt.xml diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryService.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryService.java index f2c3ae30..139f2a13 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryService.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryService.java @@ -13,8 +13,7 @@ */ @EnableDiscoveryClient @SpringBootApplication(scanBasePackages = "datawave.microservice", exclude = {ErrorMvcAutoConfiguration.class}) -// TODO: import the other query logics -// @ImportResource("classpath:QueryLogicFactory.xml") +@ImportResource("${queryLogicFactoryLocation:classpath:QueryLogicFactory.xml}") public class QueryService { @Bean diff --git a/query-microservices/query/service/src/main/resources/config/bootstrap.yml b/query-microservices/query/service/src/main/resources/config/bootstrap.yml index f049a2f5..2f8ddb10 100644 --- a/query-microservices/query/service/src/main/resources/config/bootstrap.yml +++ b/query-microservices/query/service/src/main/resources/config/bootstrap.yml @@ -1,6 +1,9 @@ spring: application: name: query + config: + # Locations to check for configuration overrides + import: "optional:file:/etc/config/override[.yml],optional:file:/etc/config/override[.properties]" cloud: config: # Disable consul-first config by default. We'll turn it back on in the consul profile if that profile is enabled. diff --git a/query-microservices/query/service/src/test/resources/QueryLogicFactory.xml b/query-microservices/query/service/src/test/resources/QueryLogicFactory.xml new file mode 100644 index 00000000..3bb53db7 --- /dev/null +++ b/query-microservices/query/service/src/test/resources/QueryLogicFactory.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + diff --git a/query-microservices/query/service/src/test/resources/QueryLogicFactoryAlt.xml b/query-microservices/query/service/src/test/resources/QueryLogicFactoryAlt.xml new file mode 100644 index 00000000..66b10066 --- /dev/null +++ b/query-microservices/query/service/src/test/resources/QueryLogicFactoryAlt.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + diff --git a/query-microservices/query/service/src/test/resources/config/application.yml b/query-microservices/query/service/src/test/resources/config/application.yml index 9f138ff5..1ce8c239 100644 --- a/query-microservices/query/service/src/test/resources/config/application.yml +++ b/query-microservices/query/service/src/test/resources/config/application.yml @@ -13,6 +13,9 @@ spring: hazelcast.client.enabled: false +# Uncomment the following line to override the query logic factory being used +# queryLogicFactoryLocation: "classpath:QueryLogicFactoryAlt.xml" + query: storage: backend: @@ -25,7 +28,12 @@ query: true logic: + baseQueryLogic: + logicName: "BaseQueryLogic" + eventQuery: + logicName: "EventQuery" baseEventQuery: + logicName: "BaseEventQuery" accumuloPassword: "value" tableName: "value" dateIndexTableName: "value" From 2e2640fc323ebda398694357a8aa92182a3ca468 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Tue, 20 Apr 2021 20:46:59 -0400 Subject: [PATCH 030/218] Fix versions to work with the 3.2 merge --- query-microservices/query/query/pom.xml | 2 +- query-microservices/query/service/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/query-microservices/query/query/pom.xml b/query-microservices/query/query/pom.xml index 88e94aad..dac2de1f 100644 --- a/query-microservices/query/query/pom.xml +++ b/query-microservices/query/query/pom.xml @@ -10,7 +10,7 @@ query 1.0-SNAPSHOT - 3.1.5-SNAPSHOT + 3.2.1-SNAPSHOT 1.10-SNAPSHOT diff --git a/query-microservices/query/service/pom.xml b/query-microservices/query/service/pom.xml index f5c64542..66811969 100644 --- a/query-microservices/query/service/pom.xml +++ b/query-microservices/query/service/pom.xml @@ -12,7 +12,7 @@ DATAWAVE Query Microservice datawave.microservice.query.QueryService - 3.1.5-SNAPSHOT + 3.2.1-SNAPSHOT 1.0-SNAPSHOT 1.0-SNAPSHOT 1.0-SNAPSHOT From 4fa021992bcc2ab18b7e6f42242b27e5373c1b44 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Wed, 21 Apr 2021 17:43:54 -0400 Subject: [PATCH 031/218] Got the query checkpoint and restart working. See the QueryLogicTestHarness for an example on how it is run. --- .../query/configuration/QueryData.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/QueryData.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/QueryData.java index f17d9e4f..83e226a8 100644 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/QueryData.java +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/QueryData.java @@ -2,6 +2,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import java.util.Collections; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Range; @@ -64,6 +65,19 @@ public void setQuery(String query) { } public Collection getRanges() { + if (isFinished()) { + return Collections.emptySet(); + } else if (lastResult != null) { + List newRanges = new ArrayList<>(); + for (Range range : ranges) { + if (range.contains(lastResult.getKey())) { + newRanges.add(new Range(lastResult.getKey(), false, range.getEndKey(), range.isEndKeyInclusive())); + } else { + newRanges.add(range); + } + } + return newRanges; + } return ranges; } From 56752db32ab40a348236b3534708e828352d43f5 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 23 Apr 2021 15:44:33 -0400 Subject: [PATCH 032/218] WIP: Working on enabling QueryLogicFactory configuration via spring properties. --- .../microservice/query/QueryParameters.java | 12 + .../query/QueryParametersImpl.java | 49 +- .../query/config/QueryProperties.java | 57 +- query-microservices/query/pom.xml | 2 +- query-microservices/query/query/pom.xml | 10 + .../query/logic/BaseQueryLogic.java | 44 +- .../microservice/query/logic/QueryLogic.java | 30 +- .../query/logic/QueryLogicFactory.java | 23 + .../microservice/query/logic/RoleManager.java | 14 - .../event/DefaultResponseObjectFactory.java | 117 +++ query-microservices/query/service/pom.xml | 10 + .../microservice/query/QueryController.java | 28 +- .../query/QueryManagementService.java | 244 ++++-- .../microservice/query/QueryService.java | 11 - .../query/config/QueryServiceConfig.java | 24 +- .../query/web/BaseQueryResponseAdvice.java | 2 +- .../microservice/query/web/QueryMetrics.java | 2 +- .../query/web/QuerySessionIdAdvice.java | 4 +- .../QueryMetricsEnrichmentFilterAdvice.java | 32 +- .../microservice/query/QueryServiceTest.java | 2 + .../resources/MyTestQueryLogicFactory.xml | 784 ++++++++++++++++++ .../src/test/resources/QueryLogicFactory.xml | 26 - .../test/resources/QueryLogicFactoryAlt.xml | 11 +- .../src/test/resources/config/application.yml | 213 +++-- 24 files changed, 1453 insertions(+), 298 deletions(-) create mode 100644 query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogicFactory.java delete mode 100644 query-microservices/query/query/src/main/java/datawave/microservice/query/logic/RoleManager.java create mode 100644 query-microservices/query/query/src/main/java/datawave/microservice/query/result/event/DefaultResponseObjectFactory.java create mode 100644 query-microservices/query/service/src/test/resources/MyTestQueryLogicFactory.xml delete mode 100644 query-microservices/query/service/src/test/resources/QueryLogicFactory.xml diff --git a/query-microservices/query/api/src/main/java/datawave/microservice/query/QueryParameters.java b/query-microservices/query/api/src/main/java/datawave/microservice/query/QueryParameters.java index 0a422bdb..02345c48 100644 --- a/query-microservices/query/api/src/main/java/datawave/microservice/query/QueryParameters.java +++ b/query-microservices/query/api/src/main/java/datawave/microservice/query/QueryParameters.java @@ -25,6 +25,8 @@ public interface QueryParameters extends ParameterValidator { String QUERY_PARAMS = "params"; String QUERY_VISIBILITY = "columnVisibility"; String QUERY_LOGIC_NAME = "logicName"; + String QUERY_POOL = "pool"; + String QUERY_MAX_CONCURRENT_TASKS = "maxConcurrentTasks"; String getQuery(); @@ -80,6 +82,16 @@ public interface QueryParameters extends ParameterValidator { void setLogicName(String logicName); + String getPool(); + + void setPool(String pool); + + int getMaxConcurrentTasks(); + + void setMaxConcurrentTasks(int maxConcurrentTasks); + + boolean isMaxConcurrentTasksOverridden(); + MultiValueMap getRequestHeaders(); void setRequestHeaders(MultiValueMap requestHeaders); diff --git a/query-microservices/query/api/src/main/java/datawave/microservice/query/QueryParametersImpl.java b/query-microservices/query/api/src/main/java/datawave/microservice/query/QueryParametersImpl.java index e0695cd7..e22c5f67 100644 --- a/query-microservices/query/api/src/main/java/datawave/microservice/query/QueryParametersImpl.java +++ b/query-microservices/query/api/src/main/java/datawave/microservice/query/QueryParametersImpl.java @@ -14,11 +14,12 @@ import java.util.Date; import java.util.List; import java.util.Map; +import java.util.Objects; public class QueryParametersImpl implements QueryParameters { private static final List KNOWN_PARAMS = Arrays.asList(QUERY_STRING, QUERY_NAME, QUERY_PERSISTENCE, QUERY_PAGESIZE, QUERY_PAGETIMEOUT, - QUERY_AUTHORIZATIONS, QUERY_EXPIRATION, QUERY_TRACE, QUERY_BEGIN, QUERY_END, QUERY_VISIBILITY, QUERY_LOGIC_NAME, + QUERY_AUTHORIZATIONS, QUERY_EXPIRATION, QUERY_TRACE, QUERY_BEGIN, QUERY_END, QUERY_VISIBILITY, QUERY_LOGIC_NAME, QUERY_POOL, QUERY_MAX_RESULTS_OVERRIDE); protected String query; @@ -35,6 +36,9 @@ public class QueryParametersImpl implements QueryParameters { protected Date endDate; protected String visibility; protected String logicName; + protected String pool; + protected boolean isMaxConcurrentTasksOverridden; + protected int maxConcurrentTasks; protected MultiValueMap requestHeaders; public QueryParametersImpl() { @@ -88,6 +92,9 @@ public void validate(Map> parameters) throws IllegalArgument } else if (QUERY_MAX_RESULTS_OVERRIDE.equals(param)) { this.maxResultsOverride = Long.parseLong(values.get(0)); this.isMaxResultsOverridden = true; + } else if (QUERY_MAX_CONCURRENT_TASKS.equals(param)) { + this.maxConcurrentTasks = Integer.parseInt(values.get(0)); + this.isMaxConcurrentTasksOverridden = true; } else if (QUERY_AUTHORIZATIONS.equals(param)) { // ensure that auths are comma separated with no empty values or spaces Splitter splitter = Splitter.on(',').omitEmptyStrings().trimResults(); @@ -116,6 +123,8 @@ public void validate(Map> parameters) throws IllegalArgument this.visibility = values.get(0); } else if (QUERY_LOGIC_NAME.equals(param)) { this.logicName = values.get(0); + } else if (QUERY_POOL.equals(param)) { + this.pool = values.get(0); } else { throw new IllegalArgumentException("Unknown condition."); } @@ -152,6 +161,10 @@ public boolean equals(Object o) { if (maxResultsOverride != that.maxResultsOverride) return false; } + if (isMaxConcurrentTasksOverridden != that.isMaxConcurrentTasksOverridden) + return false; + if (maxConcurrentTasks != that.maxConcurrentTasks) + return false; if (trace != that.trace) return false; if (!auths.equals(that.auths)) @@ -166,6 +179,8 @@ public boolean equals(Object o) { return false; if (logicName != null ? !logicName.equals(that.logicName) : that.logicName != null) return false; + if (pool != null ? !pool.equals(that.pool) : that.pool != null) + return false; if (persistenceMode != that.persistenceMode) return false; if (!query.equals(that.query)) @@ -187,6 +202,9 @@ public int hashCode() { if (isMaxResultsOverridden) { result = 31 * result + (int) (maxResultsOverride); } + if (isMaxConcurrentTasksOverridden) { + result = 31 * result + maxConcurrentTasks; + } result = 31 * result + auths.hashCode(); result = 31 * result + expirationDate.hashCode(); result = 31 * result + (trace ? 1 : 0); @@ -194,6 +212,7 @@ public int hashCode() { result = 31 * result + (endDate != null ? endDate.hashCode() : 0); result = 31 * result + (visibility != null ? visibility.hashCode() : 0); result = 31 * result + (logicName != null ? logicName.hashCode() : 0); + result = 31 * result + (pool != null ? pool.hashCode() : 0); result = 31 * result + (requestHeaders != null ? requestHeaders.hashCode() : 0); return result; } @@ -483,6 +502,31 @@ public void setLogicName(String logicName) { this.logicName = logicName; } + @Override + public String getPool() { + return pool; + } + + @Override + public void setPool(String pool) { + this.pool = pool; + } + + @Override + public int getMaxConcurrentTasks() { + return maxConcurrentTasks; + } + + @Override + public void setMaxConcurrentTasks(int maxConcurrentTasks) { + this.maxConcurrentTasks = maxConcurrentTasks; + } + + @Override + public boolean isMaxConcurrentTasksOverridden() { + return false; + } + @Override public MultiValueMap getRequestHeaders() { return requestHeaders; @@ -521,6 +565,9 @@ public void clear() { this.endDate = null; this.visibility = null; this.logicName = null; + this.pool = null; + this.isMaxConcurrentTasksOverridden = false; + this.maxConcurrentTasks = 0; this.requestHeaders = null; } } diff --git a/query-microservices/query/api/src/main/java/datawave/microservice/query/config/QueryProperties.java b/query-microservices/query/api/src/main/java/datawave/microservice/query/config/QueryProperties.java index 28e57e44..94438f93 100644 --- a/query-microservices/query/api/src/main/java/datawave/microservice/query/config/QueryProperties.java +++ b/query-microservices/query/api/src/main/java/datawave/microservice/query/config/QueryProperties.java @@ -2,25 +2,20 @@ import org.springframework.validation.annotation.Validated; +import javax.annotation.Nonnegative; import javax.validation.Valid; -import java.util.Map; +import javax.validation.constraints.NotEmpty; @Validated public class QueryProperties { - @Valid - private Map logic; - @Valid private QueryExpirationProperties expiration; - public Map getLogic() { - return logic; - } + @NotEmpty + private String privilegedRole = "PrivilegedUser"; - public void setLogic(Map logic) { - this.logic = logic; - } + private DefaultParameters defaultParams = new DefaultParameters(); public QueryExpirationProperties getExpiration() { return expiration; @@ -29,4 +24,46 @@ public QueryExpirationProperties getExpiration() { public void setExpiration(QueryExpirationProperties expiration) { this.expiration = expiration; } + + public String getPrivilegedRole() { + return privilegedRole; + } + + public void setPrivilegedRole(String privilegedRole) { + this.privilegedRole = privilegedRole; + } + + public DefaultParameters getDefaultParams() { + return defaultParams; + } + + public void setDefaultParams(DefaultParameters defaultParams) { + this.defaultParams = defaultParams; + } + + @Validated + public static class DefaultParameters { + + @NotEmpty + private String pool = "unassigned"; + + @Nonnegative + private int maxConcurrentTasks = 10; + + public String getPool() { + return pool; + } + + public void setPool(String pool) { + this.pool = pool; + } + + public int getMaxConcurrentTasks() { + return maxConcurrentTasks; + } + + public void setMaxConcurrentTasks(int maxConcurrentTasks) { + this.maxConcurrentTasks = maxConcurrentTasks; + } + } } diff --git a/query-microservices/query/pom.xml b/query-microservices/query/pom.xml index 321f066b..7e74c8e2 100644 --- a/query-microservices/query/pom.xml +++ b/query-microservices/query/pom.xml @@ -11,8 +11,8 @@ 1.0-SNAPSHOT pom - query api + query service diff --git a/query-microservices/query/query/pom.xml b/query-microservices/query/query/pom.xml index dac2de1f..df86f1a1 100644 --- a/query-microservices/query/query/pom.xml +++ b/query-microservices/query/query/pom.xml @@ -12,6 +12,7 @@ 3.2.1-SNAPSHOT 1.10-SNAPSHOT + 1.3-SNAPSHOT @@ -20,6 +21,11 @@ audit-api ${version.microservice.audit-api} + + gov.nsa.datawave.microservice + base-rest-responses + ${version.microservice.base-rest-responses} + gov.nsa.datawave.webservices datawave-ws-client @@ -49,6 +55,10 @@ gov.nsa.datawave.microservice audit-api + + gov.nsa.datawave.microservice + base-rest-responses + gov.nsa.datawave.webservices datawave-ws-client diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/BaseQueryLogic.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/BaseQueryLogic.java index 68607e62..749e4730 100644 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/BaseQueryLogic.java +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/BaseQueryLogic.java @@ -15,7 +15,7 @@ import org.springframework.beans.factory.annotation.Required; import javax.inject.Inject; -import java.security.Principal; +import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -30,6 +30,7 @@ public abstract class BaseQueryLogic implements QueryLogic { private AuditType auditType = null; private Map dnResultLimits = null; protected long maxResults = -1L; + protected int maxConcurrentTasks = -1; protected ScannerBase scanner; @SuppressWarnings("unchecked") protected Iterator iterator = (Iterator) Collections.emptyList().iterator(); @@ -38,10 +39,8 @@ public abstract class BaseQueryLogic implements QueryLogic { private boolean collectQueryMetrics = true; private String _connPoolName; private Set authorizedDNs; - protected Principal principal; - protected RoleManager roleManager; + protected Set requiredRoles; protected MarkingFunctions markingFunctions; - @Inject protected ResponseObjectFactory responseObjectFactory; protected SelectorExtractor selectorExtractor; @@ -72,8 +71,7 @@ public BaseQueryLogic(BaseQueryLogic other) { setPageByteTrigger(other.getPageByteTrigger()); setCollectQueryMetrics(other.getCollectQueryMetrics()); setConnPoolName(other.getConnPoolName()); - setPrincipal(other.getPrincipal()); - setRoleManager(other.getRoleManager()); + setRequiredRoles(other.getRequiredRoles()); setSelectorExtractor(other.getSelectorExtractor()); } @@ -108,12 +106,12 @@ public void setResponseObjectFactory(ResponseObjectFactory responseObjectFactory this.responseObjectFactory = responseObjectFactory; } - public Principal getPrincipal() { - return principal; + public Set getRequiredRoles() { + return requiredRoles; } - public void setPrincipal(Principal principal) { - this.principal = principal; + public void setRequiredRoles(Set requiredRoles) { + this.requiredRoles = requiredRoles; } @Override @@ -126,6 +124,11 @@ public long getMaxResults() { return this.maxResults; } + @Override + public int getMaxConcurrentTasks() { + return this.maxConcurrentTasks; + } + @Override @Deprecated public long getMaxRowsToScan() { @@ -147,6 +150,11 @@ public void setMaxResults(long maxResults) { this.maxResults = maxResults; } + @Override + public void setMaxConcurrentTasks(int maxConcurrentTasks) { + this.maxConcurrentTasks = maxConcurrentTasks; + } + @Override @Deprecated public void setMaxRowsToScan(long maxRowsToScan) { @@ -265,14 +273,6 @@ public void setCollectQueryMetrics(boolean collectQueryMetrics) { this.collectQueryMetrics = collectQueryMetrics; } - public RoleManager getRoleManager() { - return roleManager; - } - - public void setRoleManager(RoleManager roleManager) { - this.roleManager = roleManager; - } - /** {@inheritDoc} */ @Override public String getConnPoolName() { @@ -285,13 +285,9 @@ public void setConnPoolName(final String connPoolName) { _connPoolName = connPoolName; } - public boolean canRunQuery() { - return this.canRunQuery(this.getPrincipal()); - } - /** {@inheritDoc} */ - public boolean canRunQuery(Principal principal) { - return this.roleManager == null || this.roleManager.canRunQuery(this, principal); + public boolean canRunQuery(Collection userRoles) { + return this.requiredRoles == null || userRoles.containsAll(requiredRoles); } @Override diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogic.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogic.java index 4598129a..aa2f32a9 100644 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogic.java +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogic.java @@ -2,12 +2,12 @@ import datawave.audit.SelectorExtractor; import datawave.marking.MarkingFunctions; +import datawave.microservice.query.configuration.GenericQueryConfiguration; import datawave.validation.ParameterValidator; import datawave.webservice.common.audit.Auditor.AuditType; import datawave.webservice.common.connection.AccumuloConnectionFactory; import datawave.webservice.query.Query; import datawave.webservice.query.cache.ResultsPage; -import datawave.microservice.query.configuration.GenericQueryConfiguration; import datawave.webservice.query.exception.DatawaveErrorCode; import datawave.webservice.query.exception.QueryException; import datawave.webservice.query.result.event.ResponseObjectFactory; @@ -16,7 +16,6 @@ import org.apache.accumulo.core.security.Authorizations; import org.apache.commons.collections4.iterators.TransformIterator; -import java.security.Principal; import java.util.Collection; import java.util.List; import java.util.Map; @@ -121,6 +120,11 @@ default String getResponseClass(Query query) throws QueryException { */ long getMaxResults(); + /** + * @return max number of concurrent tasks to run for this query + */ + int getMaxConcurrentTasks(); + /** * @return the results of getMaxWork */ @@ -161,6 +165,12 @@ default String getResponseClass(Query query) throws QueryException { */ void setMaxResults(long maxResults); + /** + * @param maxConcurrentTasks + * max number of concurrent tasks to run for this query + */ + void setMaxConcurrentTasks(int maxConcurrentTasks); + /** * @param maxRowsToScan * This is now deprecated and setMaxWork should be used instead. This is equivalent to setMaxWork. @@ -243,10 +253,6 @@ default String getResponseClass(Query query) throws QueryException { */ void setCollectQueryMetrics(boolean collectQueryMetrics); - void setRoleManager(RoleManager roleManager); - - RoleManager getRoleManager(); - /** * List of parameters that can be used in the 'params' parameter to Query/create * @@ -264,18 +270,16 @@ default String getResponseClass(Query query) throws QueryException { String getConnPoolName(); /** - * Check that the user has one of the required roles principal my be null when there is no intent to control access to QueryLogic + * Check that the user has one of the required roles. userRoles my be null when there is no intent to control access to QueryLogic * - * @param principal + * @param userRoles * @return true/false */ - boolean canRunQuery(Principal principal); - - boolean canRunQuery(); // uses member Principal + boolean canRunQuery(Collection userRoles); - void setPrincipal(Principal principal); + void setRequiredRoles(Set requiredRoles); - Principal getPrincipal(); + Set getRequiredRoles(); MarkingFunctions getMarkingFunctions(); diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogicFactory.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogicFactory.java new file mode 100644 index 00000000..14369567 --- /dev/null +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogicFactory.java @@ -0,0 +1,23 @@ +package datawave.microservice.query.logic; + +import datawave.webservice.query.exception.QueryException; + +import java.util.Collection; +import java.util.List; + +public interface QueryLogicFactory { + + /** + * + * @param name + * name of query logic + * @return new instance of QueryLogic class + * @throws IllegalArgumentException + * if query logic name does not exist + */ + QueryLogic getQueryLogic(String name, Collection userRoles) throws QueryException, IllegalArgumentException, CloneNotSupportedException; + + QueryLogic getQueryLogic(String name) throws QueryException, IllegalArgumentException, CloneNotSupportedException; + + List> getQueryLogicList(); +} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/RoleManager.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/RoleManager.java deleted file mode 100644 index 2606df41..00000000 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/RoleManager.java +++ /dev/null @@ -1,14 +0,0 @@ -package datawave.microservice.query.logic; - -import java.security.Principal; -import java.util.Set; - -public interface RoleManager { - - boolean canRunQuery(QueryLogic queryLogic, Principal principal); - - void setRequiredRoles(Set requiredRoles); - - Set getRequiredRoles(); - -} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/result/event/DefaultResponseObjectFactory.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/result/event/DefaultResponseObjectFactory.java new file mode 100644 index 00000000..9621ffb1 --- /dev/null +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/result/event/DefaultResponseObjectFactory.java @@ -0,0 +1,117 @@ +package datawave.microservice.query.result.event; + +import datawave.user.AuthorizationsListBase; +import datawave.user.DefaultAuthorizationsList; +import datawave.webservice.query.Query; +import datawave.webservice.query.QueryImpl; +import datawave.webservice.query.cachedresults.CacheableQueryRow; +import datawave.webservice.query.result.EdgeQueryResponseBase; +import datawave.webservice.query.result.edge.DefaultEdge; +import datawave.webservice.query.result.edge.EdgeBase; +import datawave.webservice.query.result.event.DefaultEvent; +import datawave.webservice.query.result.event.DefaultFacets; +import datawave.webservice.query.result.event.DefaultField; +import datawave.webservice.query.result.event.DefaultFieldCardinality; +import datawave.webservice.query.result.event.EventBase; +import datawave.webservice.query.result.event.FacetsBase; +import datawave.webservice.query.result.event.FieldBase; +import datawave.webservice.query.result.event.FieldCardinalityBase; +import datawave.webservice.query.result.event.ResponseObjectFactory; +import datawave.webservice.query.result.metadata.DefaultMetadataField; +import datawave.webservice.query.result.metadata.MetadataFieldBase; +import datawave.webservice.response.objects.DefaultKey; +import datawave.webservice.response.objects.KeyBase; +import datawave.webservice.result.DefaultEdgeQueryResponse; +import datawave.webservice.result.DefaultEventQueryResponse; +import datawave.webservice.result.EventQueryResponseBase; +import datawave.webservice.result.FacetQueryResponse; +import datawave.webservice.result.FacetQueryResponseBase; +import datawave.webservice.results.datadictionary.DataDictionaryBase; +import datawave.webservice.results.datadictionary.DefaultDataDictionary; +import datawave.webservice.results.datadictionary.DefaultDescription; +import datawave.webservice.results.datadictionary.DefaultFields; +import datawave.webservice.results.datadictionary.DescriptionBase; +import datawave.webservice.results.datadictionary.FieldsBase; + +public class DefaultResponseObjectFactory extends ResponseObjectFactory { + @Override + public EventBase getEvent() { + return new DefaultEvent(); + } + + @Override + public FieldBase getField() { + return new DefaultField(); + } + + @Override + public EventQueryResponseBase getEventQueryResponse() { + return new DefaultEventQueryResponse(); + } + + // TODO: JWO: Figure out how we're going to deal with cached results + @Override + public CacheableQueryRow getCacheableQueryRow() { + return null; + } + + @Override + public EdgeBase getEdge() { + return new DefaultEdge(); + } + + @Override + public EdgeQueryResponseBase getEdgeQueryResponse() { + return new DefaultEdgeQueryResponse(); + } + + @Override + public FacetQueryResponseBase getFacetQueryResponse() { + return new FacetQueryResponse(); + } + + @Override + public FacetsBase getFacets() { + return new DefaultFacets(); + } + + @Override + public FieldCardinalityBase getFieldCardinality() { + return new DefaultFieldCardinality(); + } + + @Override + public KeyBase getKey() { + return new DefaultKey(); + } + + @Override + public AuthorizationsListBase getAuthorizationsList() { + return new DefaultAuthorizationsList(); + } + + @Override + public Query getQueryImpl() { + return new QueryImpl(); + } + + @Override + public DataDictionaryBase getDataDictionary() { + return new DefaultDataDictionary(); + } + + @Override + public FieldsBase getFields() { + return new DefaultFields(); + } + + @Override + public DescriptionBase getDescription() { + return new DefaultDescription(); + } + + @Override + public MetadataFieldBase getMetadataField() { + return new DefaultMetadataField(); + } +} diff --git a/query-microservices/query/service/pom.xml b/query-microservices/query/service/pom.xml index 66811969..f318c06e 100644 --- a/query-microservices/query/service/pom.xml +++ b/query-microservices/query/service/pom.xml @@ -16,6 +16,7 @@ 1.0-SNAPSHOT 1.0-SNAPSHOT 1.0-SNAPSHOT + 1.0-SNAPSHOT 1.6.6-SNAPSHOT 1.3-SNAPSHOT @@ -57,6 +58,11 @@ spring-boot-starter-datawave-audit ${version.microservice.starter-audit} + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-query + ${version.microservice.query-starter} + gov.nsa.datawave.webservices datawave-ws-client @@ -85,6 +91,10 @@ gov.nsa.datawave.microservice spring-boot-starter-datawave-audit + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-query + junit junit diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java index c5f9392e..d7511d96 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java @@ -1,10 +1,14 @@ package datawave.microservice.query; import com.codahale.metrics.annotation.Timed; -import datawave.microservice.query.config.QueryProperties; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.common.storage.TaskKey; import datawave.microservice.query.web.annotation.EnrichQueryMetrics; import datawave.webservice.result.GenericResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; +import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.util.MultiValueMap; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; @@ -12,19 +16,14 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import java.util.UUID; - @RestController @RequestMapping(path = "/v1", produces = MediaType.APPLICATION_JSON_VALUE) public class QueryController { + private final Logger log = LoggerFactory.getLogger(this.getClass()); - private QueryProperties queryProperties; - private QueryParameters queryParameters; private QueryManagementService queryManagementService; - public QueryController(QueryProperties queryProperties, QueryParameters queryParameters, QueryManagementService queryManagementService) { - this.queryProperties = queryProperties; - this.queryParameters = queryParameters; + public QueryController(QueryManagementService queryManagementService) { this.queryManagementService = queryManagementService; } @@ -32,17 +31,12 @@ public QueryController(QueryProperties queryProperties, QueryParameters queryPar @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE) @RequestMapping(path = "{queryLogic}/define", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public GenericResponse define(@PathVariable(name = "queryLogic") String queryLogic, @RequestParam MultiValueMap parameters) { - // validate query parameters - - // persist the query w/ query id in the query cache - - // query tracing? - - // update query metrics (i.e. DEFINED) + public GenericResponse define(@PathVariable(name = "queryLogic") String queryLogicName, @RequestParam MultiValueMap parameters, + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + TaskKey taskKey = queryManagementService.define(queryLogicName, parameters, currentUser); GenericResponse resp = new GenericResponse<>(); - resp.setResult(UUID.randomUUID().toString()); + resp.setResult(taskKey.getQueryId().toString()); return resp; } diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 04a5e451..54876344 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -3,26 +3,35 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import datawave.marking.SecurityMarking; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.common.storage.QueryPool; +import datawave.microservice.common.storage.QueryStorageCache; +import datawave.microservice.common.storage.TaskKey; import datawave.microservice.query.config.QueryExpirationProperties; import datawave.microservice.query.config.QueryProperties; import datawave.microservice.query.logic.QueryLogic; +import datawave.microservice.query.logic.QueryLogicFactory; import datawave.microservice.query.util.QueryUtil; +import datawave.security.util.ProxiedEntityUtils; import datawave.webservice.common.audit.AuditParameters; +import datawave.webservice.common.audit.PrivateAuditConstants; +import datawave.webservice.common.exception.DatawaveWebApplicationException; +import datawave.webservice.common.exception.UnauthorizedException; +import datawave.webservice.query.Query; import datawave.webservice.query.exception.BadRequestQueryException; import datawave.webservice.query.exception.DatawaveErrorCode; import datawave.webservice.query.exception.QueryException; +import datawave.webservice.query.exception.UnauthorizedQueryException; +import datawave.webservice.query.result.event.ResponseObjectFactory; +import datawave.webservice.query.util.QueryUncaughtExceptionHandler; +import datawave.webservice.result.GenericResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import org.springframework.util.MultiValueMap; -import javax.ws.rs.core.HttpHeaders; -import java.security.Principal; import java.text.MessageFormat; -import java.util.Arrays; -import java.util.Collections; import java.util.List; -import java.util.Set; @Service public class QueryManagementService { @@ -31,34 +40,109 @@ public class QueryManagementService { private static final ObjectMapper mapper = new ObjectMapper(); private QueryProperties queryProperties; + + // Note: QueryParameters need to be request scoped private QueryParameters queryParameters; private SecurityMarking securityMarking; + private QueryLogicFactory queryLogicFactory; + private ResponseObjectFactory responseObjectFactory; + private QueryStorageCache queryStorageCache; - // TODO: Pull these from configuration instead + // TODO: JWO: Pull these from configuration instead private final int PAGE_TIMEOUT_MIN = 1; private final int PAGE_TIMEOUT_MAX = QueryExpirationProperties.PAGE_TIMEOUT_MIN_DEFAULT; - public QueryManagementService(QueryProperties queryProperties, QueryParameters queryParameters, SecurityMarking securityMarking) { + public QueryManagementService(QueryProperties queryProperties, QueryParameters queryParameters, SecurityMarking securityMarking, + QueryLogicFactory queryLogicFactory, ResponseObjectFactory responseObjectFactory, QueryStorageCache queryStorageCache) { this.queryProperties = queryProperties; this.queryParameters = queryParameters; this.securityMarking = securityMarking; + this.queryLogicFactory = queryLogicFactory; + this.responseObjectFactory = responseObjectFactory; + this.queryStorageCache = queryStorageCache; + } + + public TaskKey define(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + // validate query and get a query logic + QueryLogic queryLogic = validateQuery(queryLogicName, parameters, currentUser); + + String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + log.trace(userId + " has authorizations " + currentUser.getPrimaryUser().getAuths()); + + // set some audit parameters which are used internally + String userDn = currentUser.getPrimaryUser().getDn().subjectDN(); + setInternalAuditParameters(queryLogicName, userDn, parameters); + + try { + // persist the query w/ query id in the query storage cache + // TODO: JWO: storeQuery assumes that this is a 'create' call, but this is a 'define' call. + // @formatter:off + TaskKey taskKey = queryStorageCache.storeQuery( + new QueryPool(getPoolName()), + createQuery(queryLogicName, parameters, userDn, currentUser.getDNs()), + getMaxConcurrentTasks(queryLogic)); + // @formatter:on + + // TODO: JWO: Figure out how to make query tracing work with our new architecture. Datawave issue #1155 + + // TODO: JWO: Figure out how to make query metrics work with our new architecture. Datawave issue #1156 + + return taskKey; + } catch (DatawaveWebApplicationException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error storing query", e); + throw new BadRequestQueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR, e); + } + } + + protected String getPoolName() { + return (queryParameters.getPool() != null) ? queryParameters.getPool() : queryProperties.getDefaultParams().getPool(); } - // A few items that are cached by the validateQuery call - private static class QueryData { - QueryLogic logic = null; - Principal p = null; - Set proxyServers = null; - String userDn = null; - String userid = null; - List dnList = null; + protected int getMaxConcurrentTasks(QueryLogic queryLogic) { + // if there's an override, use it + if (queryParameters.isMaxConcurrentTasksOverridden()) { + return queryParameters.getMaxConcurrentTasks(); + } + // if the query logic has a limit, use it + else if (queryLogic.getMaxConcurrentTasks() > 0) { + return queryLogic.getMaxConcurrentTasks(); + } + // otherwise, use the configuration default + else { + return queryProperties.getDefaultParams().getMaxConcurrentTasks(); + } + } + + protected Query createQuery(String queryLogicName, MultiValueMap parameters, String userDn, List dnList) { + Query q = responseObjectFactory.getQueryImpl(); + q.initialize(userDn, dnList, queryLogicName, queryParameters, queryParameters.getUnknownParameters(parameters)); + q.setColumnVisibility(securityMarking.toColumnVisibilityString()); + q.setUncaughtExceptionHandler(new QueryUncaughtExceptionHandler()); + Thread.currentThread().setUncaughtExceptionHandler(q.getUncaughtExceptionHandler()); + return q; } /** * This method will provide some initial query validation for the define and create query calls. */ - private void validateQuery(String queryLogicName, MultiValueMap parameters, HttpHeaders httpHeaders) throws QueryException { + protected QueryLogic validateQuery(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) + throws QueryException { + // validate the query parameters + validateParameters(queryLogicName, parameters); + + // create the query logic, and perform query logic parameter validation + QueryLogic queryLogic = createQueryLogic(queryLogicName, currentUser); + validateQueryLogic(queryLogic, parameters, currentUser); + // validate the security markings + validateSecurityMarkings(parameters); + + return queryLogic; + } + + protected void validateParameters(String queryLogicName, MultiValueMap parameters) throws QueryException { // add query logic name to parameters parameters.add(QueryParameters.QUERY_LOGIC_NAME, queryLogicName); @@ -100,66 +184,76 @@ private void validateQuery(String queryLogicName, MultiValueMap p log.error("Invalid begin and/or end date: " + queryParameters.getBeginDate() + " - " + queryParameters.getEndDate()); throw new BadRequestQueryException(DatawaveErrorCode.BEGIN_DATE_AFTER_END_DATE); } + } + + protected QueryLogic createQueryLogic(String queryLogicName, ProxiedUserDetails currentUser) throws QueryException { + // will throw IllegalArgumentException if not defined + try { + return queryLogicFactory.getQueryLogic(queryLogicName, currentUser.getPrimaryUser().getRoles()); + } catch (Exception e) { + log.error("Failed to get query logic for " + queryLogicName, e); + throw new BadRequestQueryException(DatawaveErrorCode.QUERY_LOGIC_ERROR, e); + } + } + + protected void validateQueryLogic(QueryLogic queryLogic, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + queryLogic.validate(parameters); + + // always check against the max + if (queryLogic.getMaxPageSize() > 0 && queryParameters.getPagesize() > queryLogic.getMaxPageSize()) { + log.error("Invalid page size: " + queryParameters.getPagesize() + " vs " + queryLogic.getMaxPageSize()); + throw new BadRequestQueryException(DatawaveErrorCode.PAGE_SIZE_TOO_LARGE, MessageFormat.format("Max = {0}.", queryLogic.getMaxPageSize())); + } - // // TODO: Get this working! - // // will throw IllegalArgumentException if not defined - // try { - // qd.logic = queryLogicFactory.getQueryLogic(queryLogicName, ctx.getCallerPrincipal()); - // } catch (Exception e) { - // log.error("Failed to get query logic for " + queryLogicName, e); - // throw new BadRequestQueryException(DatawaveErrorCode.QUERY_LOGIC_ERROR, e); - // } - // qd.logic.validate(parameters); - // - // try { - // securityMarking.clear(); - // securityMarking.validate(parameters); - // } catch (IllegalArgumentException e) { - // log.error("Failed security markings validation", e); - // throw new BadRequestQueryException(DatawaveErrorCode.SECURITY_MARKING_CHECK_ERROR, e); - // } - // // Find out who/what called this method - // qd.proxyServers = null; - // qd.p = ctx.getCallerPrincipal(); - // qd.userDn = qd.p.getName(); - // qd.userid = qd.userDn; - // qd.dnList = Collections.singletonList(qd.userid); - // if (qd.p instanceof DatawavePrincipal) { - // DatawavePrincipal dp = (DatawavePrincipal) qd.p; - // qd.userid = dp.getShortName(); - // qd.userDn = dp.getUserDN().subjectDN(); - // String[] dns = dp.getDNs(); - // Arrays.sort(dns); - // qd.dnList = Arrays.asList(dns); - // qd.proxyServers = dp.getProxyServers(); - // } - // log.trace(qd.userid + " has authorizations " + ((qd.p instanceof DatawavePrincipal) ? ((DatawavePrincipal) qd.p).getAuthorizations() : "")); - // - // // always check against the max - // if (qd.logic.getMaxPageSize() > 0 && queryParameters.getPagesize() > qd.logic.getMaxPageSize()) { - // log.error("Invalid page size: " + queryParameters.getPagesize() + " vs " + qd.logic.getMaxPageSize()); - // throw new BadRequestQueryException(DatawaveErrorCode.PAGE_SIZE_TOO_LARGE, MessageFormat.format("Max = {0}.", qd.logic.getMaxPageSize())); - // } - // - // // validate the max results override relative to the max results on a query logic - // // privileged users however can set whatever they want - // if (queryParameters.isMaxResultsOverridden() && qd.logic.getMaxResults() >= 0) { - // if (!ctx.isCallerInRole(PRIVILEGED_USER)) { - // if (queryParameters.getMaxResultsOverride() < 0 || (qd.logic.getMaxResults() < queryParameters.getMaxResultsOverride())) { - // log.error("Invalid max results override: " + queryParameters.getMaxResultsOverride() + " vs " + qd.logic.getMaxResults()); - // throw new BadRequestQueryException(DatawaveErrorCode.INVALID_MAX_RESULTS_OVERRIDE); - // } - // } - // } - // - // // Set private audit-related parameters, stripping off any that the user might have passed in first. - // // These are parameters that aren't passed in by the user, but rather are computed from other sources. - // PrivateAuditConstants.stripPrivateParameters(parameters); - // parameters.add(PrivateAuditConstants.LOGIC_CLASS, queryLogicName); - // parameters.putSingle(PrivateAuditConstants.COLUMN_VISIBILITY, marking.toColumnVisibilityString()); - // parameters.add(PrivateAuditConstants.USER_DN, qd.userDn); - // - // return qd; + // If the user is not privileged, make sure they didn't exceed the limits for the following parameters + if (!currentUser.getPrimaryUser().getRoles().contains(queryProperties.getPrivilegedRole())) { + // validate the max results override relative to the max results on a query logic + // privileged users however can set whatever they want + if (queryParameters.isMaxResultsOverridden() && queryLogic.getMaxResults() >= 0) { + if (queryParameters.getMaxResultsOverride() < 0 || (queryLogic.getMaxResults() < queryParameters.getMaxResultsOverride())) { + log.error("Invalid max results override: " + queryParameters.getMaxResultsOverride() + " vs " + queryLogic.getMaxResults()); + throw new BadRequestQueryException(DatawaveErrorCode.INVALID_MAX_RESULTS_OVERRIDE); + } + } + + // validate the max concurrent tasks override relative to the max concurrent tasks on a query logic + // privileged users however can set whatever they want + if (queryParameters.isMaxConcurrentTasksOverridden() && queryLogic.getMaxConcurrentTasks() >= 0) { + if (queryParameters.getMaxConcurrentTasks() < 0 || (queryLogic.getMaxConcurrentTasks() < queryParameters.getMaxConcurrentTasks())) { + log.error("Invalid max concurrent tasks override: " + queryParameters.getMaxConcurrentTasks() + " vs " + + queryLogic.getMaxConcurrentTasks()); + throw new BadRequestQueryException(DatawaveErrorCode.INVALID_MAX_CONCURRENT_TASKS_OVERRIDE); + } + } + } + + // Verify that the calling principal has access to the query logic. + List dnList = currentUser.getDNs(); + if (!queryLogic.containsDNWithAccess(dnList)) { + UnauthorizedQueryException qe = new UnauthorizedQueryException("None of the DNs used have access to this query logic: " + dnList, 401); + GenericResponse response = new GenericResponse<>(); + response.addException(qe); + throw new UnauthorizedException(qe, response); + } + } + + protected void validateSecurityMarkings(MultiValueMap parameters) throws QueryException { + try { + securityMarking.clear(); + securityMarking.validate(parameters); + } catch (IllegalArgumentException e) { + log.error("Failed security markings validation", e); + throw new BadRequestQueryException(DatawaveErrorCode.SECURITY_MARKING_CHECK_ERROR, e); + } + } + + protected void setInternalAuditParameters(String queryLogicName, String userDn, MultiValueMap parameters) { + // Set private audit-related parameters, stripping off any that the user might have passed in first. + // These are parameters that aren't passed in by the user, but rather are computed from other sources. + PrivateAuditConstants.stripPrivateParameters(parameters); + parameters.add(PrivateAuditConstants.LOGIC_CLASS, queryLogicName); + parameters.set(PrivateAuditConstants.COLUMN_VISIBILITY, securityMarking.toColumnVisibilityString()); + parameters.add(PrivateAuditConstants.USER_DN, userDn); } private String writeValueAsString(Object object) { diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryService.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryService.java index 139f2a13..aaf85e33 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryService.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryService.java @@ -4,24 +4,13 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.web.servlet.error.ErrorMvcAutoConfiguration; import org.springframework.cloud.client.discovery.EnableDiscoveryClient; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ImportResource; -import org.springframework.web.context.annotation.RequestScope; /** * Launcher for the query service */ @EnableDiscoveryClient @SpringBootApplication(scanBasePackages = "datawave.microservice", exclude = {ErrorMvcAutoConfiguration.class}) -@ImportResource("${queryLogicFactoryLocation:classpath:QueryLogicFactory.xml}") public class QueryService { - - @Bean - @RequestScope - public QueryParameters queryParameters() { - return new QueryParametersImpl(); - } - public static void main(String[] args) { SpringApplication.run(QueryService.class, args); } diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java index 9513dce1..6cfb5fe6 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java @@ -1,15 +1,33 @@ package datawave.microservice.query.config; +import datawave.microservice.query.QueryParameters; +import datawave.microservice.query.QueryParametersImpl; +import datawave.microservice.query.result.event.DefaultResponseObjectFactory; +import datawave.webservice.query.result.event.ResponseObjectFactory; import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; - -import javax.validation.Valid; +import org.springframework.context.annotation.ImportResource; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.web.context.annotation.RequestScope; @Configuration public class QueryServiceConfig { + @Bean + @Order(Ordered.LOWEST_PRECEDENCE) + @RequestScope + public QueryParameters queryParameters() { + return new QueryParametersImpl(); + } + + @Bean + @Order(Ordered.LOWEST_PRECEDENCE) + public ResponseObjectFactory responseObjectFactory() { + return new DefaultResponseObjectFactory(); + } + @Bean @ConfigurationProperties("query") public QueryProperties queryProperties() { diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java index d09e70f0..4adbca10 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java @@ -11,7 +11,7 @@ import org.springframework.web.bind.annotation.ControllerAdvice; import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyAdvice; -// TODO: Should this be in the API or in a starter? +// TODO: JWO: Should this be in the API or in a starter? /** * A {@link ControllerAdvice} that implements {@link ResponseBodyAdvice} in order to allow access to {@link BaseQueryResponse} objects before they are written * out to the response body. This is primarily used to write the page number, is last page, and partial results headers for the response. diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java index 129998a5..fdac9df1 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java @@ -3,7 +3,7 @@ import datawave.webservice.query.metric.BaseQueryMetric; import org.springframework.stereotype.Component; -// TODO: Remove this placeholder and replace it with the real thing once it's ready. +// TODO: JWO: Remove this placeholder and replace it with the real thing once it's ready. @Component public class QueryMetrics { diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java index 707d6452..f473d0f9 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java @@ -19,8 +19,8 @@ import java.util.Arrays; import java.util.UUID; -// TODO: Update to enable based on properties -// TODO: Should this be in the API or in a starter? +// TODO: JWO: Update to enable based on properties +// TODO: JWO: Should this be in the API or in a starter? @ControllerAdvice public class QuerySessionIdAdvice implements ResponseBodyAdvice { private final Logger log = Logger.getLogger(QuerySessionIdAdvice.class); diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java index af7a0b25..6b78dabd 100644 --- a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java +++ b/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java @@ -2,7 +2,8 @@ import datawave.microservice.common.storage.QueryCache; import datawave.microservice.common.storage.QueryState; -import datawave.microservice.query.config.QueryProperties; +import datawave.microservice.common.storage.QueryStorageCache; +import datawave.microservice.query.logic.QueryLogicFactory; import datawave.microservice.query.web.QueryMetrics; import datawave.microservice.query.web.annotation.EnrichQueryMetrics; import datawave.webservice.query.metric.BaseQueryMetric; @@ -29,15 +30,15 @@ public class QueryMetricsEnrichmentFilterAdvice extends BaseMethodStatsFilter im private final Logger log = Logger.getLogger(this.getClass()); - private final QueryProperties queryProperties; + private final QueryLogicFactory queryLogicFactory; - private final QueryCache queryCache; + private final QueryStorageCache queryStorageCache; private final QueryMetrics queryMetrics; - public QueryMetricsEnrichmentFilterAdvice(QueryProperties queryProperties, QueryCache queryCache, QueryMetrics queryMetrics) { - this.queryProperties = queryProperties; - this.queryCache = queryCache; + public QueryMetricsEnrichmentFilterAdvice(QueryLogicFactory queryLogicFactory, QueryStorageCache queryStorageCache, QueryMetrics queryMetrics) { + this.queryLogicFactory = queryLogicFactory; + this.queryStorageCache = queryStorageCache; this.queryMetrics = queryMetrics; } @@ -89,11 +90,22 @@ public Object beforeBodyWrite(Object body, @NonNull MethodParameter returnType, public void postProcess(ResponseMethodStats responseStats) { String queryId = QueryMetricsEnrichmentContext.getQueryId(); if (queryId != null) { - if (queryCache != null) { - QueryState queryState = queryCache.getQuery(UUID.fromString(queryId)); - if (queryState != null && queryProperties.getLogic().get(queryState.getQueryLogic()).isMetricsEnabled()) { + if (queryStorageCache != null) { + // TODO: JWO: Need to figure out where the query metrics object is supposed to come from. I assumed + // that it would be contained in the QueryState similar to how it is stored in the RunningQuery, + // but that does not appear to be the case. + QueryState queryState = null; + + boolean isMetricsEnabled = false; + // try { + // isMetricsEnabled = queryLogicFactory.getQueryLogic(queryState.getQueryLogic()).getCollectQueryMetrics(); + // } catch (Exception e) { + // log.warn("Unable to determine if query logic '" + queryState.getQueryLogic() + "' supports metrics"); + // } + + if (queryState != null && isMetricsEnabled) { try { - // TODO: This probably shouldn't be instantiated here, and we also shouldn't hard code the basequerymetric implementation. + // TODO: JWO: This probably shouldn't be instantiated here, and we also shouldn't hard code the basequerymetric implementation. // just using this as a placeholder until we start to bring things together. BaseQueryMetric metric = new QueryMetric(); switch (QueryMetricsEnrichmentContext.getMethodType()) { diff --git a/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java b/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java index d43d266a..cf28f748 100644 --- a/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java +++ b/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java @@ -6,6 +6,7 @@ import datawave.security.authorization.DatawaveUser; import datawave.security.authorization.SubjectIssuerDNPair; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; @@ -55,6 +56,7 @@ public void setup() { } @Test + @Ignore public void testQuery() { Collection roles = Collections.singleton("AuthorizedUser"); DatawaveUser uathDWUser = new DatawaveUser(DN, USER, null, roles, null, System.currentTimeMillis()); diff --git a/query-microservices/query/service/src/test/resources/MyTestQueryLogicFactory.xml b/query-microservices/query/service/src/test/resources/MyTestQueryLogicFactory.xml new file mode 100644 index 00000000..a12f5656 --- /dev/null +++ b/query-microservices/query/service/src/test/resources/MyTestQueryLogicFactory.xml @@ -0,0 +1,784 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${query.logic.logics.BaseEventQuery.contentFieldNames} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2611 + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/query-microservices/query/service/src/test/resources/QueryLogicFactory.xml b/query-microservices/query/service/src/test/resources/QueryLogicFactory.xml deleted file mode 100644 index 3bb53db7..00000000 --- a/query-microservices/query/service/src/test/resources/QueryLogicFactory.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/query-microservices/query/service/src/test/resources/QueryLogicFactoryAlt.xml b/query-microservices/query/service/src/test/resources/QueryLogicFactoryAlt.xml index 66b10066..ad5d589c 100644 --- a/query-microservices/query/service/src/test/resources/QueryLogicFactoryAlt.xml +++ b/query-microservices/query/service/src/test/resources/QueryLogicFactoryAlt.xml @@ -2,14 +2,11 @@ + https://www.springframework.org/schema/beans/spring-beans.xsd + http://www.springframework.org/schema/lang + https://www.springframework.org/schema/lang/spring-lang.xsd"> diff --git a/query-microservices/query/service/src/test/resources/config/application.yml b/query-microservices/query/service/src/test/resources/config/application.yml index 1ce8c239..9cb0dc3a 100644 --- a/query-microservices/query/service/src/test/resources/config/application.yml +++ b/query-microservices/query/service/src/test/resources/config/application.yml @@ -13,8 +13,55 @@ spring: hazelcast.client.enabled: false -# Uncomment the following line to override the query logic factory being used -# queryLogicFactoryLocation: "classpath:QueryLogicFactoryAlt.xml" +warehouse: + accumulo: + zookeepers: 'localhost:2181' + instanceName: 'my-instance-01' + username: 'root' + password: 'password' + statsd: + host: localhost + port: 8125 + tables: + shard: + name: 'shard' + index: + name: 'shardIndex' + reverseIndex: + name: 'shardReverseIndex' + dateIndex: + name: 'dateIndex' + metadata: + name: 'DatawaveMetadata' + model: + name: 'DatawaveMetadata' + defaults: + queryThreads: 100 + indexLookupThreads: 100 + dateIndexThreads: 20 + fullTableScanEnabled: false + baseIteratorPriority: 100 + maxIndexScanTimeMillis: 31536000000 + eventPerDayThreshold: 40000 + shardsPerDayThreshold: 20 + maxTermThreshold: 2000 + maxDepthThreshold: 2000 + maxUnfieldedExpansionThreshold: 50 + maxValueExpansionThreshold: 50 + maxOrExpansionThreshold: 500 + maxOrRangeThreshold: 10 + maxOrExpansionFstThreshold: 750 + maxFieldIndexRangeSplit: 16 + maxIvaratorSources: 20 + maxEvaluationPipelines: 16 + maxPipelineCachedResults: 16 + hdfsSiteConfigURLs: 'file:///etc/hadoop/conf/core-site.xml,file:///etc/hadoop/conf/hdfs-site.xml' + ivaratorFstHdfsBaseURIs: 'hdfs:///IvaratorCache' + ivaratorCacheBufferSize: 10000 + ivaratorMaxOpenFiles: 100 + ivaratorCacheScanPersistThreshold: 100000 + ivaratorCacheScanTimeoutMinutes: 60 + modelName: 'DATAWAVE' query: storage: @@ -28,86 +75,88 @@ query: true logic: - baseQueryLogic: - logicName: "BaseQueryLogic" - eventQuery: - logicName: "EventQuery" - baseEventQuery: - logicName: "BaseEventQuery" - accumuloPassword: "value" - tableName: "value" - dateIndexTableName: "value" - dateIndexHelperFactory: "someRef" - defaultDateTypeName: "value" - metadataTableName: "value" - metadataHelperFactory: "someRef" - indexTableName: "value" - reverseIndexTableName: "value" - maxResults: "value" - queryThreads: "value" - indexLookupThreads: "value" - dateIndexThreads: "value" - fullTableScanEnabled: "value" - includeDataTypeAsField: "value" - disableIndexOnlyDocuments: "value" - indexOnlyFilterFunctionsEnabled: "value" - includeHierarchyFields: "value" - hierarchyFieldOptions: "someRef" - baseIteratorPriority: "value" - maxIndexScanTimeMillis: "value" - collapseUids: "value" - collapseUidsThreshold: "value" - useEnrichers: "true" - contentFieldNames: - - "CONTENT" - realmSuffixExclusionPatterns: - - "<.*>$" - minimumSelectivity: ".2" - enricherClassNames: - - "datawave.query.enrich.DatawaveTermFrequencyEnricher" - useFilters: "value" - filterClassNames: - - "value" - filterOptions: "someRef" - auditType: "NONE" - logicDescription: "Retrieve sharded events/documents, leveraging the global index tables as needed" - eventPerDayThreshold: "40000" - shardsPerDayThreshold: "value" - maxTermThreshold: "value" - maxDepthThreshold: "value" - maxUnfieldedExpansionThreshold: "value" - maxValueExpansionThreshold: "value" - maxOrExpansionThreshold: "value" - maxOrRangeThreshold: "value" - maxOrExpansionFstThreshold: "value" - maxFieldIndexRangeSplit: "value" - maxIvaratorSources: "value" - maxEvaluationPipelines: "value" - maxPipelineCachedResults: "value" - hdfsSiteConfigURLs: "value" - zookeeperConfig: "value" - ivaratorCacheDirConfigs: "someRef" - ivaratorFstHdfsBaseURIs: "value" - ivaratorCacheBufferSize: "10000" - ivaratorMaxOpenFiles: "value" - ivaratorCacheScanPersistThreshold: "100000" - ivaratorCacheScanTimeoutMinutes: "value" - eventQueryDatqeDecoratorTransformer: "someRef" - modelTableName: "value" - modelName: "DATAWAVE" - querySyntaxParsers: - "JEXL": null - "LUCENE": "value-ref" - "LUCENE-UUID": "value-ref" - "TOKENIZED-LUCENE": "value-ref" - queryPlanner: "someRef" - sendTimingToStatsd: "false" - collectQueryMetrics: "true" - logTimingDetails: "false" - statsdHost: "localhost" - statsdPort: "8125" - selectorExtractor: "someRef" - evaluationOnlyFields: "value" + factory: + enabled: true + # Uncomment the following line to override the query logic beans to load + xmlBeansPath: "classpath:MyTestQueryLogicFactory.xml" + + # The max page size that a user can request. 0 turns off this feature + maxPageSize: 10000 + + # The number of bytes at which a page will be returned, event if the pagesize has not been reached. 0 turns off this feature + pageByteTrigger: 0 + logics: + BaseEventQuery: + accumuloPassword: ${warehouse.accumulo.password} + tableName: ${warehouse.tables.shard.name} + dateIndexTableName: ${warehouse.tables.dateIndex.name} + defaultDateTypeName: "EVENT" + metadataTableName: ${warehouse.tables.metadata.name} + indexTableName: ${warehouse.tables.index.name} + reverseIndexTableName: ${warehouse.tables.reverseIndex.name} + maxResults: -1 + queryThreads: ${warehouse.defaults.queryThreads} + indexLookupThreads: ${warehouse.defaults.indexLookupThreads} + dateIndexThreads: ${warehouse.defaults.dateIndexThreads} + fullTableScanEnabled: ${warehouse.defaults.fullTableScanEnabled} + includeDataTypeAsField: false + disableIndexOnlyDocuments: false + indexOnlyFilterFunctionsEnabled: false + includeHierarchyFields: false + hierarchyFieldOptions: "" + baseIteratorPriority: ${warehouse.defaults.baseIteratorPriority} + maxIndexScanTimeMillis: ${warehouse.defaults.maxIndexScanTimeMillis} + collapseUids: false + collapseUidsThreshold: -1 + useEnrichers: true + contentFieldNames: + - 'CONTENT' + realmSuffixExclusionPatterns: '<.*>$' + minimumSelectivity: .2 + enricherClassNames: 'datawave.query.enrich.DatawaveTermFrequencyEnricher' + useFilters: false + filterClassNames: '' + filterOptions: '' + auditType: "NONE" + logicDescription: "Retrieve sharded events/documents, leveraging the global index tables as needed" + eventPerDayThreshold: ${warehouse.defaults.eventPerDayThreshold} + shardsPerDayThreshold: ${warehouse.defaults.shardsPerDayThreshold} + maxTermThreshold: ${warehouse.defaults.maxTermThreshold} + maxDepthThreshold: ${warehouse.defaults.maxDepthThreshold} + maxUnfieldedExpansionThreshold: ${warehouse.defaults.maxUnfieldedExpansionThreshold} + maxValueExpansionThreshold: ${warehouse.defaults.maxValueExpansionThreshold} + maxOrExpansionThreshold: ${warehouse.defaults.maxOrExpansionThreshold} + maxOrRangeThreshold: ${warehouse.defaults.maxOrRangeThreshold} + maxOrExpansionFstThreshold: ${warehouse.defaults.maxOrExpansionFstThreshold} + maxFieldIndexRangeSplit: ${warehouse.defaults.maxFieldIndexRangeSplit} + maxIvaratorSources: ${warehouse.defaults.maxIvaratorSources} + maxEvaluationPipelines: ${warehouse.defaults.maxEvaluationPipelines} + maxPipelineCachedResults: ${warehouse.defaults.maxPipelineCachedResults} + hdfsSiteConfigURLs: ${warehouse.defaults.hdfsSiteConfigURLs} + zookeeperConfig: ${warehouse.accumulo.zookeepers} + ivaratorCacheDirConfigs: '' + ivaratorFstHdfsBaseURIs: ${warehouse.accumulo.ivaratorFstHdfsBaseURIs} + ivaratorCacheBufferSize: ${warehouse.defaults.ivaratorCacheBufferSize} + ivaratorMaxOpenFiles: ${warehouse.defaults.ivaratorMaxOpenFiles} + ivaratorCacheScanPersistThreshold: ${warehouse.defaults.ivaratorCacheScanPersistThreshold} + ivaratorCacheScanTimeoutMinutes: ${warehouse.defaults.ivaratorCacheScanTimeoutMinutes} + eventQueryDataDecorators: '' + modelTableName: ${warehouse.tables.model.name} + modelName: ${warehouse.defaults.modelName} + querySyntaxParsers: |- + + + + + + + + sendTimingToStatsd: false + collectQueryMetrics: true + logTimingDetails: false + statsdHost: ${warehouse.statsd.host} + statsdPort: ${warehouse.statsd.port} + evaluationOnlyFields: "" server: port: 0 From cb774a067fc00a3bb754e1a3a268bd4cce670b98 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Mon, 26 Apr 2021 18:14:08 -0400 Subject: [PATCH 033/218] WIP: Trying to get QueryLogicFactory configured and working correctly via the spring boot datawave query starter. --- query-microservices/query/query/pom.xml | 25 +++ .../query/logic/CheckpointableQueryLogic.java | 45 ++++++ .../event/DefaultResponseObjectFactory.java | 2 + .../resources/MyTestQueryLogicFactory.xml | 145 ++++++++++-------- .../src/test/resources/config/application.yml | 62 ++++++-- 5 files changed, 201 insertions(+), 78 deletions(-) create mode 100644 query-microservices/query/query/src/main/java/datawave/microservice/query/logic/CheckpointableQueryLogic.java diff --git a/query-microservices/query/query/pom.xml b/query-microservices/query/query/pom.xml index df86f1a1..bf845784 100644 --- a/query-microservices/query/query/pom.xml +++ b/query-microservices/query/query/pom.xml @@ -13,9 +13,21 @@ 3.2.1-SNAPSHOT 1.10-SNAPSHOT 1.3-SNAPSHOT + 1.0-SNAPSHOT + + gov.nsa.datawave + datawave-ingest-core + ${version.datawave} + + + * + org.slf4j + + + gov.nsa.datawave.microservice audit-api @@ -26,6 +38,11 @@ base-rest-responses ${version.microservice.base-rest-responses} + + gov.nsa.datawave.microservice + query-storage-api + ${version.microservice.query-storage-api} + gov.nsa.datawave.webservices datawave-ws-client @@ -51,6 +68,10 @@ + + gov.nsa.datawave + datawave-ingest-core + gov.nsa.datawave.microservice audit-api @@ -59,6 +80,10 @@ gov.nsa.datawave.microservice base-rest-responses + + gov.nsa.datawave.microservice + query-storage-api + gov.nsa.datawave.webservices datawave-ws-client diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/CheckpointableQueryLogic.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/CheckpointableQueryLogic.java new file mode 100644 index 00000000..672660f3 --- /dev/null +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/CheckpointableQueryLogic.java @@ -0,0 +1,45 @@ +package datawave.microservice.query.logic; + +import datawave.microservice.common.storage.QueryCheckpoint; +import datawave.microservice.common.storage.QueryKey; +import org.apache.accumulo.core.client.Connector; + +import java.util.List; + +public interface CheckpointableQueryLogic { + + /** + * This will allow us to check if a query logic is actually checkpointable. Even if the query logic supports it, the caller may have to tell the query logic + * that it is going to be checkpointed. + */ + boolean isCheckpointable(); + + /** + * This will tell the query logic that is is going to be checkpointed. + * + * @param checkpointable + */ + void setCheckpointable(boolean checkpointable); + + /** + * This can be called at any point to get a checkpoint such that this query logic instance can be torn down to be rebuilt later. At a minimum this should be + * called after the getTransformIterator is depleted of results. + * + * @param queryKey + * - the query key to include in the checkpoint + * @return The query checkpoints + */ + List checkpoint(QueryKey queryKey); + + /** + * Implementations use the configuration to setup execution of a portion of their query. getTransformIterator should be used to get the partial results if + * any. + * + * @param connection + * - The accumulo connector + * @param checkpoint + * - Encapsulates all information needed to run a portion of the query. + */ + void setupQuery(Connector connection, QueryCheckpoint checkpoint) throws Exception; + +} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/result/event/DefaultResponseObjectFactory.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/result/event/DefaultResponseObjectFactory.java index 9621ffb1..20ad3f03 100644 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/result/event/DefaultResponseObjectFactory.java +++ b/query-microservices/query/query/src/main/java/datawave/microservice/query/result/event/DefaultResponseObjectFactory.java @@ -32,7 +32,9 @@ import datawave.webservice.results.datadictionary.DefaultFields; import datawave.webservice.results.datadictionary.DescriptionBase; import datawave.webservice.results.datadictionary.FieldsBase; +import org.springframework.stereotype.Component; +@Component("responseObjectFactory") public class DefaultResponseObjectFactory extends ResponseObjectFactory { @Override public EventBase getEvent() { diff --git a/query-microservices/query/service/src/test/resources/MyTestQueryLogicFactory.xml b/query-microservices/query/service/src/test/resources/MyTestQueryLogicFactory.xml index a12f5656..162eaa85 100644 --- a/query-microservices/query/service/src/test/resources/MyTestQueryLogicFactory.xml +++ b/query-microservices/query/service/src/test/resources/MyTestQueryLogicFactory.xml @@ -3,20 +3,15 @@ - - - + http://www.springframework.org/schema/util/spring-util.xsd"> + + @@ -46,39 +41,33 @@ - - + + - - - ${query.logic.logics.BaseEventQuery.contentFieldNames} - - - - - - - + + + + + + + - - - - - + + + + - - - - - - - + + + + + @@ -115,9 +104,9 @@ - + - + @@ -130,16 +119,14 @@ - - + + - - - - - + + + @@ -156,27 +143,6 @@ - - - - - - - - - - - - - - - - - - - - @@ -197,6 +163,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/query-microservices/query/service/src/test/resources/config/application.yml b/query-microservices/query/service/src/test/resources/config/application.yml index 9cb0dc3a..24f33d46 100644 --- a/query-microservices/query/service/src/test/resources/config/application.yml +++ b/query-microservices/query/service/src/test/resources/config/application.yml @@ -64,6 +64,23 @@ warehouse: modelName: 'DATAWAVE' query: + parser: + skipTokenizeUnfieldedFields: + - "DOMETA" + tokenizedFields: + - "CONTENT" + uuidTypes: + - fieldName: "EVENT_ID" + definedView: "LuceneUUIDEventQuery" + allowedWildcardAfter: 28 + - fieldName: "UUID" + definedView: "LuceneUUIDEventQuery" + - fieldName: "PARENT_UUID" + definedView: "LuceneUUIDEventQuery" + - fieldName: "PAGE_ID" + definedView: "LuceneUUIDEventQuery" + - fieldName: "PAGE_TITLE" + definedView: "LuceneUUIDEventQuery" storage: backend: LOCAL @@ -73,7 +90,6 @@ query: true sendNotifications: true - logic: factory: enabled: true @@ -103,7 +119,8 @@ query: disableIndexOnlyDocuments: false indexOnlyFilterFunctionsEnabled: false includeHierarchyFields: false - hierarchyFieldOptions: "" + hierarchyFieldOptions: + "FOO": "BAR" baseIteratorPriority: ${warehouse.defaults.baseIteratorPriority} maxIndexScanTimeMillis: ${warehouse.defaults.maxIndexScanTimeMillis} collapseUids: false @@ -111,12 +128,16 @@ query: useEnrichers: true contentFieldNames: - 'CONTENT' - realmSuffixExclusionPatterns: '<.*>$' + realmSuffixExclusionPatterns: + - '<.*>$' minimumSelectivity: .2 - enricherClassNames: 'datawave.query.enrich.DatawaveTermFrequencyEnricher' + enricherClassNames: + - 'datawave.query.enrich.DatawaveTermFrequencyEnricher' useFilters: false - filterClassNames: '' - filterOptions: '' + filterClassNames: + - 'foo.bar' + filterOptions: + - 'bar.foo' auditType: "NONE" logicDescription: "Retrieve sharded events/documents, leveraging the global index tables as needed" eventPerDayThreshold: ${warehouse.defaults.eventPerDayThreshold} @@ -134,23 +155,32 @@ query: maxPipelineCachedResults: ${warehouse.defaults.maxPipelineCachedResults} hdfsSiteConfigURLs: ${warehouse.defaults.hdfsSiteConfigURLs} zookeeperConfig: ${warehouse.accumulo.zookeepers} - ivaratorCacheDirConfigs: '' + ivaratorCacheDirConfigs: + - basePathURI: "hdfs:///IvaratorCache" ivaratorFstHdfsBaseURIs: ${warehouse.accumulo.ivaratorFstHdfsBaseURIs} ivaratorCacheBufferSize: ${warehouse.defaults.ivaratorCacheBufferSize} ivaratorMaxOpenFiles: ${warehouse.defaults.ivaratorMaxOpenFiles} ivaratorCacheScanPersistThreshold: ${warehouse.defaults.ivaratorCacheScanPersistThreshold} ivaratorCacheScanTimeoutMinutes: ${warehouse.defaults.ivaratorCacheScanTimeoutMinutes} - eventQueryDataDecorators: '' + eventQueryDataDecoratorTransformer: + requestedDecorators: + - "CSV" + - "WIKIPEDIA" + dataDecorators: + "CSV": + "EVENT_ID": "https://localhost:8443/DataWave/Query/lookupUUID/EVENT_ID?uuid=@field_value@&parameters=data.decorators:CSV" + "UUID": "https://localhost:8443/DataWave/Query/lookupUUID/UUID?uuid=@field_value@&parameters=data.decorators:CSV" + "PARENT_UUID": "https://localhost:8443/DataWave/Query/lookupUUID/PARENT_UUID?uuid=@field_value@&parameters=data.decorators:CSV" + "WIKIPEDIA": + "PAGE_ID": "https://localhost:8443/DataWave/Query/lookupUUID/PAGE_ID?uuid=@field_value@&parameters=data.decorators:WIKIPEDIA" + "PAGE_TITLE": "https://localhost:8443/DataWave/Query/lookupUUID/PAGE_TITLE?uuid=@field_value@&parameters=data.decorators:WIKIPEDIA" modelTableName: ${warehouse.tables.model.name} modelName: ${warehouse.defaults.modelName} - querySyntaxParsers: |- - - - - - - - + querySyntaxParsers: + JEXL: "" + LUCENE: "LuceneToJexlQueryParser" + LUCENE-UUID: "LuceneToJexlUUIDQueryParser" + TOKENIZED-LUCENE: "TokenizedLuceneToJexlQueryParser" sendTimingToStatsd: false collectQueryMetrics: true logTimingDetails: false From 0bb2de0e45598a7fb3b739182ed2c0b1ac5b92d9 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 28 Apr 2021 12:42:57 -0400 Subject: [PATCH 034/218] WIP: Got QueryLogicFactory working in the spring boot datawave query starter. --- .../{query => query-service}/README.md | 0 .../{query => query-service}/api/pom.xml | 0 .../microservice/query/QueryParameters.java | 0 .../query/QueryParametersImpl.java | 0 .../microservice/query/QueryPersistence.java | 0 .../config/QueryExpirationProperties.java | 0 .../query/config/QueryLogicProperties.java | 0 .../query/config/QueryProperties.java | 0 .../{query => query-service}/pom.xml | 1 - .../{query => query-service}/service/pom.xml | 4 +- .../service/src/main/docker/Dockerfile | 0 .../microservice/query/QueryController.java | 0 .../query/QueryManagementService.java | 0 .../microservice/query/QueryService.java | 0 .../query/config/QueryServiceConfig.java | 0 .../microservice/query/util/QueryUtil.java | 0 .../query/web/BaseQueryResponseAdvice.java | 0 .../microservice/query/web/QueryMetrics.java | 0 .../query/web/QuerySessionIdAdvice.java | 0 .../web/annotation/EnrichQueryMetrics.java | 0 .../annotation/GenerateQuerySessionId.java | 0 .../web/filter/BaseMethodStatsFilter.java | 0 .../query/web/filter/LoggingStatsFilter.java | 0 .../QueryMetricsEnrichmentFilterAdvice.java | 0 .../src/main/resources/config/application.yml | 0 .../src/main/resources/config/bootstrap.yml | 0 .../service/src/main/resources/log4j2.yml | 0 .../microservice/query/QueryServiceTest.java | 0 .../resources/MyTestQueryLogicFactory.xml | 815 ++++++++++++++++++ .../src/test/resources/config/application.yml | 66 +- .../src/test/resources/config/bootstrap.yml | 0 .../src/test/resources/log4j2-test.xml | 0 query-microservices/query/query/pom.xml | 127 --- .../GenericQueryConfiguration.java | 222 ----- .../query/configuration/QueryData.java | 118 --- .../query/configuration/Result.java | 107 --- .../query/configuration/ResultContext.java | 17 - .../query/exception/EmptyObjectException.java | 7 - .../iterator/DatawaveTransformIterator.java | 76 -- .../query/logic/BaseQueryLogic.java | 344 -------- .../query/logic/CheckpointableQueryLogic.java | 45 - .../microservice/query/logic/Flushable.java | 17 - .../microservice/query/logic/QueryLogic.java | 369 -------- .../query/logic/QueryLogicFactory.java | 23 - .../query/logic/QueryLogicTransformer.java | 26 - .../event/DefaultResponseObjectFactory.java | 119 --- .../resources/MyTestQueryLogicFactory.xml | 805 ----------------- .../test/resources/QueryLogicFactoryAlt.xml | 23 - 48 files changed, 853 insertions(+), 2478 deletions(-) rename query-microservices/{query => query-service}/README.md (100%) rename query-microservices/{query => query-service}/api/pom.xml (100%) rename query-microservices/{query => query-service}/api/src/main/java/datawave/microservice/query/QueryParameters.java (100%) rename query-microservices/{query => query-service}/api/src/main/java/datawave/microservice/query/QueryParametersImpl.java (100%) rename query-microservices/{query => query-service}/api/src/main/java/datawave/microservice/query/QueryPersistence.java (100%) rename query-microservices/{query => query-service}/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java (100%) rename query-microservices/{query => query-service}/api/src/main/java/datawave/microservice/query/config/QueryLogicProperties.java (100%) rename query-microservices/{query => query-service}/api/src/main/java/datawave/microservice/query/config/QueryProperties.java (100%) rename query-microservices/{query => query-service}/pom.xml (97%) rename query-microservices/{query => query-service}/service/pom.xml (97%) rename query-microservices/{query => query-service}/service/src/main/docker/Dockerfile (100%) rename query-microservices/{query => query-service}/service/src/main/java/datawave/microservice/query/QueryController.java (100%) rename query-microservices/{query => query-service}/service/src/main/java/datawave/microservice/query/QueryManagementService.java (100%) rename query-microservices/{query => query-service}/service/src/main/java/datawave/microservice/query/QueryService.java (100%) rename query-microservices/{query => query-service}/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java (100%) rename query-microservices/{query => query-service}/service/src/main/java/datawave/microservice/query/util/QueryUtil.java (100%) rename query-microservices/{query => query-service}/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java (100%) rename query-microservices/{query => query-service}/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java (100%) rename query-microservices/{query => query-service}/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java (100%) rename query-microservices/{query => query-service}/service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java (100%) rename query-microservices/{query => query-service}/service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java (100%) rename query-microservices/{query => query-service}/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java (100%) rename query-microservices/{query => query-service}/service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java (100%) rename query-microservices/{query => query-service}/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java (100%) rename query-microservices/{query => query-service}/service/src/main/resources/config/application.yml (100%) rename query-microservices/{query => query-service}/service/src/main/resources/config/bootstrap.yml (100%) rename query-microservices/{query => query-service}/service/src/main/resources/log4j2.yml (100%) rename query-microservices/{query => query-service}/service/src/test/java/datawave/microservice/query/QueryServiceTest.java (100%) create mode 100644 query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml rename query-microservices/{query => query-service}/service/src/test/resources/config/application.yml (97%) rename query-microservices/{query => query-service}/service/src/test/resources/config/bootstrap.yml (100%) rename query-microservices/{query => query-service}/service/src/test/resources/log4j2-test.xml (100%) delete mode 100644 query-microservices/query/query/pom.xml delete mode 100644 query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/GenericQueryConfiguration.java delete mode 100644 query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/QueryData.java delete mode 100644 query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/Result.java delete mode 100644 query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/ResultContext.java delete mode 100644 query-microservices/query/query/src/main/java/datawave/microservice/query/exception/EmptyObjectException.java delete mode 100644 query-microservices/query/query/src/main/java/datawave/microservice/query/iterator/DatawaveTransformIterator.java delete mode 100644 query-microservices/query/query/src/main/java/datawave/microservice/query/logic/BaseQueryLogic.java delete mode 100644 query-microservices/query/query/src/main/java/datawave/microservice/query/logic/CheckpointableQueryLogic.java delete mode 100644 query-microservices/query/query/src/main/java/datawave/microservice/query/logic/Flushable.java delete mode 100644 query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogic.java delete mode 100644 query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogicFactory.java delete mode 100644 query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogicTransformer.java delete mode 100644 query-microservices/query/query/src/main/java/datawave/microservice/query/result/event/DefaultResponseObjectFactory.java delete mode 100644 query-microservices/query/service/src/test/resources/MyTestQueryLogicFactory.xml delete mode 100644 query-microservices/query/service/src/test/resources/QueryLogicFactoryAlt.xml diff --git a/query-microservices/query/README.md b/query-microservices/query-service/README.md similarity index 100% rename from query-microservices/query/README.md rename to query-microservices/query-service/README.md diff --git a/query-microservices/query/api/pom.xml b/query-microservices/query-service/api/pom.xml similarity index 100% rename from query-microservices/query/api/pom.xml rename to query-microservices/query-service/api/pom.xml diff --git a/query-microservices/query/api/src/main/java/datawave/microservice/query/QueryParameters.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParameters.java similarity index 100% rename from query-microservices/query/api/src/main/java/datawave/microservice/query/QueryParameters.java rename to query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParameters.java diff --git a/query-microservices/query/api/src/main/java/datawave/microservice/query/QueryParametersImpl.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParametersImpl.java similarity index 100% rename from query-microservices/query/api/src/main/java/datawave/microservice/query/QueryParametersImpl.java rename to query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParametersImpl.java diff --git a/query-microservices/query/api/src/main/java/datawave/microservice/query/QueryPersistence.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryPersistence.java similarity index 100% rename from query-microservices/query/api/src/main/java/datawave/microservice/query/QueryPersistence.java rename to query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryPersistence.java diff --git a/query-microservices/query/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java similarity index 100% rename from query-microservices/query/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java rename to query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java diff --git a/query-microservices/query/api/src/main/java/datawave/microservice/query/config/QueryLogicProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryLogicProperties.java similarity index 100% rename from query-microservices/query/api/src/main/java/datawave/microservice/query/config/QueryLogicProperties.java rename to query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryLogicProperties.java diff --git a/query-microservices/query/api/src/main/java/datawave/microservice/query/config/QueryProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java similarity index 100% rename from query-microservices/query/api/src/main/java/datawave/microservice/query/config/QueryProperties.java rename to query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java diff --git a/query-microservices/query/pom.xml b/query-microservices/query-service/pom.xml similarity index 97% rename from query-microservices/query/pom.xml rename to query-microservices/query-service/pom.xml index 7e74c8e2..792b8307 100644 --- a/query-microservices/query/pom.xml +++ b/query-microservices/query-service/pom.xml @@ -12,7 +12,6 @@ pom api - query service diff --git a/query-microservices/query/service/pom.xml b/query-microservices/query-service/service/pom.xml similarity index 97% rename from query-microservices/query/service/pom.xml rename to query-microservices/query-service/service/pom.xml index f318c06e..e315c156 100644 --- a/query-microservices/query/service/pom.xml +++ b/query-microservices/query-service/service/pom.xml @@ -16,9 +16,9 @@ 1.0-SNAPSHOT 1.0-SNAPSHOT 1.0-SNAPSHOT - 1.0-SNAPSHOT 1.6.6-SNAPSHOT 1.3-SNAPSHOT + 1.0-SNAPSHOT @@ -61,7 +61,7 @@ gov.nsa.datawave.microservice spring-boot-starter-datawave-query - ${version.microservice.query-starter} + ${version.microservice.starter-query} gov.nsa.datawave.webservices diff --git a/query-microservices/query/service/src/main/docker/Dockerfile b/query-microservices/query-service/service/src/main/docker/Dockerfile similarity index 100% rename from query-microservices/query/service/src/main/docker/Dockerfile rename to query-microservices/query-service/service/src/main/docker/Dockerfile diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java similarity index 100% rename from query-microservices/query/service/src/main/java/datawave/microservice/query/QueryController.java rename to query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java similarity index 100% rename from query-microservices/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java rename to query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/QueryService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryService.java similarity index 100% rename from query-microservices/query/service/src/main/java/datawave/microservice/query/QueryService.java rename to query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryService.java diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java similarity index 100% rename from query-microservices/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java rename to query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/util/QueryUtil.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/util/QueryUtil.java similarity index 100% rename from query-microservices/query/service/src/main/java/datawave/microservice/query/util/QueryUtil.java rename to query-microservices/query-service/service/src/main/java/datawave/microservice/query/util/QueryUtil.java diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java similarity index 100% rename from query-microservices/query/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java rename to query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java similarity index 100% rename from query-microservices/query/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java rename to query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java similarity index 100% rename from query-microservices/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java rename to query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java similarity index 100% rename from query-microservices/query/service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java rename to query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java similarity index 100% rename from query-microservices/query/service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java rename to query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java similarity index 100% rename from query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java rename to query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java similarity index 100% rename from query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java rename to query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java diff --git a/query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java similarity index 100% rename from query-microservices/query/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java rename to query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java diff --git a/query-microservices/query/service/src/main/resources/config/application.yml b/query-microservices/query-service/service/src/main/resources/config/application.yml similarity index 100% rename from query-microservices/query/service/src/main/resources/config/application.yml rename to query-microservices/query-service/service/src/main/resources/config/application.yml diff --git a/query-microservices/query/service/src/main/resources/config/bootstrap.yml b/query-microservices/query-service/service/src/main/resources/config/bootstrap.yml similarity index 100% rename from query-microservices/query/service/src/main/resources/config/bootstrap.yml rename to query-microservices/query-service/service/src/main/resources/config/bootstrap.yml diff --git a/query-microservices/query/service/src/main/resources/log4j2.yml b/query-microservices/query-service/service/src/main/resources/log4j2.yml similarity index 100% rename from query-microservices/query/service/src/main/resources/log4j2.yml rename to query-microservices/query-service/service/src/main/resources/log4j2.yml diff --git a/query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java similarity index 100% rename from query-microservices/query/service/src/test/java/datawave/microservice/query/QueryServiceTest.java rename to query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java diff --git a/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml b/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml new file mode 100644 index 00000000..f736a016 --- /dev/null +++ b/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml @@ -0,0 +1,815 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 2611 + + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/query-microservices/query/service/src/test/resources/config/application.yml b/query-microservices/query-service/service/src/test/resources/config/application.yml similarity index 97% rename from query-microservices/query/service/src/test/resources/config/application.yml rename to query-microservices/query-service/service/src/test/resources/config/application.yml index 24f33d46..296ecb50 100644 --- a/query-microservices/query/service/src/test/resources/config/application.yml +++ b/query-microservices/query-service/service/src/test/resources/config/application.yml @@ -13,6 +13,35 @@ spring: hazelcast.client.enabled: false +server: + port: 0 + non-secure-port: 0 + servlet.context-path: /query-starter-test + ssl: + client-auth: NEED + trust-store: 'classpath:testCA.p12' + trust-store-type: PKCS12 + trust-store-password: 'ChangeIt' + key-store: 'classpath:testServer.p12' + key-store-type: PKCS12 + key-store-password: 'ChangeIt' + outbound-ssl: + key-store: ${server.ssl.key-store} + key-store-password: ${server.ssl.key-store-password} + key-store-type: ${server.ssl.key-store-type} + trust-store: ${server.ssl.trust-store} + trust-store-password: ${server.ssl.trust-store-password} + trust-store-type: ${server.ssl.trust-store-type} + +management: + endpoints: + web: + base-path: "/mgmt" + +logging: + level: + datawave.microservice: DEBUG + warehouse: accumulo: zookeepers: 'localhost:2181' @@ -63,6 +92,13 @@ warehouse: ivaratorCacheScanTimeoutMinutes: 60 modelName: 'DATAWAVE' +datawave: + metadata: + all-metadata-auths: + - PRIVATE,PUBLIC + type-substitutions: + "[datawave.data.type.DateType]": "datawave.data.type.RawDateType" + query: parser: skipTokenizeUnfieldedFields: @@ -188,36 +224,6 @@ query: statsdPort: ${warehouse.statsd.port} evaluationOnlyFields: "" -server: - port: 0 - non-secure-port: 0 - servlet.context-path: /query - ssl: - client-auth: NEED - trust-store: 'classpath:testCA.p12' - trust-store-type: PKCS12 - trust-store-password: 'ChangeIt' - key-store: 'classpath:testServer.p12' - key-store-type: PKCS12 - key-store-password: 'ChangeIt' - outbound-ssl: - key-store: ${server.ssl.key-store} - key-store-password: ${server.ssl.key-store-password} - key-store-type: ${server.ssl.key-store-type} - trust-store: ${server.ssl.trust-store} - trust-store-password: ${server.ssl.trust-store-password} - trust-store-type: ${server.ssl.trust-store-type} - -management: - endpoints: - web: - base-path: "/mgmt" - -logging: - level: - datawave.microservice: DEBUG - io.undertow.request: FATAL - audit-client: discovery: enabled: false diff --git a/query-microservices/query/service/src/test/resources/config/bootstrap.yml b/query-microservices/query-service/service/src/test/resources/config/bootstrap.yml similarity index 100% rename from query-microservices/query/service/src/test/resources/config/bootstrap.yml rename to query-microservices/query-service/service/src/test/resources/config/bootstrap.yml diff --git a/query-microservices/query/service/src/test/resources/log4j2-test.xml b/query-microservices/query-service/service/src/test/resources/log4j2-test.xml similarity index 100% rename from query-microservices/query/service/src/test/resources/log4j2-test.xml rename to query-microservices/query-service/service/src/test/resources/log4j2-test.xml diff --git a/query-microservices/query/query/pom.xml b/query-microservices/query/query/pom.xml deleted file mode 100644 index bf845784..00000000 --- a/query-microservices/query/query/pom.xml +++ /dev/null @@ -1,127 +0,0 @@ - - - 4.0.0 - - gov.nsa.datawave.microservice - datawave-microservice-parent - 1.8 - - - query - 1.0-SNAPSHOT - - 3.2.1-SNAPSHOT - 1.10-SNAPSHOT - 1.3-SNAPSHOT - 1.0-SNAPSHOT - - - - - gov.nsa.datawave - datawave-ingest-core - ${version.datawave} - - - * - org.slf4j - - - - - gov.nsa.datawave.microservice - audit-api - ${version.microservice.audit-api} - - - gov.nsa.datawave.microservice - base-rest-responses - ${version.microservice.base-rest-responses} - - - gov.nsa.datawave.microservice - query-storage-api - ${version.microservice.query-storage-api} - - - gov.nsa.datawave.webservices - datawave-ws-client - ${version.datawave} - - - * - org.slf4j - - - - - gov.nsa.datawave.webservices - datawave-ws-common - ${version.datawave} - - - * - org.slf4j - - - - - - - - gov.nsa.datawave - datawave-ingest-core - - - gov.nsa.datawave.microservice - audit-api - - - gov.nsa.datawave.microservice - base-rest-responses - - - gov.nsa.datawave.microservice - query-storage-api - - - gov.nsa.datawave.webservices - datawave-ws-client - - - gov.nsa.datawave.webservices - datawave-ws-common - - - junit - junit - test - - - org.junit.jupiter - junit-jupiter-engine - test - - - org.junit.vintage - junit-vintage-engine - test - - - - - - - true - - - false - - datawave-github-mvn-repo - https://raw.githubusercontent.com/NationalSecurityAgency/datawave/mvn-repo - - - diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/GenericQueryConfiguration.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/GenericQueryConfiguration.java deleted file mode 100644 index 13c8424c..00000000 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/GenericQueryConfiguration.java +++ /dev/null @@ -1,222 +0,0 @@ -package datawave.microservice.query.configuration; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.common.collect.Iterators; -import datawave.microservice.query.logic.BaseQueryLogic; -import datawave.util.TableName; -import datawave.webservice.query.Query; -import org.apache.accumulo.core.client.BatchScanner; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.security.Authorizations; - -import javax.xml.bind.annotation.XmlTransient; -import java.util.Collections; -import java.util.Date; -import java.util.Iterator; -import java.util.Set; - -/** - *

- * A basic query configuration object that contains the information needed to run a query. - *

- * - *

- * Provides some "expected" default values for parameters. This configuration object also encapsulates iterators and their options that would be set on a - * {@link BatchScanner}. - *

- * - */ -public abstract class GenericQueryConfiguration { - // is this execution expected to be checkpointable (changes how we allocate ranges to scanners) - private boolean checkpointable = false; - - private Connector connector = null; - private Set authorizations = Collections.singleton(Authorizations.EMPTY); - - private Query query = null; - - // Leave in a top-level query for backwards-compatibility purposes - private String queryString = null; - - private Date beginDate = null; - private Date endDate = null; - - // The max number of next + seek calls made by the underlying iterators - private Long maxWork = -1L; - - protected int baseIteratorPriority = 100; - - // Table name - private String tableName = TableName.SHARD; - - private Iterator queries = Collections.emptyIterator(); - - protected boolean bypassAccumulo; - - /** - * Empty default constructor - */ - public GenericQueryConfiguration() { - - } - - /** - * Pulls the table name, max query results, and max rows to scan from the provided argument - * - * @param configuredLogic - * A pre-configured BaseQueryLogic to initialize the Configuration with - */ - public GenericQueryConfiguration(BaseQueryLogic configuredLogic) { - this(configuredLogic.getConfig()); - } - - public GenericQueryConfiguration(GenericQueryConfiguration genericConfig) { - this.setBaseIteratorPriority(genericConfig.getBaseIteratorPriority()); - this.setBypassAccumulo(genericConfig.getBypassAccumulo()); - this.setAuthorizations(genericConfig.getAuthorizations()); - this.setBeginDate(genericConfig.getBeginDate()); - this.setConnector(genericConfig.getConnector()); - this.setEndDate(genericConfig.getEndDate()); - this.setMaxWork(genericConfig.getMaxWork()); - this.setQueries(genericConfig.getQueries()); - this.setQueryString(genericConfig.getQueryString()); - this.setTableName(genericConfig.getTableName()); - } - - /** - * Return the configured {@code Iterator} - * - * @return - */ - public Iterator getQueries() { - return Iterators.unmodifiableIterator(this.queries); - } - - /** - * Set the queries to be run. - * - * @param queries - */ - public void setQueries(Iterator queries) { - this.queries = queries; - } - - public boolean isCheckpointable() { - return checkpointable; - } - - public void setCheckpointable(boolean checkpointable) { - this.checkpointable = checkpointable; - } - - @JsonIgnore - @XmlTransient - public Connector getConnector() { - return connector; - } - - public void setConnector(Connector connector) { - this.connector = connector; - } - - public Query getQuery() { - return query; - } - - public void setQuery(Query query) { - this.query = query; - } - - public void setQueryString(String query) { - this.queryString = query; - } - - public String getQueryString() { - return queryString; - } - - public Set getAuthorizations() { - return authorizations; - } - - public void setAuthorizations(Set auths) { - this.authorizations = auths; - } - - public int getBaseIteratorPriority() { - return baseIteratorPriority; - } - - public void setBaseIteratorPriority(final int baseIteratorPriority) { - this.baseIteratorPriority = baseIteratorPriority; - } - - public Date getBeginDate() { - return beginDate; - } - - public void setBeginDate(Date beginDate) { - this.beginDate = beginDate; - } - - public Date getEndDate() { - return endDate; - } - - public void setEndDate(Date endDate) { - this.endDate = endDate; - } - - public Long getMaxWork() { - return maxWork; - } - - public void setMaxWork(Long maxWork) { - this.maxWork = maxWork; - } - - public String getTableName() { - return tableName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - public boolean getBypassAccumulo() { - return bypassAccumulo; - } - - public void setBypassAccumulo(boolean bypassAccumulo) { - this.bypassAccumulo = bypassAccumulo; - } - - /** - * Checks for non-null, sane values for the configured values - * - * @return True if all of the encapsulated values have legitimate values, otherwise false - */ - public boolean canRunQuery() { - // Ensure we were given connector and authorizations - if (null == this.getConnector() || null == this.getAuthorizations()) { - return false; - } - - // Ensure valid dates - if (null == this.getBeginDate() || null == this.getEndDate() || endDate.before(beginDate)) { - return false; - } - - // A non-empty table was given - if (null == getTableName() || this.getTableName().isEmpty()) { - return false; - } - - // At least one QueryData was provided - if (null == this.queries) { - return false; - } - - return true; - } -} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/QueryData.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/QueryData.java deleted file mode 100644 index 83e226a8..00000000 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/QueryData.java +++ /dev/null @@ -1,118 +0,0 @@ -package datawave.microservice.query.configuration; - -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import java.util.Collections; -import org.apache.accumulo.core.client.IteratorSetting; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.data.Value; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; - -/** - * Class to encapsulate all required information to run a query. - * - */ -public class QueryData implements ResultContext { - List settings = Lists.newArrayList(); - String query; - Collection ranges = Sets.newHashSet(); - Collection columnFamilies = Sets.newHashSet(); - Map.Entry lastResult; - boolean finished = false; - - public QueryData() {} - - public QueryData(String query, Collection ranges, List settings) { - setQuery(query); - setRanges(ranges); - setSettings(settings); - } - - public QueryData(QueryData other) { - this(other.getQuery(), other.getRanges(), other.getSettings()); - } - - public QueryData(QueryData other, Collection ranges) { - setQuery(other.getQuery()); - setSettings(other.getSettings()); - setRanges(ranges); - } - - public QueryData(String queryString, ArrayList ranges, List settings, Collection columnFamilies) { - this(queryString, ranges, settings); - this.columnFamilies.addAll(columnFamilies); - } - - public List getSettings() { - return settings; - } - - public void setSettings(List settings) { - this.settings = Lists.newArrayList(settings); - } - - public String getQuery() { - return query; - } - - public void setQuery(String query) { - this.query = query; - } - - public Collection getRanges() { - if (isFinished()) { - return Collections.emptySet(); - } else if (lastResult != null) { - List newRanges = new ArrayList<>(); - for (Range range : ranges) { - if (range.contains(lastResult.getKey())) { - newRanges.add(new Range(lastResult.getKey(), false, range.getEndKey(), range.isEndKeyInclusive())); - } else { - newRanges.add(range); - } - } - return newRanges; - } - return ranges; - } - - public Collection getColumnFamilies() { - return columnFamilies; - } - - public void setRanges(Collection ranges) { - if (null != ranges) - this.ranges.addAll(ranges); - } - - public void addIterator(IteratorSetting cfg) { - this.settings.add(cfg); - } - - public void setLastResult(Map.Entry result) { - this.lastResult = result; - if (this.lastResult == null) { - this.finished = true; - } - } - - public boolean isFinished() { - return this.finished; - } - - public Map.Entry getLastResult() { - return lastResult; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(256); - sb.append("Query: '").append(this.query).append("', Ranges: ").append(this.ranges).append(", Settings: ").append(this.settings); - return sb.toString(); - } -} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/Result.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/Result.java deleted file mode 100644 index 7ae4b197..00000000 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/Result.java +++ /dev/null @@ -1,107 +0,0 @@ -package datawave.microservice.query.configuration; - -import com.google.common.base.Function; -import com.google.common.base.Predicate; -import com.google.common.collect.Iterators; -import com.google.common.collect.Maps; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Value; -import org.apache.commons.lang3.builder.EqualsBuilder; -import org.apache.commons.lang3.builder.HashCodeBuilder; - -import javax.annotation.Nullable; -import java.util.Iterator; -import java.util.Map; - -public class Result implements Map.Entry { - private final T context; - private final Key key; - private Value value; - - public Result(Key k, Value v) { - this(null, k, v); - } - - public Result(T context, Key k, Value v) { - this.context = context; - this.key = k; - this.value = v; - } - - public T getContext() { - return context; - } - - @Override - public Key getKey() { - return key; - } - - @Override - public Value getValue() { - return value; - } - - @Override - public Value setValue(Value value) { - throw new UnsupportedOperationException("This value is immutable"); - } - - @Override - public boolean equals(Object o) { - if (o instanceof Result) { - Result other = (Result) o; - return new EqualsBuilder().append(context, other.context).append(key, other.key).append(value, other.value).isEquals(); - } - return false; - } - - @Override - public int hashCode() { - return new HashCodeBuilder().append(context).append(key).append(value).toHashCode(); - } - - public Map.Entry returnKeyValue() { - Map.Entry entry = (key == null ? null : Maps.immutableEntry(key, value)); - if (context != null) { - context.setLastResult(entry); - } - return entry; - } - - public static Iterator> keyValueIterator(Iterator it) { - return Iterators.filter(Iterators.transform(it, new Function>() { - @Override - public Map.Entry apply(@Nullable Result input) { - if (input == null) { - return null; - } - return input.returnKeyValue(); - } - }), new Predicate>() { - @Override - public boolean apply(Map.@org.checkerframework.checker.nullness.qual.Nullable Entry keyValueEntry) { - return keyValueEntry != null; - } - }); - } - - public static Iterator resultIterator(final ResultContext context, Iterator> it) { - return Iterators.filter(Iterators.transform(it, new Function,Result>() { - @Nullable - @Override - public Result apply(@Nullable Map.Entry keyValueEntry) { - if (keyValueEntry == null) { - return null; - } - return new Result(context, keyValueEntry.getKey(), keyValueEntry.getValue()); - } - }), new Predicate() { - - @Override - public boolean apply(@org.checkerframework.checker.nullness.qual.Nullable Result result) { - return result != null; - } - }); - } -} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/ResultContext.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/ResultContext.java deleted file mode 100644 index 22efba4b..00000000 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/configuration/ResultContext.java +++ /dev/null @@ -1,17 +0,0 @@ -package datawave.microservice.query.configuration; - -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Value; - -import java.util.Map; - -public interface ResultContext { - /** - * Set the last result returned. Setting a result of null denotes this scan is finished. - * - * @param result - */ - void setLastResult(Map.Entry result); - - boolean isFinished(); -} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/exception/EmptyObjectException.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/exception/EmptyObjectException.java deleted file mode 100644 index 38633c58..00000000 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/exception/EmptyObjectException.java +++ /dev/null @@ -1,7 +0,0 @@ -package datawave.microservice.query.exception; - -// used when a transformer gets a non-null empty object -// and the TransformIterator should call next instead of returning null -public class EmptyObjectException extends RuntimeException { - -} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/iterator/DatawaveTransformIterator.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/iterator/DatawaveTransformIterator.java deleted file mode 100644 index 7dafea21..00000000 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/iterator/DatawaveTransformIterator.java +++ /dev/null @@ -1,76 +0,0 @@ -package datawave.microservice.query.iterator; - -import datawave.microservice.query.exception.EmptyObjectException; -import datawave.microservice.query.logic.Flushable; -import org.apache.commons.collections4.Transformer; -import org.apache.commons.collections4.iterators.TransformIterator; -import org.apache.log4j.Logger; - -import java.util.Iterator; - -public class DatawaveTransformIterator extends TransformIterator { - - private Logger log = Logger.getLogger(DatawaveTransformIterator.class); - private O next = null; - - public DatawaveTransformIterator() { - super(); - } - - public DatawaveTransformIterator(Iterator iterator) { - super(iterator); - } - - public DatawaveTransformIterator(Iterator iterator, Transformer transformer) { - super(iterator, transformer); - } - - @Override - public boolean hasNext() { - - if (next == null) { - next = getNext(); - } - return (next != null); - } - - @Override - public O next() { - - O o = null; - if (next == null) { - o = getNext(); - } else { - o = next; - next = null; - } - return o; - } - - private O getNext() { - - boolean done = false; - O o = null; - while (super.hasNext() && !done) { - try { - o = super.next(); - done = true; - } catch (EmptyObjectException e) { - // not yet done, so continue fetching next - } - } - // see if there are any results cached by the transformer - if (o == null && getTransformer() instanceof Flushable) { - done = false; - while (!done) { - try { - o = ((Flushable) getTransformer()).flush(); - done = true; - } catch (EmptyObjectException e) { - // not yet done, so continue flushing - } - } - } - return o; - } -} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/BaseQueryLogic.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/BaseQueryLogic.java deleted file mode 100644 index 749e4730..00000000 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/BaseQueryLogic.java +++ /dev/null @@ -1,344 +0,0 @@ -package datawave.microservice.query.logic; - -import datawave.audit.SelectorExtractor; -import datawave.marking.MarkingFunctions; -import datawave.microservice.query.configuration.GenericQueryConfiguration; -import datawave.microservice.query.iterator.DatawaveTransformIterator; -import datawave.webservice.common.audit.Auditor.AuditType; -import datawave.webservice.query.Query; -import datawave.webservice.query.result.event.ResponseObjectFactory; -import org.apache.accumulo.core.client.BatchScanner; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.ScannerBase; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.commons.collections4.iterators.TransformIterator; -import org.springframework.beans.factory.annotation.Required; - -import javax.inject.Inject; -import java.util.Collection; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public abstract class BaseQueryLogic implements QueryLogic { - - private GenericQueryConfiguration baseConfig; - private String logicName = "No logicName was set"; - private String logicDescription = "Not configured"; - private AuditType auditType = null; - private Map dnResultLimits = null; - protected long maxResults = -1L; - protected int maxConcurrentTasks = -1; - protected ScannerBase scanner; - @SuppressWarnings("unchecked") - protected Iterator iterator = (Iterator) Collections.emptyList().iterator(); - private int maxPageSize = 0; - private long pageByteTrigger = 0; - private boolean collectQueryMetrics = true; - private String _connPoolName; - private Set authorizedDNs; - protected Set requiredRoles; - protected MarkingFunctions markingFunctions; - protected ResponseObjectFactory responseObjectFactory; - protected SelectorExtractor selectorExtractor; - - public static final String BYPASS_ACCUMULO = "rfile.debug"; - - public BaseQueryLogic() { - getConfig().setBaseIteratorPriority(100); - } - - public BaseQueryLogic(BaseQueryLogic other) { - // Generic Query Config variables - setTableName(other.getTableName()); - setMaxWork(other.getMaxWork()); - setMaxResults(other.getMaxResults()); - setBaseIteratorPriority(other.getBaseIteratorPriority()); - setBypassAccumulo(other.getBypassAccumulo()); - - // Other variables - setMaxResults(other.maxResults); - setMarkingFunctions(other.getMarkingFunctions()); - setResponseObjectFactory(other.getResponseObjectFactory()); - setLogicName(other.getLogicName()); - setLogicDescription(other.getLogicDescription()); - setAuditType(other.getAuditType(null)); - this.scanner = other.scanner; - this.iterator = other.iterator; - setMaxPageSize(other.getMaxPageSize()); - setPageByteTrigger(other.getPageByteTrigger()); - setCollectQueryMetrics(other.getCollectQueryMetrics()); - setConnPoolName(other.getConnPoolName()); - setRequiredRoles(other.getRequiredRoles()); - setSelectorExtractor(other.getSelectorExtractor()); - } - - public GenericQueryConfiguration getConfig() { - if (baseConfig == null) { - baseConfig = new GenericQueryConfiguration() {}; - } - - return baseConfig; - } - - @Override - public String getPlan(Connector connection, Query settings, Set runtimeQueryAuthorizations, boolean expandFields, boolean expandValues) - throws Exception { - // for many query logics, the query is what it is - return settings.getQuery(); - } - - public MarkingFunctions getMarkingFunctions() { - return markingFunctions; - } - - public void setMarkingFunctions(MarkingFunctions markingFunctions) { - this.markingFunctions = markingFunctions; - } - - public ResponseObjectFactory getResponseObjectFactory() { - return responseObjectFactory; - } - - public void setResponseObjectFactory(ResponseObjectFactory responseObjectFactory) { - this.responseObjectFactory = responseObjectFactory; - } - - public Set getRequiredRoles() { - return requiredRoles; - } - - public void setRequiredRoles(Set requiredRoles) { - this.requiredRoles = requiredRoles; - } - - @Override - public String getTableName() { - return getConfig().getTableName(); - } - - @Override - public long getMaxResults() { - return this.maxResults; - } - - @Override - public int getMaxConcurrentTasks() { - return this.maxConcurrentTasks; - } - - @Override - @Deprecated - public long getMaxRowsToScan() { - return getMaxWork(); - } - - @Override - public long getMaxWork() { - return getConfig().getMaxWork(); - } - - @Override - public void setTableName(String tableName) { - getConfig().setTableName(tableName); - } - - @Override - public void setMaxResults(long maxResults) { - this.maxResults = maxResults; - } - - @Override - public void setMaxConcurrentTasks(int maxConcurrentTasks) { - this.maxConcurrentTasks = maxConcurrentTasks; - } - - @Override - @Deprecated - public void setMaxRowsToScan(long maxRowsToScan) { - setMaxWork(maxRowsToScan); - } - - @Override - public void setMaxWork(long maxWork) { - getConfig().setMaxWork(maxWork); - } - - @Override - public int getMaxPageSize() { - return maxPageSize; - } - - @Override - public void setMaxPageSize(int maxPageSize) { - this.maxPageSize = maxPageSize; - } - - @Override - public long getPageByteTrigger() { - return pageByteTrigger; - } - - @Override - public void setPageByteTrigger(long pageByteTrigger) { - this.pageByteTrigger = pageByteTrigger; - } - - @Override - public int getBaseIteratorPriority() { - return getConfig().getBaseIteratorPriority(); - } - - @Override - public void setBaseIteratorPriority(final int baseIteratorPriority) { - getConfig().setBaseIteratorPriority(baseIteratorPriority); - } - - @Override - public Iterator iterator() { - return iterator; - } - - @Override - public TransformIterator getTransformIterator(Query settings) { - return new DatawaveTransformIterator(this.iterator(), this.getTransformer(settings)); - } - - @Override - public String getLogicName() { - return logicName; - } - - @Override - public void setLogicName(String logicName) { - this.logicName = logicName; - } - - public boolean getBypassAccumulo() { - return getConfig().getBypassAccumulo(); - } - - public void setBypassAccumulo(boolean bypassAccumulo) { - getConfig().setBypassAccumulo(bypassAccumulo); - } - - @Override - public abstract Object clone() throws CloneNotSupportedException; - - public void close() { - if (null == scanner) - return; - if (scanner instanceof BatchScanner) { - scanner.close(); - } - } - - /* - * Implementations must override if they want a query-specific value returned - */ - @Override - public AuditType getAuditType(Query query) { - return auditType; - } - - @Override - public AuditType getAuditType() { - return auditType; - } - - @Override - @Required - // enforces that the unit tests will fail and the application will not deploy unless this property is set - public void setAuditType(AuditType auditType) { - this.auditType = auditType; - } - - @Override - public void setLogicDescription(String logicDescription) { - this.logicDescription = logicDescription; - } - - @Override - public String getLogicDescription() { - return logicDescription; - } - - public boolean getCollectQueryMetrics() { - return collectQueryMetrics; - } - - public void setCollectQueryMetrics(boolean collectQueryMetrics) { - this.collectQueryMetrics = collectQueryMetrics; - } - - /** {@inheritDoc} */ - @Override - public String getConnPoolName() { - return _connPoolName; - } - - /** {@inheritDoc} */ - @Override - public void setConnPoolName(final String connPoolName) { - _connPoolName = connPoolName; - } - - /** {@inheritDoc} */ - public boolean canRunQuery(Collection userRoles) { - return this.requiredRoles == null || userRoles.containsAll(requiredRoles); - } - - @Override - public final void validate(Map> parameters) throws IllegalArgumentException { - Set requiredParams = getRequiredQueryParameters(); - for (String required : requiredParams) { - List values = parameters.get(required); - if (null == values) { - throw new IllegalArgumentException("Required parameter " + required + " not found"); - } - } - } - - @Override - public List getSelectors(Query settings) throws IllegalArgumentException { - List selectorList = null; - if (this.selectorExtractor != null) { - try { - selectorList = this.selectorExtractor.extractSelectors(settings); - } catch (Exception e) { - throw new IllegalArgumentException(e.getMessage(), e); - } - } - return selectorList; - } - - public void setSelectorExtractor(SelectorExtractor selectorExtractor) { - this.selectorExtractor = selectorExtractor; - } - - public SelectorExtractor getSelectorExtractor() { - return selectorExtractor; - } - - @Override - public Set getAuthorizedDNs() { - return authorizedDNs; - } - - @Override - public void setAuthorizedDNs(Set authorizedDNs) { - this.authorizedDNs = authorizedDNs; - } - - @Override - public void setDnResultLimits(Map dnResultLimits) { - this.dnResultLimits = dnResultLimits; - } - - @Override - public Map getDnResultLimits() { - return dnResultLimits; - } -} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/CheckpointableQueryLogic.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/CheckpointableQueryLogic.java deleted file mode 100644 index 672660f3..00000000 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/CheckpointableQueryLogic.java +++ /dev/null @@ -1,45 +0,0 @@ -package datawave.microservice.query.logic; - -import datawave.microservice.common.storage.QueryCheckpoint; -import datawave.microservice.common.storage.QueryKey; -import org.apache.accumulo.core.client.Connector; - -import java.util.List; - -public interface CheckpointableQueryLogic { - - /** - * This will allow us to check if a query logic is actually checkpointable. Even if the query logic supports it, the caller may have to tell the query logic - * that it is going to be checkpointed. - */ - boolean isCheckpointable(); - - /** - * This will tell the query logic that is is going to be checkpointed. - * - * @param checkpointable - */ - void setCheckpointable(boolean checkpointable); - - /** - * This can be called at any point to get a checkpoint such that this query logic instance can be torn down to be rebuilt later. At a minimum this should be - * called after the getTransformIterator is depleted of results. - * - * @param queryKey - * - the query key to include in the checkpoint - * @return The query checkpoints - */ - List checkpoint(QueryKey queryKey); - - /** - * Implementations use the configuration to setup execution of a portion of their query. getTransformIterator should be used to get the partial results if - * any. - * - * @param connection - * - The accumulo connector - * @param checkpoint - * - Encapsulates all information needed to run a portion of the query. - */ - void setupQuery(Connector connection, QueryCheckpoint checkpoint) throws Exception; - -} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/Flushable.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/Flushable.java deleted file mode 100644 index 6ad37ff9..00000000 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/Flushable.java +++ /dev/null @@ -1,17 +0,0 @@ -package datawave.microservice.query.logic; - -import datawave.microservice.query.exception.EmptyObjectException; - -public interface Flushable { - - /** - * The flush method is used to return an results that were cached from the calls to transform(Object). If this method will be called multiple times until a - * null is returned. If EmptyObjectException is thrown instead of returning null, then flush will be called again. - * - * @return A cached object or null if no more exist. - * @throws EmptyObjectException - * if the current cached result is empty, and flush should be called again. - */ - T flush() throws EmptyObjectException; - -} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogic.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogic.java deleted file mode 100644 index aa2f32a9..00000000 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogic.java +++ /dev/null @@ -1,369 +0,0 @@ -package datawave.microservice.query.logic; - -import datawave.audit.SelectorExtractor; -import datawave.marking.MarkingFunctions; -import datawave.microservice.query.configuration.GenericQueryConfiguration; -import datawave.validation.ParameterValidator; -import datawave.webservice.common.audit.Auditor.AuditType; -import datawave.webservice.common.connection.AccumuloConnectionFactory; -import datawave.webservice.query.Query; -import datawave.webservice.query.cache.ResultsPage; -import datawave.webservice.query.exception.DatawaveErrorCode; -import datawave.webservice.query.exception.QueryException; -import datawave.webservice.query.result.event.ResponseObjectFactory; -import datawave.webservice.result.BaseResponse; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.commons.collections4.iterators.TransformIterator; - -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public interface QueryLogic extends Iterable, Cloneable, ParameterValidator { - - /** - * A mechanism to get the normalized query without actually setting up the query. This can be called with having to call initialize. - * - * The default implementation is to return the query string as the normalized query - * - * @param connection - * - Accumulo connector to use for this query - * @param settings - * - query settings (query, begin date, end date, etc.) - * @param runtimeQueryAuthorizations - * - authorizations that have been calculated for this query based on the caller and server. - * @param expandFields - * - should unfielded terms be expanded - * @param expandValues - * - should regex/ranges be expanded into discrete values - */ - String getPlan(Connector connection, Query settings, Set runtimeQueryAuthorizations, boolean expandFields, boolean expandValues) - throws Exception; - - /** - * Implementations create a configuration using the connection, settings, and runtimeQueryAuthorizations. - * - * @param connection - * - Accumulo connector to use for this query - * @param settings - * - query settings (query, begin date, end date, etc.) - * @param runtimeQueryAuthorizations - * - authorizations that have been calculated for this query based on the caller and server. - * @throws Exception - */ - GenericQueryConfiguration initialize(Connector connection, Query settings, Set runtimeQueryAuthorizations) throws Exception; - - /** - * - * @param settings - * - query settings (query, begin date, end date, etc.) - * @return list of selectors used in the Query - */ - List getSelectors(Query settings); - - SelectorExtractor getSelectorExtractor(); - - /** - * Implementations use the configuration to run their query. It is expected that initialize has already been called. - * - * @param configuration - * Encapsulates all information needed to run a query (whether the query is a BatchScanner, a MapReduce job, etc) - */ - void setupQuery(GenericQueryConfiguration configuration) throws Exception; - - /** - * @return a copy of this instance - */ - Object clone() throws CloneNotSupportedException; - - /** - * @return priority from AccumuloConnectionFactory - */ - AccumuloConnectionFactory.Priority getConnectionPriority(); - - /** - * @return Transformer that will convert Key,Value to a Result object - */ - QueryLogicTransformer getTransformer(Query settings); - - default String getResponseClass(Query query) throws QueryException { - try { - QueryLogicTransformer t = this.getTransformer(query); - BaseResponse refResponse = t.createResponse(new ResultsPage()); - return refResponse.getClass().getCanonicalName(); - } catch (RuntimeException e) { - throw new QueryException(DatawaveErrorCode.QUERY_TRANSFORM_ERROR); - } - } - - /** - * Allows for the customization of handling query results, e.g. allows for aggregation of query results before returning to the client. - * - * @param settings - * The query settings object - * @return Return a TransformIterator for the QueryLogic implementation - */ - TransformIterator getTransformIterator(Query settings); - - /** - * release resources - */ - void close(); - - /** @return the tableName */ - String getTableName(); - - /** - * @return max number of results to pass back to the caller - */ - long getMaxResults(); - - /** - * @return max number of concurrent tasks to run for this query - */ - int getMaxConcurrentTasks(); - - /** - * @return the results of getMaxWork - */ - @Deprecated - long getMaxRowsToScan(); - - /** - * @return max number of nexts + seeks performed by the underlying iterators in total - */ - long getMaxWork(); - - /** - * @return max number of records to return in a page (max pagesize allowed) - */ - int getMaxPageSize(); - - /** - * @return the number of bytes at which a page will be returned, even if pagesize has not been reached - */ - long getPageByteTrigger(); - - /** - * Returns the base iterator priority. - * - * @return base iterator priority - */ - int getBaseIteratorPriority(); - - /** - * @param tableName - * the name of the table - */ - void setTableName(String tableName); - - /** - * @param maxResults - * max number of results to pass back to the caller - */ - void setMaxResults(long maxResults); - - /** - * @param maxConcurrentTasks - * max number of concurrent tasks to run for this query - */ - void setMaxConcurrentTasks(int maxConcurrentTasks); - - /** - * @param maxRowsToScan - * This is now deprecated and setMaxWork should be used instead. This is equivalent to setMaxWork. - */ - @Deprecated - void setMaxRowsToScan(long maxRowsToScan); - - /** - * @param maxWork - * max work which is normally calculated as the number of next + seek calls made by the underlying iterators - */ - void setMaxWork(long maxWork); - - /** - * @param maxPageSize - * max number of records in a page (max pagesize allowed) - */ - void setMaxPageSize(int maxPageSize); - - /** - * @param pageByteTrigger - * the number of bytes at which a page will be returned, even if pagesize has not been reached - */ - void setPageByteTrigger(long pageByteTrigger); - - /** - * Sets the base iterator priority - * - * @param priority - * base iterator priority - */ - void setBaseIteratorPriority(final int priority); - - /** - * @param logicName - * name of the query logic - */ - void setLogicName(String logicName); - - /** - * @return name of the query logic - */ - String getLogicName(); - - /** - * @param logicDescription - * a brief description of this logic type - */ - void setLogicDescription(String logicDescription); - - /** - * @return the audit level for this logic - */ - AuditType getAuditType(Query query); - - /** - * @return the audit level for this logic for a specific query - */ - AuditType getAuditType(); - - /** - * @param auditType - * the audit level for this logic - */ - void setAuditType(AuditType auditType); - - /** - * @return a brief description of this logic type - */ - String getLogicDescription(); - - /** - * @return should query metrics be collected for this query logic - */ - boolean getCollectQueryMetrics(); - - /** - * @param collectQueryMetrics - * whether query metrics be collected for this query logic - */ - void setCollectQueryMetrics(boolean collectQueryMetrics); - - /** - * List of parameters that can be used in the 'params' parameter to Query/create - * - * @return the supported parameters - */ - Set getOptionalQueryParameters(); - - /** - * @param connPoolName - * The name of the connection pool to set. - */ - void setConnPoolName(String connPoolName); - - /** @return the connPoolName */ - String getConnPoolName(); - - /** - * Check that the user has one of the required roles. userRoles my be null when there is no intent to control access to QueryLogic - * - * @param userRoles - * @return true/false - */ - boolean canRunQuery(Collection userRoles); - - void setRequiredRoles(Set requiredRoles); - - Set getRequiredRoles(); - - MarkingFunctions getMarkingFunctions(); - - void setMarkingFunctions(MarkingFunctions markingFunctions); - - ResponseObjectFactory getResponseObjectFactory(); - - void setResponseObjectFactory(ResponseObjectFactory responseObjectFactory); - - /** - * List of parameters that must be passed from the client for this query logic to work - * - * @return the required parameters - */ - Set getRequiredQueryParameters(); - - /** - * - * @return set of example queries - */ - Set getExampleQueries(); - - /** - * Return the DNs authorized access to this query logic. - * - * @return the set of DNs authorized access to this query logic, possibly null or empty - */ - Set getAuthorizedDNs(); - - /** - * Set the DNs authorized access to this query logic. - * - * @param allowedDNs - * the DNs authorized access - */ - void setAuthorizedDNs(Set allowedDNs); - - /** - * Return whether or not the provided collection of DNs contains at least oneDN that is authorized for access to this query logic. This will return true in - * the following cases: - *
    - *
  • The set of authorized DNs for this query logic is null or empty.
    • - *
  • The set of authorized DNs is not empty, and the provided collection contains a DN that is also found within the set of authorized DNs.
    • - *
- * - * @param dns - * the DNs to determine access rights for - * @return true if the collection contains at least one DN that has access to this query logic, or false otherwise - */ - default boolean containsDNWithAccess(Collection dns) { - Set authorizedDNs = getAuthorizedDNs(); - return authorizedDNs == null || authorizedDNs.isEmpty() || (dns != null && dns.stream().anyMatch(authorizedDNs::contains)); - } - - /** - * Set the map of DNs to query result limits. This should override the default limit returned by {@link #getMaxResults()} for any included DNs. - * - * @param dnResultLimits - * the map of DNs to query result limits - */ - void setDnResultLimits(Map dnResultLimits); - - /** - * Return the map of DNs to result limits. - * - * @return the map of DNs to query result limits - */ - Map getDnResultLimits(); - - /** - * Return the maximum number of results to include for the query for any DN present in the specified collection. If limits are found for multiple DNs in the - * collection, the smallest value will be returned. If the provided collection is null or empty, or if no limits are found for any DN, the value of - * {@link #getMaxResults()} will be returned. - * - * @param dns - * the DNs to determine the maximum number of results to include for the query. It's expected that this list represents all the DNs in the DN - * chain for an individual user. - * @return the maximum number of results to include - */ - default long getResultLimit(Collection dns) { - Map dnResultLimits = getDnResultLimits(); - if (dnResultLimits == null || dns == null) { - return getMaxResults(); - } - return dns.stream().filter(dnResultLimits::containsKey).map(dnResultLimits::get).min(Long::compareTo).orElseGet(this::getMaxResults); - } -} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogicFactory.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogicFactory.java deleted file mode 100644 index 14369567..00000000 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogicFactory.java +++ /dev/null @@ -1,23 +0,0 @@ -package datawave.microservice.query.logic; - -import datawave.webservice.query.exception.QueryException; - -import java.util.Collection; -import java.util.List; - -public interface QueryLogicFactory { - - /** - * - * @param name - * name of query logic - * @return new instance of QueryLogic class - * @throws IllegalArgumentException - * if query logic name does not exist - */ - QueryLogic getQueryLogic(String name, Collection userRoles) throws QueryException, IllegalArgumentException, CloneNotSupportedException; - - QueryLogic getQueryLogic(String name) throws QueryException, IllegalArgumentException, CloneNotSupportedException; - - List> getQueryLogicList(); -} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogicTransformer.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogicTransformer.java deleted file mode 100644 index 1b896b73..00000000 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/logic/QueryLogicTransformer.java +++ /dev/null @@ -1,26 +0,0 @@ -package datawave.microservice.query.logic; - -import datawave.microservice.query.exception.EmptyObjectException; -import datawave.webservice.query.cache.ResultsPage; -import datawave.webservice.result.BaseQueryResponse; -import org.apache.commons.collections4.Transformer; - -public interface QueryLogicTransformer extends Transformer { - - /* - * @return a jaxb response object that is specific to this QueryLogic - */ - BaseQueryResponse createResponse(ResultsPage resultList); - - /** - * Transforms the input object (leaving it unchanged) into some output object. - * - * @param input - * the object to be transformed, should be left unchanged - * @return a transformed object - * @throws EmptyObjectException - * if the result is empty - */ - @Override - O transform(I input) throws EmptyObjectException; -} diff --git a/query-microservices/query/query/src/main/java/datawave/microservice/query/result/event/DefaultResponseObjectFactory.java b/query-microservices/query/query/src/main/java/datawave/microservice/query/result/event/DefaultResponseObjectFactory.java deleted file mode 100644 index 20ad3f03..00000000 --- a/query-microservices/query/query/src/main/java/datawave/microservice/query/result/event/DefaultResponseObjectFactory.java +++ /dev/null @@ -1,119 +0,0 @@ -package datawave.microservice.query.result.event; - -import datawave.user.AuthorizationsListBase; -import datawave.user.DefaultAuthorizationsList; -import datawave.webservice.query.Query; -import datawave.webservice.query.QueryImpl; -import datawave.webservice.query.cachedresults.CacheableQueryRow; -import datawave.webservice.query.result.EdgeQueryResponseBase; -import datawave.webservice.query.result.edge.DefaultEdge; -import datawave.webservice.query.result.edge.EdgeBase; -import datawave.webservice.query.result.event.DefaultEvent; -import datawave.webservice.query.result.event.DefaultFacets; -import datawave.webservice.query.result.event.DefaultField; -import datawave.webservice.query.result.event.DefaultFieldCardinality; -import datawave.webservice.query.result.event.EventBase; -import datawave.webservice.query.result.event.FacetsBase; -import datawave.webservice.query.result.event.FieldBase; -import datawave.webservice.query.result.event.FieldCardinalityBase; -import datawave.webservice.query.result.event.ResponseObjectFactory; -import datawave.webservice.query.result.metadata.DefaultMetadataField; -import datawave.webservice.query.result.metadata.MetadataFieldBase; -import datawave.webservice.response.objects.DefaultKey; -import datawave.webservice.response.objects.KeyBase; -import datawave.webservice.result.DefaultEdgeQueryResponse; -import datawave.webservice.result.DefaultEventQueryResponse; -import datawave.webservice.result.EventQueryResponseBase; -import datawave.webservice.result.FacetQueryResponse; -import datawave.webservice.result.FacetQueryResponseBase; -import datawave.webservice.results.datadictionary.DataDictionaryBase; -import datawave.webservice.results.datadictionary.DefaultDataDictionary; -import datawave.webservice.results.datadictionary.DefaultDescription; -import datawave.webservice.results.datadictionary.DefaultFields; -import datawave.webservice.results.datadictionary.DescriptionBase; -import datawave.webservice.results.datadictionary.FieldsBase; -import org.springframework.stereotype.Component; - -@Component("responseObjectFactory") -public class DefaultResponseObjectFactory extends ResponseObjectFactory { - @Override - public EventBase getEvent() { - return new DefaultEvent(); - } - - @Override - public FieldBase getField() { - return new DefaultField(); - } - - @Override - public EventQueryResponseBase getEventQueryResponse() { - return new DefaultEventQueryResponse(); - } - - // TODO: JWO: Figure out how we're going to deal with cached results - @Override - public CacheableQueryRow getCacheableQueryRow() { - return null; - } - - @Override - public EdgeBase getEdge() { - return new DefaultEdge(); - } - - @Override - public EdgeQueryResponseBase getEdgeQueryResponse() { - return new DefaultEdgeQueryResponse(); - } - - @Override - public FacetQueryResponseBase getFacetQueryResponse() { - return new FacetQueryResponse(); - } - - @Override - public FacetsBase getFacets() { - return new DefaultFacets(); - } - - @Override - public FieldCardinalityBase getFieldCardinality() { - return new DefaultFieldCardinality(); - } - - @Override - public KeyBase getKey() { - return new DefaultKey(); - } - - @Override - public AuthorizationsListBase getAuthorizationsList() { - return new DefaultAuthorizationsList(); - } - - @Override - public Query getQueryImpl() { - return new QueryImpl(); - } - - @Override - public DataDictionaryBase getDataDictionary() { - return new DefaultDataDictionary(); - } - - @Override - public FieldsBase getFields() { - return new DefaultFields(); - } - - @Override - public DescriptionBase getDescription() { - return new DefaultDescription(); - } - - @Override - public MetadataFieldBase getMetadataField() { - return new DefaultMetadataField(); - } -} diff --git a/query-microservices/query/service/src/test/resources/MyTestQueryLogicFactory.xml b/query-microservices/query/service/src/test/resources/MyTestQueryLogicFactory.xml deleted file mode 100644 index 162eaa85..00000000 --- a/query-microservices/query/service/src/test/resources/MyTestQueryLogicFactory.xml +++ /dev/null @@ -1,805 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 2611 - - - - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/query-microservices/query/service/src/test/resources/QueryLogicFactoryAlt.xml b/query-microservices/query/service/src/test/resources/QueryLogicFactoryAlt.xml deleted file mode 100644 index ad5d589c..00000000 --- a/query-microservices/query/service/src/test/resources/QueryLogicFactoryAlt.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - - - - - From c4220f246fba7d3531726e43ed3b170ac83e5d1b Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 28 Apr 2021 16:53:44 -0400 Subject: [PATCH 035/218] Got the define call working for the query service. --- ...sImpl.java => DefaultQueryParameters.java} | 11 +- .../query/QueryManagementService.java | 3 +- .../query/config/QueryServiceConfig.java | 22 +- .../microservice/query/QueryServiceTest.java | 14 +- .../resources/MyTestQueryLogicFactory.xml | 106 ++++---- .../src/test/resources/config/application.yml | 246 +++++++++--------- 6 files changed, 199 insertions(+), 203 deletions(-) rename query-microservices/query-service/api/src/main/java/datawave/microservice/query/{QueryParametersImpl.java => DefaultQueryParameters.java} (98%) diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParametersImpl.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java similarity index 98% rename from query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParametersImpl.java rename to query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java index e22c5f67..d0b9b71e 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParametersImpl.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java @@ -14,9 +14,8 @@ import java.util.Date; import java.util.List; import java.util.Map; -import java.util.Objects; -public class QueryParametersImpl implements QueryParameters { +public class DefaultQueryParameters implements QueryParameters { private static final List KNOWN_PARAMS = Arrays.asList(QUERY_STRING, QUERY_NAME, QUERY_PERSISTENCE, QUERY_PAGESIZE, QUERY_PAGETIMEOUT, QUERY_AUTHORIZATIONS, QUERY_EXPIRATION, QUERY_TRACE, QUERY_BEGIN, QUERY_END, QUERY_VISIBILITY, QUERY_LOGIC_NAME, QUERY_POOL, @@ -41,7 +40,7 @@ public class QueryParametersImpl implements QueryParameters { protected int maxConcurrentTasks; protected MultiValueMap requestHeaders; - public QueryParametersImpl() { + public DefaultQueryParameters() { clear(); } @@ -149,7 +148,7 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; - QueryParametersImpl that = (QueryParametersImpl) o; + DefaultQueryParameters that = (DefaultQueryParameters) o; if (pagesize != that.pagesize) return false; @@ -247,7 +246,7 @@ public static Date parseEndDate(String s) throws ParseException { public static synchronized Date parseDate(String s, String defaultTime, String defaultMillisec) throws ParseException { Date d; ParseException e = null; - synchronized (QueryParametersImpl.dateFormat) { + synchronized (DefaultQueryParameters.dateFormat) { String str = s; if (str.equals("+24Hours")) { d = DateUtils.addDays(new Date(), 1); @@ -261,7 +260,7 @@ public static synchronized Date parseDate(String s, String defaultTime, String d } try { - d = QueryParametersImpl.dateFormat.parse(str); + d = DefaultQueryParameters.dateFormat.parse(str); // if any time value in HHmmss was set either by default or by the user // then we want to include ALL of that second by setting the milliseconds to 999 if (DateUtils.getFragmentInMilliseconds(d, Calendar.HOUR_OF_DAY) > 0) { diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 54876344..1549e6d4 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -151,7 +151,8 @@ protected void validateParameters(String queryLogicName, MultiValueMap roles = Collections.singleton("AuthorizedUser"); DatawaveUser uathDWUser = new DatawaveUser(DN, USER, null, roles, null, System.currentTimeMillis()); @@ -66,7 +64,15 @@ public void testQuery() { .build(); MultiValueMap map = new LinkedMultiValueMap<>(); - + map.set(DefaultQueryParameters.QUERY_STRING, "FIELD:SOME_VALUE"); + map.set(DefaultQueryParameters.QUERY_NAME, "The Greatest Query in the World - Tribute"); + map.set(DefaultQueryParameters.QUERY_PERSISTENCE, "PERSISTENT"); + map.set(DefaultQueryParameters.QUERY_AUTHORIZATIONS, "ALL"); + map.set(DefaultQueryParameters.QUERY_EXPIRATION, "20500101 000000.000"); + map.set(DefaultQueryParameters.QUERY_BEGIN, "20000101 000000.000"); + map.set(DefaultQueryParameters.QUERY_END, "20500101 000000.000"); + map.set(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, "ALL"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); try { diff --git a/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml b/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml index f736a016..bb02a338 100644 --- a/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml +++ b/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml @@ -22,39 +22,39 @@ - - - + + + - - + + - - - - - - - - - - - + + + + + + + + + + + - - - - - + + + + + @@ -62,12 +62,12 @@ - + - + @@ -75,61 +75,61 @@ - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + @@ -137,16 +137,16 @@ - - - - - + + + + + - + diff --git a/query-microservices/query-service/service/src/test/resources/config/application.yml b/query-microservices/query-service/service/src/test/resources/config/application.yml index 296ecb50..4c8277ab 100644 --- a/query-microservices/query-service/service/src/test/resources/config/application.yml +++ b/query-microservices/query-service/service/src/test/resources/config/application.yml @@ -16,7 +16,7 @@ hazelcast.client.enabled: false server: port: 0 non-secure-port: 0 - servlet.context-path: /query-starter-test + servlet.context-path: /query ssl: client-auth: NEED trust-store: 'classpath:testCA.p12' @@ -99,130 +99,130 @@ datawave: type-substitutions: "[datawave.data.type.DateType]": "datawave.data.type.RawDateType" -query: - parser: - skipTokenizeUnfieldedFields: - - "DOMETA" - tokenizedFields: - - "CONTENT" - uuidTypes: - - fieldName: "EVENT_ID" - definedView: "LuceneUUIDEventQuery" - allowedWildcardAfter: 28 - - fieldName: "UUID" - definedView: "LuceneUUIDEventQuery" - - fieldName: "PARENT_UUID" - definedView: "LuceneUUIDEventQuery" - - fieldName: "PAGE_ID" - definedView: "LuceneUUIDEventQuery" - - fieldName: "PAGE_TITLE" - definedView: "LuceneUUIDEventQuery" - storage: - backend: - LOCAL - lockManager: - LOCAL - syncStorage: - true - sendNotifications: - true - logic: - factory: - enabled: true - # Uncomment the following line to override the query logic beans to load - xmlBeansPath: "classpath:MyTestQueryLogicFactory.xml" + query: + parser: + skipTokenizeUnfieldedFields: + - "DOMETA" + tokenizedFields: + - "CONTENT" + uuidTypes: + - fieldName: "EVENT_ID" + definedView: "LuceneUUIDEventQuery" + allowedWildcardAfter: 28 + - fieldName: "UUID" + definedView: "LuceneUUIDEventQuery" + - fieldName: "PARENT_UUID" + definedView: "LuceneUUIDEventQuery" + - fieldName: "PAGE_ID" + definedView: "LuceneUUIDEventQuery" + - fieldName: "PAGE_TITLE" + definedView: "LuceneUUIDEventQuery" + storage: + backend: + LOCAL + lockManager: + LOCAL + syncStorage: + true + sendNotifications: + true + logic: + factory: + enabled: true + # Uncomment the following line to override the query logic beans to load + xmlBeansPath: "classpath:MyTestQueryLogicFactory.xml" - # The max page size that a user can request. 0 turns off this feature - maxPageSize: 10000 + # The max page size that a user can request. 0 turns off this feature + maxPageSize: 10000 - # The number of bytes at which a page will be returned, event if the pagesize has not been reached. 0 turns off this feature - pageByteTrigger: 0 - logics: - BaseEventQuery: - accumuloPassword: ${warehouse.accumulo.password} - tableName: ${warehouse.tables.shard.name} - dateIndexTableName: ${warehouse.tables.dateIndex.name} - defaultDateTypeName: "EVENT" - metadataTableName: ${warehouse.tables.metadata.name} - indexTableName: ${warehouse.tables.index.name} - reverseIndexTableName: ${warehouse.tables.reverseIndex.name} - maxResults: -1 - queryThreads: ${warehouse.defaults.queryThreads} - indexLookupThreads: ${warehouse.defaults.indexLookupThreads} - dateIndexThreads: ${warehouse.defaults.dateIndexThreads} - fullTableScanEnabled: ${warehouse.defaults.fullTableScanEnabled} - includeDataTypeAsField: false - disableIndexOnlyDocuments: false - indexOnlyFilterFunctionsEnabled: false - includeHierarchyFields: false - hierarchyFieldOptions: - "FOO": "BAR" - baseIteratorPriority: ${warehouse.defaults.baseIteratorPriority} - maxIndexScanTimeMillis: ${warehouse.defaults.maxIndexScanTimeMillis} - collapseUids: false - collapseUidsThreshold: -1 - useEnrichers: true - contentFieldNames: - - 'CONTENT' - realmSuffixExclusionPatterns: - - '<.*>$' - minimumSelectivity: .2 - enricherClassNames: - - 'datawave.query.enrich.DatawaveTermFrequencyEnricher' - useFilters: false - filterClassNames: - - 'foo.bar' - filterOptions: - - 'bar.foo' - auditType: "NONE" - logicDescription: "Retrieve sharded events/documents, leveraging the global index tables as needed" - eventPerDayThreshold: ${warehouse.defaults.eventPerDayThreshold} - shardsPerDayThreshold: ${warehouse.defaults.shardsPerDayThreshold} - maxTermThreshold: ${warehouse.defaults.maxTermThreshold} - maxDepthThreshold: ${warehouse.defaults.maxDepthThreshold} - maxUnfieldedExpansionThreshold: ${warehouse.defaults.maxUnfieldedExpansionThreshold} - maxValueExpansionThreshold: ${warehouse.defaults.maxValueExpansionThreshold} - maxOrExpansionThreshold: ${warehouse.defaults.maxOrExpansionThreshold} - maxOrRangeThreshold: ${warehouse.defaults.maxOrRangeThreshold} - maxOrExpansionFstThreshold: ${warehouse.defaults.maxOrExpansionFstThreshold} - maxFieldIndexRangeSplit: ${warehouse.defaults.maxFieldIndexRangeSplit} - maxIvaratorSources: ${warehouse.defaults.maxIvaratorSources} - maxEvaluationPipelines: ${warehouse.defaults.maxEvaluationPipelines} - maxPipelineCachedResults: ${warehouse.defaults.maxPipelineCachedResults} - hdfsSiteConfigURLs: ${warehouse.defaults.hdfsSiteConfigURLs} - zookeeperConfig: ${warehouse.accumulo.zookeepers} - ivaratorCacheDirConfigs: - - basePathURI: "hdfs:///IvaratorCache" - ivaratorFstHdfsBaseURIs: ${warehouse.accumulo.ivaratorFstHdfsBaseURIs} - ivaratorCacheBufferSize: ${warehouse.defaults.ivaratorCacheBufferSize} - ivaratorMaxOpenFiles: ${warehouse.defaults.ivaratorMaxOpenFiles} - ivaratorCacheScanPersistThreshold: ${warehouse.defaults.ivaratorCacheScanPersistThreshold} - ivaratorCacheScanTimeoutMinutes: ${warehouse.defaults.ivaratorCacheScanTimeoutMinutes} - eventQueryDataDecoratorTransformer: - requestedDecorators: - - "CSV" - - "WIKIPEDIA" - dataDecorators: - "CSV": - "EVENT_ID": "https://localhost:8443/DataWave/Query/lookupUUID/EVENT_ID?uuid=@field_value@&parameters=data.decorators:CSV" - "UUID": "https://localhost:8443/DataWave/Query/lookupUUID/UUID?uuid=@field_value@&parameters=data.decorators:CSV" - "PARENT_UUID": "https://localhost:8443/DataWave/Query/lookupUUID/PARENT_UUID?uuid=@field_value@&parameters=data.decorators:CSV" - "WIKIPEDIA": - "PAGE_ID": "https://localhost:8443/DataWave/Query/lookupUUID/PAGE_ID?uuid=@field_value@&parameters=data.decorators:WIKIPEDIA" - "PAGE_TITLE": "https://localhost:8443/DataWave/Query/lookupUUID/PAGE_TITLE?uuid=@field_value@&parameters=data.decorators:WIKIPEDIA" - modelTableName: ${warehouse.tables.model.name} - modelName: ${warehouse.defaults.modelName} - querySyntaxParsers: - JEXL: "" - LUCENE: "LuceneToJexlQueryParser" - LUCENE-UUID: "LuceneToJexlUUIDQueryParser" - TOKENIZED-LUCENE: "TokenizedLuceneToJexlQueryParser" - sendTimingToStatsd: false - collectQueryMetrics: true - logTimingDetails: false - statsdHost: ${warehouse.statsd.host} - statsdPort: ${warehouse.statsd.port} - evaluationOnlyFields: "" + # The number of bytes at which a page will be returned, event if the pagesize has not been reached. 0 turns off this feature + pageByteTrigger: 0 + logics: + BaseEventQuery: + accumuloPassword: ${warehouse.accumulo.password} + tableName: ${warehouse.tables.shard.name} + dateIndexTableName: ${warehouse.tables.dateIndex.name} + defaultDateTypeName: "EVENT" + metadataTableName: ${warehouse.tables.metadata.name} + indexTableName: ${warehouse.tables.index.name} + reverseIndexTableName: ${warehouse.tables.reverseIndex.name} + maxResults: -1 + queryThreads: ${warehouse.defaults.queryThreads} + indexLookupThreads: ${warehouse.defaults.indexLookupThreads} + dateIndexThreads: ${warehouse.defaults.dateIndexThreads} + fullTableScanEnabled: ${warehouse.defaults.fullTableScanEnabled} + includeDataTypeAsField: false + disableIndexOnlyDocuments: false + indexOnlyFilterFunctionsEnabled: false + includeHierarchyFields: false + hierarchyFieldOptions: + "FOO": "BAR" + baseIteratorPriority: ${warehouse.defaults.baseIteratorPriority} + maxIndexScanTimeMillis: ${warehouse.defaults.maxIndexScanTimeMillis} + collapseUids: false + collapseUidsThreshold: -1 + useEnrichers: true + contentFieldNames: + - 'CONTENT' + realmSuffixExclusionPatterns: + - '<.*>$' + minimumSelectivity: .2 + enricherClassNames: + - 'datawave.query.enrich.DatawaveTermFrequencyEnricher' + useFilters: false + filterClassNames: + - 'foo.bar' + filterOptions: + - 'bar.foo' + auditType: "NONE" + logicDescription: "Retrieve sharded events/documents, leveraging the global index tables as needed" + eventPerDayThreshold: ${warehouse.defaults.eventPerDayThreshold} + shardsPerDayThreshold: ${warehouse.defaults.shardsPerDayThreshold} + maxTermThreshold: ${warehouse.defaults.maxTermThreshold} + maxDepthThreshold: ${warehouse.defaults.maxDepthThreshold} + maxUnfieldedExpansionThreshold: ${warehouse.defaults.maxUnfieldedExpansionThreshold} + maxValueExpansionThreshold: ${warehouse.defaults.maxValueExpansionThreshold} + maxOrExpansionThreshold: ${warehouse.defaults.maxOrExpansionThreshold} + maxOrRangeThreshold: ${warehouse.defaults.maxOrRangeThreshold} + maxOrExpansionFstThreshold: ${warehouse.defaults.maxOrExpansionFstThreshold} + maxFieldIndexRangeSplit: ${warehouse.defaults.maxFieldIndexRangeSplit} + maxIvaratorSources: ${warehouse.defaults.maxIvaratorSources} + maxEvaluationPipelines: ${warehouse.defaults.maxEvaluationPipelines} + maxPipelineCachedResults: ${warehouse.defaults.maxPipelineCachedResults} + hdfsSiteConfigURLs: ${warehouse.defaults.hdfsSiteConfigURLs} + zookeeperConfig: ${warehouse.accumulo.zookeepers} + ivaratorCacheDirConfigs: + - basePathURI: "hdfs:///IvaratorCache" + ivaratorFstHdfsBaseURIs: ${warehouse.accumulo.ivaratorFstHdfsBaseURIs} + ivaratorCacheBufferSize: ${warehouse.defaults.ivaratorCacheBufferSize} + ivaratorMaxOpenFiles: ${warehouse.defaults.ivaratorMaxOpenFiles} + ivaratorCacheScanPersistThreshold: ${warehouse.defaults.ivaratorCacheScanPersistThreshold} + ivaratorCacheScanTimeoutMinutes: ${warehouse.defaults.ivaratorCacheScanTimeoutMinutes} + eventQueryDataDecoratorTransformer: + requestedDecorators: + - "CSV" + - "WIKIPEDIA" + dataDecorators: + "CSV": + "EVENT_ID": "https://localhost:8443/DataWave/Query/lookupUUID/EVENT_ID?uuid=@field_value@&parameters=data.decorators:CSV" + "UUID": "https://localhost:8443/DataWave/Query/lookupUUID/UUID?uuid=@field_value@&parameters=data.decorators:CSV" + "PARENT_UUID": "https://localhost:8443/DataWave/Query/lookupUUID/PARENT_UUID?uuid=@field_value@&parameters=data.decorators:CSV" + "WIKIPEDIA": + "PAGE_ID": "https://localhost:8443/DataWave/Query/lookupUUID/PAGE_ID?uuid=@field_value@&parameters=data.decorators:WIKIPEDIA" + "PAGE_TITLE": "https://localhost:8443/DataWave/Query/lookupUUID/PAGE_TITLE?uuid=@field_value@&parameters=data.decorators:WIKIPEDIA" + modelTableName: ${warehouse.tables.model.name} + modelName: ${warehouse.defaults.modelName} + querySyntaxParsers: + JEXL: "" + LUCENE: "LuceneToJexlQueryParser" + LUCENE-UUID: "LuceneToJexlUUIDQueryParser" + TOKENIZED-LUCENE: "TokenizedLuceneToJexlQueryParser" + sendTimingToStatsd: false + collectQueryMetrics: true + logTimingDetails: false + statsdHost: ${warehouse.statsd.host} + statsdPort: ${warehouse.statsd.port} + evaluationOnlyFields: "" audit-client: discovery: From cff323a3b00a7f7872769c981e0bcd1c1db26efa Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 30 Apr 2021 11:50:36 -0400 Subject: [PATCH 036/218] Fixed a javax validation api issue and starter pulling classes out of datawave ws common --- query-microservices/query-service/api/pom.xml | 10 +++++----- .../microservice/query/QueryManagementService.java | 12 ++---------- .../query/config/QueryServiceConfig.java | 2 +- .../microservice/query/QueryServiceTest.java | 2 +- .../src/test/resources/config/application.yml | 2 +- 5 files changed, 10 insertions(+), 18 deletions(-) diff --git a/query-microservices/query-service/api/pom.xml b/query-microservices/query-service/api/pom.xml index e5a635d5..6269478f 100644 --- a/query-microservices/query-service/api/pom.xml +++ b/query-microservices/query-service/api/pom.xml @@ -11,7 +11,7 @@ 1.0-SNAPSHOT 1.3-SNAPSHOT - 2.0.1.Final + 2.0.2 @@ -21,8 +21,8 @@ ${version.microservice.base-rest-responses} - javax.validation - validation-api + jakarta.validation + jakarta.validation-api ${version.validation-api} @@ -33,8 +33,8 @@ base-rest-responses - javax.validation - validation-api + jakarta.validation + jakarta.validation-api junit diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 1549e6d4..f97b0a1b 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import datawave.marking.SecurityMarking; import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.common.audit.PrivateAuditConstants; import datawave.microservice.common.storage.QueryPool; import datawave.microservice.common.storage.QueryStorageCache; import datawave.microservice.common.storage.TaskKey; @@ -14,9 +15,6 @@ import datawave.microservice.query.util.QueryUtil; import datawave.security.util.ProxiedEntityUtils; import datawave.webservice.common.audit.AuditParameters; -import datawave.webservice.common.audit.PrivateAuditConstants; -import datawave.webservice.common.exception.DatawaveWebApplicationException; -import datawave.webservice.common.exception.UnauthorizedException; import datawave.webservice.query.Query; import datawave.webservice.query.exception.BadRequestQueryException; import datawave.webservice.query.exception.DatawaveErrorCode; @@ -24,7 +22,6 @@ import datawave.webservice.query.exception.UnauthorizedQueryException; import datawave.webservice.query.result.event.ResponseObjectFactory; import datawave.webservice.query.util.QueryUncaughtExceptionHandler; -import datawave.webservice.result.GenericResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; @@ -88,8 +85,6 @@ public TaskKey define(String queryLogicName, MultiValueMap parame // TODO: JWO: Figure out how to make query metrics work with our new architecture. Datawave issue #1156 return taskKey; - } catch (DatawaveWebApplicationException e) { - throw e; } catch (Exception e) { log.error("Unknown error storing query", e); throw new BadRequestQueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR, e); @@ -231,10 +226,7 @@ protected void validateQueryLogic(QueryLogic queryLogic, MultiValueMap dnList = currentUser.getDNs(); if (!queryLogic.containsDNWithAccess(dnList)) { - UnauthorizedQueryException qe = new UnauthorizedQueryException("None of the DNs used have access to this query logic: " + dnList, 401); - GenericResponse response = new GenericResponse<>(); - response.addException(qe); - throw new UnauthorizedException(qe, response); + throw new UnauthorizedQueryException("None of the DNs used have access to this query logic: " + dnList, 401); } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java index a2967672..4a7a9f89 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java @@ -10,7 +10,7 @@ @Configuration public class QueryServiceConfig { - + @Bean @ConditionalOnMissingBean @RequestScope diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java index 7a4c1f2f..83ecc5a8 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java @@ -72,7 +72,7 @@ public void testQuery() { map.set(DefaultQueryParameters.QUERY_BEGIN, "20000101 000000.000"); map.set(DefaultQueryParameters.QUERY_END, "20500101 000000.000"); map.set(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, "ALL"); - + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); try { diff --git a/query-microservices/query-service/service/src/test/resources/config/application.yml b/query-microservices/query-service/service/src/test/resources/config/application.yml index 4c8277ab..e7b5d9fa 100644 --- a/query-microservices/query-service/service/src/test/resources/config/application.yml +++ b/query-microservices/query-service/service/src/test/resources/config/application.yml @@ -173,7 +173,7 @@ datawave: filterClassNames: - 'foo.bar' filterOptions: - - 'bar.foo' + 'bar': "foo" auditType: "NONE" logicDescription: "Retrieve sharded events/documents, leveraging the global index tables as needed" eventPerDayThreshold: ${warehouse.defaults.eventPerDayThreshold} From 9f3fb7bab9c30f4e84680d3d9c06d16bc8113ac3 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 5 May 2021 12:44:53 -0400 Subject: [PATCH 037/218] lock stuff --- .../microservice/lock/LockManager.java | 6 + .../datawave/microservice/lock/Semaphore.java | 17 +++ .../hazelcast/HazelcastLockManager.java | 33 +++++ .../hazelcast/HazelcastSemaphore.java | 69 +++++++++++ .../zookeeper/ZookeeperLockManager.java | 29 +++++ .../zookeeper/ZookeeperSemaphore.java | 90 ++++++++++++++ .../lock/local/LocalLockManager.java | 17 +++ .../lock/local/LocalSemaphore.java | 71 +++++++++++ .../microservice/query/QueryController.java | 47 ++++++- .../query/QueryManagementService.java | 115 +++++++++++++++++- .../src/test/resources/config/application.yml | 3 + 11 files changed, 489 insertions(+), 8 deletions(-) create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/LockManager.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Semaphore.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLockManager.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastSemaphore.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLockManager.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperSemaphore.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLockManager.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalSemaphore.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/LockManager.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/LockManager.java new file mode 100644 index 00000000..f452382f --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/LockManager.java @@ -0,0 +1,6 @@ +package datawave.microservice.lock; + +public interface LockManager { + + Semaphore getSemaphore(String name, int permits) throws Exception; +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Semaphore.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Semaphore.java new file mode 100644 index 00000000..1156fff6 --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Semaphore.java @@ -0,0 +1,17 @@ +package datawave.microservice.lock; + +import java.util.concurrent.TimeUnit; + +public interface Semaphore { + String getName(); + void acquire() throws Exception; + void acquire(int permits) throws Exception; + int availablePermits() throws Exception; + int drainPermits() throws Exception; + void release(); + void release(int permits); + boolean tryAcquire(); + boolean tryAcquire(int permits); + boolean tryAcquire(long timeout, TimeUnit unit) throws InterruptedException; + boolean tryAcquire(int permits, long timeout, TimeUnit unit) throws InterruptedException; +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLockManager.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLockManager.java new file mode 100644 index 00000000..24b4632d --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLockManager.java @@ -0,0 +1,33 @@ +package datawave.microservice.lock.distributed.hazelcast; + +import com.hazelcast.core.HazelcastInstance; +import com.hazelcast.cp.CPSubsystem; +import com.hazelcast.cp.ISemaphore; +import datawave.microservice.lock.LockManager; +import datawave.microservice.lock.Semaphore; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +@Component("lockManager") +@ConditionalOnMissingBean(name = "lockManager") +@ConditionalOnBean(HazelcastInstance.class) +@ConditionalOnProperty(name = "datawave.lock.type", havingValue = "hazelcast") +public class HazelcastLockManager implements LockManager { + + private CPSubsystem cpSubsystem; + + public HazelcastLockManager(HazelcastInstance hazelcastInstance) { + this.cpSubsystem = hazelcastInstance.getCPSubsystem(); + } + + @Override + public Semaphore getSemaphore(String name, int permits) throws Exception { + ISemaphore iSemaphore = cpSubsystem.getSemaphore(name); + if (!iSemaphore.init(permits)) { + throw new Exception("Unable to initialize ISemaphore"); + } + return new HazelcastSemaphore(iSemaphore); + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastSemaphore.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastSemaphore.java new file mode 100644 index 00000000..c89d9157 --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastSemaphore.java @@ -0,0 +1,69 @@ +package datawave.microservice.lock.distributed.hazelcast; + +import com.hazelcast.cp.ISemaphore; +import datawave.microservice.lock.Semaphore; + +import java.util.concurrent.TimeUnit; + +public class HazelcastSemaphore implements Semaphore { + private ISemaphore semaphore; + + HazelcastSemaphore(ISemaphore semaphore) { + this.semaphore = semaphore; + } + + @Override + public String getName() { + return semaphore.getName(); + } + + @Override + public void acquire() throws InterruptedException { + semaphore.acquire(); + } + + @Override + public void acquire(int permits) throws InterruptedException { + semaphore.acquire(permits); + } + + @Override + public int availablePermits() { + return semaphore.availablePermits(); + } + + @Override + public int drainPermits() { + return semaphore.drainPermits(); + } + + @Override + public void release() { + semaphore.release(); + } + + @Override + public void release(int permits) { + semaphore.release(permits); + } + + @Override + public boolean tryAcquire() { + return semaphore.tryAcquire(); + } + + @Override + public boolean tryAcquire(int permits) { + return semaphore.tryAcquire(permits); + } + + @Override + public boolean tryAcquire(long timeout, TimeUnit unit) throws InterruptedException { + return semaphore.tryAcquire(timeout, unit); + } + + @Override + public boolean tryAcquire(int permits, long timeout, TimeUnit unit) throws InterruptedException { + return semaphore.tryAcquire(permits, timeout, unit); + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLockManager.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLockManager.java new file mode 100644 index 00000000..49a52595 --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLockManager.java @@ -0,0 +1,29 @@ +package datawave.microservice.lock.distributed.zookeeper; + +import datawave.microservice.lock.LockManager; +import datawave.microservice.lock.Semaphore; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreV2; +import org.apache.curator.framework.recipes.shared.SharedCount; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +@Component("lockManager") +@ConditionalOnMissingBean(name = "lockManager") +@ConditionalOnBean(CuratorFramework.class) +@ConditionalOnProperty(name = "datawave.lock.type", havingValue = "zookeeper") +public class ZookeeperLockManager implements LockManager { + + final private CuratorFramework curatorFramework; + + public ZookeeperLockManager(CuratorFramework curatorFramework) { + this.curatorFramework = curatorFramework; + } + + @Override + public Semaphore getSemaphore(String name, int permits) throws Exception { + return new ZookeeperSemaphore(name, new InterProcessSemaphoreV2(curatorFramework, name, new SharedCount(curatorFramework, name + "/permits", permits)), curatorFramework); + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperSemaphore.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperSemaphore.java new file mode 100644 index 00000000..78806a73 --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperSemaphore.java @@ -0,0 +1,90 @@ +package datawave.microservice.lock.distributed.zookeeper; + +import datawave.microservice.lock.Semaphore; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreV2; +import org.apache.curator.framework.recipes.locks.Lease; + +import java.util.LinkedList; +import java.util.concurrent.TimeUnit; + +public class ZookeeperSemaphore implements Semaphore { + + private String path; + private InterProcessSemaphoreV2 semaphore; + final private LinkedList leases = new LinkedList<>(); + private CuratorFramework curatorFramework; + + ZookeeperSemaphore(String path, InterProcessSemaphoreV2 semaphore, CuratorFramework curatorFramework) { + this.path = path; + this.semaphore = semaphore; + this.curatorFramework = curatorFramework;; + } + + @Override + public String getName() { + return null; + } + + @Override + public void acquire() throws Exception { + synchronized(leases) { + leases.push(semaphore.acquire()); + } + } + + @Override + public void acquire(int permits) throws Exception { + synchronized (leases) { + semaphore.acquire(permits).forEach(leases::push); + } + } + + @Override + public int availablePermits() throws Exception { + return curatorFramework.getChildren().forPath(path).size(); + } + + @Override + public int drainPermits() throws Exception { + throw new UnsupportedOperationException("Unable to drainPermits for ZookeeperSemaphore"); + } + + @Override + public void release() { + synchronized(leases) { + if (leases.size() > 0) { + semaphore.returnLease(leases.pop()); + } + } + } + + @Override + public void release(int permits) { + synchronized(leases) { + while (permits-- > 0 && !leases.isEmpty()) { + semaphore.returnLease(leases.pop()); + } + } + } + + @Override + public boolean tryAcquire() { + throw new UnsupportedOperationException("Unable to tryAcquire for ZookeeperSemaphore"); + } + + @Override + public boolean tryAcquire(int permits) { + throw new UnsupportedOperationException("Unable to tryAcquire for ZookeeperSemaphore"); + } + + @Override + public boolean tryAcquire(long timeout, TimeUnit unit) throws InterruptedException { + throw new UnsupportedOperationException("Unable to tryAcquire for ZookeeperSemaphore"); + } + + @Override + public boolean tryAcquire(int permits, long timeout, TimeUnit unit) throws InterruptedException { + throw new UnsupportedOperationException("Unable to tryAcquire for ZookeeperSemaphore"); + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLockManager.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLockManager.java new file mode 100644 index 00000000..bee76e80 --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLockManager.java @@ -0,0 +1,17 @@ +package datawave.microservice.lock.local; + +import datawave.microservice.lock.LockManager; +import datawave.microservice.lock.Semaphore; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +@Component("lockManager") +@ConditionalOnMissingBean(name = "lockManager") +@ConditionalOnProperty(name = "datawave.lock.type", havingValue = "local") +public class LocalLockManager implements LockManager { + @Override + public Semaphore getSemaphore(String name, int permits) { + return new LocalSemaphore(name, new java.util.concurrent.Semaphore(permits)); + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalSemaphore.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalSemaphore.java new file mode 100644 index 00000000..ac2ff412 --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalSemaphore.java @@ -0,0 +1,71 @@ +package datawave.microservice.lock.local; + +import datawave.microservice.lock.Semaphore; + +import java.util.concurrent.TimeUnit; + +public class LocalSemaphore implements Semaphore { + + private String name; + private java.util.concurrent.Semaphore semaphore; + + LocalSemaphore(String name, java.util.concurrent.Semaphore semaphore) { + this.name = name; + this.semaphore = semaphore; + } + + @Override + public String getName() { + return name; + } + + @Override + public void acquire() throws InterruptedException { + semaphore.acquire(); + } + + @Override + public void acquire(int permits) throws InterruptedException { + semaphore.acquire(permits); + } + + @Override + public int availablePermits() { + return semaphore.availablePermits(); + } + + @Override + public int drainPermits() { + return semaphore.drainPermits(); + } + + @Override + public void release() { + semaphore.release(); + } + + @Override + public void release(int permits) { + semaphore.release(); + } + + @Override + public boolean tryAcquire() { + return semaphore.tryAcquire(); + } + + @Override + public boolean tryAcquire(int permits) { + return semaphore.tryAcquire(permits); + } + + @Override + public boolean tryAcquire(long timeout, TimeUnit unit) throws InterruptedException { + return semaphore.tryAcquire(timeout, unit); + } + + @Override + public boolean tryAcquire(int permits, long timeout, TimeUnit unit) throws InterruptedException { + return semaphore.tryAcquire(permits, timeout, unit); + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index d7511d96..326e66da 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -1,9 +1,11 @@ package datawave.microservice.query; import com.codahale.metrics.annotation.Timed; +import com.hazelcast.core.HazelcastInstance; import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.common.storage.TaskKey; import datawave.microservice.query.web.annotation.EnrichQueryMetrics; +import datawave.webservice.result.BaseQueryResponse; import datawave.webservice.result.GenericResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,8 +23,8 @@ public class QueryController { private final Logger log = LoggerFactory.getLogger(this.getClass()); - private QueryManagementService queryManagementService; - + private final QueryManagementService queryManagementService; + public QueryController(QueryManagementService queryManagementService) { this.queryManagementService = queryManagementService; } @@ -31,13 +33,48 @@ public QueryController(QueryManagementService queryManagementService) { @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE) @RequestMapping(path = "{queryLogic}/define", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public GenericResponse define(@PathVariable(name = "queryLogic") String queryLogicName, @RequestParam MultiValueMap parameters, + public GenericResponse define(@PathVariable(name = "queryLogic") String queryLogic, @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { - TaskKey taskKey = queryManagementService.define(queryLogicName, parameters, currentUser); + TaskKey taskKey = queryManagementService.define(queryLogic, parameters, currentUser); GenericResponse resp = new GenericResponse<>(); resp.setResult(taskKey.getQueryId().toString()); return resp; } - + + @Timed(name = "dw.query.createQuery", absolute = true) + @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE) + @RequestMapping(path = "{queryLogic}/define", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse create(@PathVariable(name = "queryLogic") String queryLogic, @RequestParam MultiValueMap parameters, + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + TaskKey taskKey = queryManagementService.create(queryLogic, parameters, currentUser); + + GenericResponse resp = new GenericResponse<>(); + resp.setHasResults(true); + resp.setResult(taskKey.getQueryId().toString()); + return resp; + } + + @Timed(name = "dw.query.next", absolute = true) + @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.NEXT) + @RequestMapping(path = "{queryId}/next", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public BaseQueryResponse next(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + // need to find the query in the query storage cache, and then check for results on the results queue + + // figure out what user this is + + // check to see if we're already handling a next call for this user/query + + // check to see if the query id exists in the query cache + + // make sure that this is the caller's query + + // get the next set of results + + // return the results to the user + + return null; + } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index f97b0a1b..e8bf17fe 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -3,11 +3,13 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import datawave.marking.SecurityMarking; +import datawave.microservice.audit.AuditClient; import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.common.audit.PrivateAuditConstants; import datawave.microservice.common.storage.QueryPool; import datawave.microservice.common.storage.QueryStorageCache; import datawave.microservice.common.storage.TaskKey; +import datawave.microservice.lock.LockManager; import datawave.microservice.query.config.QueryExpirationProperties; import datawave.microservice.query.config.QueryProperties; import datawave.microservice.query.logic.QueryLogic; @@ -15,6 +17,7 @@ import datawave.microservice.query.util.QueryUtil; import datawave.security.util.ProxiedEntityUtils; import datawave.webservice.common.audit.AuditParameters; +import datawave.webservice.common.audit.Auditor; import datawave.webservice.query.Query; import datawave.webservice.query.exception.BadRequestQueryException; import datawave.webservice.query.exception.DatawaveErrorCode; @@ -40,25 +43,42 @@ public class QueryManagementService { // Note: QueryParameters need to be request scoped private QueryParameters queryParameters; + // Note: SecurityMarking needs to be request scoped private SecurityMarking securityMarking; private QueryLogicFactory queryLogicFactory; private ResponseObjectFactory responseObjectFactory; private QueryStorageCache queryStorageCache; + private AuditClient auditClient; + private LockManager lockManager; // TODO: JWO: Pull these from configuration instead private final int PAGE_TIMEOUT_MIN = 1; private final int PAGE_TIMEOUT_MAX = QueryExpirationProperties.PAGE_TIMEOUT_MIN_DEFAULT; public QueryManagementService(QueryProperties queryProperties, QueryParameters queryParameters, SecurityMarking securityMarking, - QueryLogicFactory queryLogicFactory, ResponseObjectFactory responseObjectFactory, QueryStorageCache queryStorageCache) { + QueryLogicFactory queryLogicFactory, ResponseObjectFactory responseObjectFactory, QueryStorageCache queryStorageCache, AuditClient auditClient, LockManager lockManager) { this.queryProperties = queryProperties; this.queryParameters = queryParameters; this.securityMarking = securityMarking; this.queryLogicFactory = queryLogicFactory; this.responseObjectFactory = responseObjectFactory; this.queryStorageCache = queryStorageCache; + this.auditClient = auditClient; + this.lockManager = lockManager; } - + + /** + * Defines a datawave query. + * + * Validates the query parameters using the base validation, query logic-specific validation, and markings validation. + * If the parameters are valid, the query will be stored in the query storage cache where it can be acted upon in a subsequent call. + * + * @param queryLogicName + * @param parameters + * @param currentUser + * @return + * @throws QueryException + */ public TaskKey define(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { // validate query and get a query logic QueryLogic queryLogic = validateQuery(queryLogicName, parameters, currentUser); @@ -90,7 +110,96 @@ public TaskKey define(String queryLogicName, MultiValueMap parame throw new BadRequestQueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR, e); } } - + + /** + * Creates a datawave query. + * + * Validates the query parameters using the base validation, query logic-specific validation, and markings validation. + * If the parameters are valid, the query will be stored in the query storage cache where it can be acted upon in a subsequent call. + * + * @param queryLogicName + * @param parameters + * @param currentUser + * @return + * @throws QueryException + */ + public TaskKey create(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + // validate query and get a query logic + QueryLogic queryLogic = validateQuery(queryLogicName, parameters, currentUser); + + String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + log.trace(userId + " has authorizations " + currentUser.getPrimaryUser().getAuths()); + + // set some audit parameters which are used internally + String userDn = currentUser.getPrimaryUser().getDn().subjectDN(); + setInternalAuditParameters(queryLogicName, userDn, parameters); + + // send an audit record to the auditor + Query query = createQuery(queryLogicName, parameters, userDn, currentUser.getDNs()); + audit(query, queryLogic, parameters, currentUser); + + try { + // persist the query w/ query id in the query storage cache + // @formatter:off + TaskKey taskKey = queryStorageCache.storeQuery( + new QueryPool(getPoolName()), + query, + getMaxConcurrentTasks(queryLogic)); + // @formatter:on + + // TODO: JWO: Figure out how to make query tracing work with our new architecture. Datawave issue #1155 + + // TODO: JWO: Figure out how to make query metrics work with our new architecture. Datawave issue #1156 + + return taskKey; + } catch (Exception e) { + log.error("Unknown error storing query", e); + throw new BadRequestQueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR, e); + } + } + + protected void audit(Query query, QueryLogic queryLogic, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + Auditor.AuditType auditType = queryLogic.getAuditType(query); + + parameters.add(PrivateAuditConstants.AUDIT_TYPE, auditType.name()); + if (auditType != Auditor.AuditType.NONE) { + // audit the query before execution + try { + try { + List selectors = queryLogic.getSelectors(query); + if (selectors != null && !selectors.isEmpty()) { + parameters.put(PrivateAuditConstants.SELECTORS, selectors); + } + } catch (Exception e) { + log.error("Error accessing query selector", e); + } + + // is the user didn't set an audit id, use the query id + if (!parameters.containsKey(AuditParameters.AUDIT_ID)){ + parameters.set(AuditParameters.AUDIT_ID, query.getId().toString()); + } + + // TODO: Write a test to ensure that the audit id we set is used + // @formatter:off + auditClient.submit(new AuditClient.Request.Builder() + .withParams(parameters) + .withQueryExpression(query.getQuery()) + .withProxiedUserDetails(currentUser) + .withMarking(securityMarking) + .withAuditType(auditType) + .withQueryLogic(queryLogic.getLogicName()) + .build()); + // @formatter:on + } catch (IllegalArgumentException e) { + log.error("Error validating audit parameters", e); + throw new BadRequestQueryException(DatawaveErrorCode.MISSING_REQUIRED_PARAMETER, e); + } catch (Exception e) { + log.error("Error auditing query", e); + throw new BadRequestQueryException(DatawaveErrorCode.QUERY_AUDITING_ERROR, e); + } + } + } + protected String getPoolName() { return (queryParameters.getPool() != null) ? queryParameters.getPool() : queryProperties.getDefaultParams().getPool(); } diff --git a/query-microservices/query-service/service/src/test/resources/config/application.yml b/query-microservices/query-service/service/src/test/resources/config/application.yml index e7b5d9fa..cf3c378a 100644 --- a/query-microservices/query-service/service/src/test/resources/config/application.yml +++ b/query-microservices/query-service/service/src/test/resources/config/application.yml @@ -93,6 +93,9 @@ warehouse: modelName: 'DATAWAVE' datawave: + lock: + type: local + metadata: all-metadata-auths: - PRIVATE,PUBLIC From 542e317325bfdf416e7e95cfdc0401f62413edf3 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 5 May 2021 13:38:42 -0400 Subject: [PATCH 038/218] added lock implementation to lockmanager --- .../java/datawave/microservice/lock/Lock.java | 17 +++++++ .../microservice/lock/LockManager.java | 2 + .../datawave/microservice/lock/Semaphore.java | 10 ++++ .../distributed/hazelcast/HazelcastLock.java | 44 +++++++++++++++++ .../hazelcast/HazelcastLockManager.java | 6 +++ .../hazelcast/HazelcastSemaphore.java | 2 +- .../distributed/zookeeper/ZookeeperLock.java | 46 ++++++++++++++++++ .../zookeeper/ZookeeperLockManager.java | 7 +++ .../zookeeper/ZookeeperSemaphore.java | 11 ++--- .../microservice/lock/local/LocalLock.java | 47 +++++++++++++++++++ .../lock/local/LocalLockManager.java | 8 ++++ .../lock/local/LocalSemaphore.java | 4 +- 12 files changed, 195 insertions(+), 9 deletions(-) create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Lock.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLock.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLock.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLock.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Lock.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Lock.java new file mode 100644 index 00000000..5ac01777 --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Lock.java @@ -0,0 +1,17 @@ +package datawave.microservice.lock; + +import java.util.concurrent.TimeUnit; + +public interface Lock { + String getName(); + + void lock() throws Exception; + + void lockInterruptibly() throws InterruptedException; + + boolean tryLock(); + + boolean tryLock(long time, TimeUnit unit) throws Exception; + + void unlock() throws Exception; +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/LockManager.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/LockManager.java index f452382f..7d3e24af 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/LockManager.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/LockManager.java @@ -3,4 +3,6 @@ public interface LockManager { Semaphore getSemaphore(String name, int permits) throws Exception; + + Lock getLock(String name); } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Semaphore.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Semaphore.java index 1156fff6..c8b6b18f 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Semaphore.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Semaphore.java @@ -4,14 +4,24 @@ public interface Semaphore { String getName(); + void acquire() throws Exception; + void acquire(int permits) throws Exception; + int availablePermits() throws Exception; + int drainPermits() throws Exception; + void release(); + void release(int permits); + boolean tryAcquire(); + boolean tryAcquire(int permits); + boolean tryAcquire(long timeout, TimeUnit unit) throws InterruptedException; + boolean tryAcquire(int permits, long timeout, TimeUnit unit) throws InterruptedException; } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLock.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLock.java new file mode 100644 index 00000000..571579e6 --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLock.java @@ -0,0 +1,44 @@ +package datawave.microservice.lock.distributed.hazelcast; + +import com.hazelcast.cp.lock.FencedLock; +import datawave.microservice.lock.Lock; + +import java.util.concurrent.TimeUnit; + +public class HazelcastLock implements Lock { + private final FencedLock lock; + + HazelcastLock(FencedLock lock) { + this.lock = lock; + } + + @Override + public String getName() { + return lock.getName(); + } + + @Override + public void lock() { + lock.lock(); + } + + @Override + public void lockInterruptibly() throws InterruptedException { + lock.lockInterruptibly(); + } + + @Override + public boolean tryLock() { + return lock.tryLock(); + } + + @Override + public boolean tryLock(long time, TimeUnit unit) throws InterruptedException { + return lock.tryLock(time, unit); + } + + @Override + public void unlock() { + lock.unlock(); + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLockManager.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLockManager.java index 24b4632d..bbaaf93a 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLockManager.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLockManager.java @@ -3,6 +3,7 @@ import com.hazelcast.core.HazelcastInstance; import com.hazelcast.cp.CPSubsystem; import com.hazelcast.cp.ISemaphore; +import datawave.microservice.lock.Lock; import datawave.microservice.lock.LockManager; import datawave.microservice.lock.Semaphore; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; @@ -30,4 +31,9 @@ public Semaphore getSemaphore(String name, int permits) throws Exception { } return new HazelcastSemaphore(iSemaphore); } + + @Override + public Lock getLock(String name) { + return new HazelcastLock(cpSubsystem.getLock(name)); + } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastSemaphore.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastSemaphore.java index c89d9157..37c70a25 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastSemaphore.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastSemaphore.java @@ -6,7 +6,7 @@ import java.util.concurrent.TimeUnit; public class HazelcastSemaphore implements Semaphore { - private ISemaphore semaphore; + private final ISemaphore semaphore; HazelcastSemaphore(ISemaphore semaphore) { this.semaphore = semaphore; diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLock.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLock.java new file mode 100644 index 00000000..f2c83582 --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLock.java @@ -0,0 +1,46 @@ +package datawave.microservice.lock.distributed.zookeeper; + +import datawave.microservice.lock.Lock; +import org.apache.curator.framework.recipes.locks.InterProcessMutex; + +import java.util.concurrent.TimeUnit; + +public class ZookeeperLock implements Lock { + private final String path; + private final InterProcessMutex lock; + + ZookeeperLock(String path, InterProcessMutex lock) { + this.path = path; + this.lock = lock; + } + + @Override + public String getName() { + return path; + } + + @Override + public void lock() throws Exception { + lock.acquire(); + } + + @Override + public void lockInterruptibly() throws InterruptedException { + throw new UnsupportedOperationException("Unable to lockInterruptibly for ZookeeperLock"); + } + + @Override + public boolean tryLock() { + throw new UnsupportedOperationException("Unable to tryLock for ZookeeperLock"); + } + + @Override + public boolean tryLock(long time, TimeUnit unit) throws Exception { + return lock.acquire(time, unit); + } + + @Override + public void unlock() throws Exception { + lock.release(); + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLockManager.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLockManager.java index 49a52595..d3fb98d4 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLockManager.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLockManager.java @@ -1,8 +1,10 @@ package datawave.microservice.lock.distributed.zookeeper; +import datawave.microservice.lock.Lock; import datawave.microservice.lock.LockManager; import datawave.microservice.lock.Semaphore; import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreV2; import org.apache.curator.framework.recipes.shared.SharedCount; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; @@ -26,4 +28,9 @@ public ZookeeperLockManager(CuratorFramework curatorFramework) { public Semaphore getSemaphore(String name, int permits) throws Exception { return new ZookeeperSemaphore(name, new InterProcessSemaphoreV2(curatorFramework, name, new SharedCount(curatorFramework, name + "/permits", permits)), curatorFramework); } + + @Override + public Lock getLock(String name) { + return new ZookeeperLock(name, new InterProcessMutex(curatorFramework, name)); + } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperSemaphore.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperSemaphore.java index 78806a73..27854f7e 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperSemaphore.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperSemaphore.java @@ -9,11 +9,10 @@ import java.util.concurrent.TimeUnit; public class ZookeeperSemaphore implements Semaphore { - - private String path; - private InterProcessSemaphoreV2 semaphore; - final private LinkedList leases = new LinkedList<>(); - private CuratorFramework curatorFramework; + private final String path; + private final InterProcessSemaphoreV2 semaphore; + private final LinkedList leases = new LinkedList<>(); + private final CuratorFramework curatorFramework; ZookeeperSemaphore(String path, InterProcessSemaphoreV2 semaphore, CuratorFramework curatorFramework) { this.path = path; @@ -23,7 +22,7 @@ public class ZookeeperSemaphore implements Semaphore { @Override public String getName() { - return null; + return path; } @Override diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLock.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLock.java new file mode 100644 index 00000000..4a7b2bd3 --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLock.java @@ -0,0 +1,47 @@ +package datawave.microservice.lock.local; + +import datawave.microservice.lock.Lock; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +public class LocalLock implements Lock { + + private final String name; + private final ReentrantLock lock; + + LocalLock(String name, ReentrantLock lock) { + this.name = name; + this.lock = lock; + } + + @Override + public String getName() { + return name; + } + + @Override + public void lock() { + lock.lock(); + } + + @Override + public void lockInterruptibly() throws InterruptedException { + lock.lockInterruptibly(); + } + + @Override + public boolean tryLock() { + return lock.tryLock(); + } + + @Override + public boolean tryLock(long time, TimeUnit unit) throws InterruptedException { + return lock.tryLock(time, unit); + } + + @Override + public void unlock() { + lock.unlock(); + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLockManager.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLockManager.java index bee76e80..254db1c8 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLockManager.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLockManager.java @@ -1,11 +1,14 @@ package datawave.microservice.lock.local; +import datawave.microservice.lock.Lock; import datawave.microservice.lock.LockManager; import datawave.microservice.lock.Semaphore; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; +import java.util.concurrent.locks.ReentrantLock; + @Component("lockManager") @ConditionalOnMissingBean(name = "lockManager") @ConditionalOnProperty(name = "datawave.lock.type", havingValue = "local") @@ -14,4 +17,9 @@ public class LocalLockManager implements LockManager { public Semaphore getSemaphore(String name, int permits) { return new LocalSemaphore(name, new java.util.concurrent.Semaphore(permits)); } + + @Override + public Lock getLock(String name) { + return new LocalLock(name, new ReentrantLock()); + } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalSemaphore.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalSemaphore.java index ac2ff412..88155f8e 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalSemaphore.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalSemaphore.java @@ -6,8 +6,8 @@ public class LocalSemaphore implements Semaphore { - private String name; - private java.util.concurrent.Semaphore semaphore; + private final String name; + private final java.util.concurrent.Semaphore semaphore; LocalSemaphore(String name, java.util.concurrent.Semaphore semaphore) { this.name = name; From 7f29ca11d70ad39d347e388a867e6b0c4104f6b3 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Wed, 5 May 2021 21:38:29 +0000 Subject: [PATCH 039/218] Created a Hazelcast implementation of the Query Lock Manager. --- .../java/datawave/microservice/lock/Lock.java | 10 ++--- .../microservice/lock/LockManager.java | 4 +- .../datawave/microservice/lock/Semaphore.java | 20 ++++----- .../distributed/hazelcast/HazelcastLock.java | 14 +++---- .../hazelcast/HazelcastLockManager.java | 8 ++-- .../hazelcast/HazelcastSemaphore.java | 24 +++++------ .../distributed/zookeeper/ZookeeperLock.java | 14 +++---- .../zookeeper/ZookeeperLockManager.java | 11 ++--- .../zookeeper/ZookeeperSemaphore.java | 33 +++++++-------- .../microservice/lock/local/LocalLock.java | 16 ++++---- .../lock/local/LocalLockManager.java | 2 +- .../lock/local/LocalSemaphore.java | 26 ++++++------ .../microservice/query/QueryController.java | 24 +++++------ .../query/QueryManagementService.java | 41 ++++++++++--------- 14 files changed, 125 insertions(+), 122 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Lock.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Lock.java index 5ac01777..96e1127c 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Lock.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Lock.java @@ -4,14 +4,14 @@ public interface Lock { String getName(); - + void lock() throws Exception; - + void lockInterruptibly() throws InterruptedException; - + boolean tryLock(); - + boolean tryLock(long time, TimeUnit unit) throws Exception; - + void unlock() throws Exception; } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/LockManager.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/LockManager.java index 7d3e24af..45326534 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/LockManager.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/LockManager.java @@ -1,8 +1,8 @@ package datawave.microservice.lock; public interface LockManager { - + Semaphore getSemaphore(String name, int permits) throws Exception; - + Lock getLock(String name); } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Semaphore.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Semaphore.java index c8b6b18f..8b847051 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Semaphore.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Semaphore.java @@ -4,24 +4,24 @@ public interface Semaphore { String getName(); - + void acquire() throws Exception; - + void acquire(int permits) throws Exception; - + int availablePermits() throws Exception; - + int drainPermits() throws Exception; - + void release(); - + void release(int permits); - + boolean tryAcquire(); - + boolean tryAcquire(int permits); - + boolean tryAcquire(long timeout, TimeUnit unit) throws InterruptedException; - + boolean tryAcquire(int permits, long timeout, TimeUnit unit) throws InterruptedException; } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLock.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLock.java index 571579e6..c9735de3 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLock.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLock.java @@ -7,36 +7,36 @@ public class HazelcastLock implements Lock { private final FencedLock lock; - + HazelcastLock(FencedLock lock) { this.lock = lock; } - + @Override public String getName() { return lock.getName(); } - + @Override public void lock() { lock.lock(); } - + @Override public void lockInterruptibly() throws InterruptedException { lock.lockInterruptibly(); } - + @Override public boolean tryLock() { return lock.tryLock(); } - + @Override public boolean tryLock(long time, TimeUnit unit) throws InterruptedException { return lock.tryLock(time, unit); } - + @Override public void unlock() { lock.unlock(); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLockManager.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLockManager.java index bbaaf93a..8973faaa 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLockManager.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLockManager.java @@ -16,13 +16,13 @@ @ConditionalOnBean(HazelcastInstance.class) @ConditionalOnProperty(name = "datawave.lock.type", havingValue = "hazelcast") public class HazelcastLockManager implements LockManager { - + private CPSubsystem cpSubsystem; - + public HazelcastLockManager(HazelcastInstance hazelcastInstance) { this.cpSubsystem = hazelcastInstance.getCPSubsystem(); } - + @Override public Semaphore getSemaphore(String name, int permits) throws Exception { ISemaphore iSemaphore = cpSubsystem.getSemaphore(name); @@ -31,7 +31,7 @@ public Semaphore getSemaphore(String name, int permits) throws Exception { } return new HazelcastSemaphore(iSemaphore); } - + @Override public Lock getLock(String name) { return new HazelcastLock(cpSubsystem.getLock(name)); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastSemaphore.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastSemaphore.java index 37c70a25..4a1a30df 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastSemaphore.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastSemaphore.java @@ -7,61 +7,61 @@ public class HazelcastSemaphore implements Semaphore { private final ISemaphore semaphore; - + HazelcastSemaphore(ISemaphore semaphore) { this.semaphore = semaphore; } - + @Override public String getName() { return semaphore.getName(); } - + @Override public void acquire() throws InterruptedException { semaphore.acquire(); } - + @Override public void acquire(int permits) throws InterruptedException { semaphore.acquire(permits); } - + @Override public int availablePermits() { return semaphore.availablePermits(); } - + @Override public int drainPermits() { return semaphore.drainPermits(); } - + @Override public void release() { semaphore.release(); } - + @Override public void release(int permits) { semaphore.release(permits); } - + @Override public boolean tryAcquire() { return semaphore.tryAcquire(); } - + @Override public boolean tryAcquire(int permits) { return semaphore.tryAcquire(permits); } - + @Override public boolean tryAcquire(long timeout, TimeUnit unit) throws InterruptedException { return semaphore.tryAcquire(timeout, unit); } - + @Override public boolean tryAcquire(int permits, long timeout, TimeUnit unit) throws InterruptedException { return semaphore.tryAcquire(permits, timeout, unit); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLock.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLock.java index f2c83582..d1a37ff9 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLock.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLock.java @@ -8,37 +8,37 @@ public class ZookeeperLock implements Lock { private final String path; private final InterProcessMutex lock; - + ZookeeperLock(String path, InterProcessMutex lock) { this.path = path; this.lock = lock; } - + @Override public String getName() { return path; } - + @Override public void lock() throws Exception { lock.acquire(); } - + @Override public void lockInterruptibly() throws InterruptedException { throw new UnsupportedOperationException("Unable to lockInterruptibly for ZookeeperLock"); } - + @Override public boolean tryLock() { throw new UnsupportedOperationException("Unable to tryLock for ZookeeperLock"); } - + @Override public boolean tryLock(long time, TimeUnit unit) throws Exception { return lock.acquire(time, unit); } - + @Override public void unlock() throws Exception { lock.release(); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLockManager.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLockManager.java index d3fb98d4..11bbb267 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLockManager.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLockManager.java @@ -17,18 +17,19 @@ @ConditionalOnBean(CuratorFramework.class) @ConditionalOnProperty(name = "datawave.lock.type", havingValue = "zookeeper") public class ZookeeperLockManager implements LockManager { - + final private CuratorFramework curatorFramework; - + public ZookeeperLockManager(CuratorFramework curatorFramework) { this.curatorFramework = curatorFramework; } - + @Override public Semaphore getSemaphore(String name, int permits) throws Exception { - return new ZookeeperSemaphore(name, new InterProcessSemaphoreV2(curatorFramework, name, new SharedCount(curatorFramework, name + "/permits", permits)), curatorFramework); + return new ZookeeperSemaphore(name, new InterProcessSemaphoreV2(curatorFramework, name, new SharedCount(curatorFramework, name + "/permits", permits)), + curatorFramework); } - + @Override public Lock getLock(String name) { return new ZookeeperLock(name, new InterProcessMutex(curatorFramework, name)); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperSemaphore.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperSemaphore.java index 27854f7e..36708df3 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperSemaphore.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperSemaphore.java @@ -13,75 +13,76 @@ public class ZookeeperSemaphore implements Semaphore { private final InterProcessSemaphoreV2 semaphore; private final LinkedList leases = new LinkedList<>(); private final CuratorFramework curatorFramework; - + ZookeeperSemaphore(String path, InterProcessSemaphoreV2 semaphore, CuratorFramework curatorFramework) { this.path = path; this.semaphore = semaphore; - this.curatorFramework = curatorFramework;; + this.curatorFramework = curatorFramework; + ; } - + @Override public String getName() { return path; } - + @Override public void acquire() throws Exception { - synchronized(leases) { + synchronized (leases) { leases.push(semaphore.acquire()); } } - + @Override public void acquire(int permits) throws Exception { synchronized (leases) { semaphore.acquire(permits).forEach(leases::push); } } - + @Override public int availablePermits() throws Exception { return curatorFramework.getChildren().forPath(path).size(); } - + @Override public int drainPermits() throws Exception { throw new UnsupportedOperationException("Unable to drainPermits for ZookeeperSemaphore"); } - + @Override public void release() { - synchronized(leases) { + synchronized (leases) { if (leases.size() > 0) { semaphore.returnLease(leases.pop()); } } } - + @Override public void release(int permits) { - synchronized(leases) { + synchronized (leases) { while (permits-- > 0 && !leases.isEmpty()) { semaphore.returnLease(leases.pop()); } } } - + @Override public boolean tryAcquire() { throw new UnsupportedOperationException("Unable to tryAcquire for ZookeeperSemaphore"); } - + @Override public boolean tryAcquire(int permits) { throw new UnsupportedOperationException("Unable to tryAcquire for ZookeeperSemaphore"); } - + @Override public boolean tryAcquire(long timeout, TimeUnit unit) throws InterruptedException { throw new UnsupportedOperationException("Unable to tryAcquire for ZookeeperSemaphore"); } - + @Override public boolean tryAcquire(int permits, long timeout, TimeUnit unit) throws InterruptedException { throw new UnsupportedOperationException("Unable to tryAcquire for ZookeeperSemaphore"); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLock.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLock.java index 4a7b2bd3..a7838409 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLock.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLock.java @@ -6,40 +6,40 @@ import java.util.concurrent.locks.ReentrantLock; public class LocalLock implements Lock { - + private final String name; private final ReentrantLock lock; - + LocalLock(String name, ReentrantLock lock) { this.name = name; this.lock = lock; } - + @Override public String getName() { return name; } - + @Override public void lock() { lock.lock(); } - + @Override public void lockInterruptibly() throws InterruptedException { lock.lockInterruptibly(); } - + @Override public boolean tryLock() { return lock.tryLock(); } - + @Override public boolean tryLock(long time, TimeUnit unit) throws InterruptedException { return lock.tryLock(time, unit); } - + @Override public void unlock() { lock.unlock(); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLockManager.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLockManager.java index 254db1c8..cc54f257 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLockManager.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLockManager.java @@ -17,7 +17,7 @@ public class LocalLockManager implements LockManager { public Semaphore getSemaphore(String name, int permits) { return new LocalSemaphore(name, new java.util.concurrent.Semaphore(permits)); } - + @Override public Lock getLock(String name) { return new LocalLock(name, new ReentrantLock()); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalSemaphore.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalSemaphore.java index 88155f8e..e0abe842 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalSemaphore.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalSemaphore.java @@ -5,65 +5,65 @@ import java.util.concurrent.TimeUnit; public class LocalSemaphore implements Semaphore { - + private final String name; private final java.util.concurrent.Semaphore semaphore; - + LocalSemaphore(String name, java.util.concurrent.Semaphore semaphore) { this.name = name; this.semaphore = semaphore; } - + @Override public String getName() { return name; } - + @Override public void acquire() throws InterruptedException { semaphore.acquire(); } - + @Override public void acquire(int permits) throws InterruptedException { semaphore.acquire(permits); } - + @Override public int availablePermits() { return semaphore.availablePermits(); } - + @Override public int drainPermits() { return semaphore.drainPermits(); } - + @Override public void release() { semaphore.release(); } - + @Override public void release(int permits) { semaphore.release(); } - + @Override public boolean tryAcquire() { return semaphore.tryAcquire(); } - + @Override public boolean tryAcquire(int permits) { return semaphore.tryAcquire(permits); } - + @Override public boolean tryAcquire(long timeout, TimeUnit unit) throws InterruptedException { return semaphore.tryAcquire(timeout, unit); } - + @Override public boolean tryAcquire(int permits, long timeout, TimeUnit unit) throws InterruptedException { return semaphore.tryAcquire(permits, timeout, unit); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 326e66da..467f3953 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -24,7 +24,7 @@ public class QueryController { private final Logger log = LoggerFactory.getLogger(this.getClass()); private final QueryManagementService queryManagementService; - + public QueryController(QueryManagementService queryManagementService) { this.queryManagementService = queryManagementService; } @@ -41,40 +41,40 @@ public GenericResponse define(@PathVariable(name = "queryLogic") String resp.setResult(taskKey.getQueryId().toString()); return resp; } - + @Timed(name = "dw.query.createQuery", absolute = true) @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE) @RequestMapping(path = "{queryLogic}/define", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public GenericResponse create(@PathVariable(name = "queryLogic") String queryLogic, @RequestParam MultiValueMap parameters, - @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { TaskKey taskKey = queryManagementService.create(queryLogic, parameters, currentUser); - + GenericResponse resp = new GenericResponse<>(); resp.setHasResults(true); resp.setResult(taskKey.getQueryId().toString()); return resp; } - + @Timed(name = "dw.query.next", absolute = true) @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.NEXT) @RequestMapping(path = "{queryId}/next", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public BaseQueryResponse next(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { // need to find the query in the query storage cache, and then check for results on the results queue - + // figure out what user this is - + // check to see if we're already handling a next call for this user/query - + // check to see if the query id exists in the query cache - + // make sure that this is the caller's query - + // get the next set of results - + // return the results to the user - + return null; } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index e8bf17fe..a86a8be1 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -56,7 +56,8 @@ public class QueryManagementService { private final int PAGE_TIMEOUT_MAX = QueryExpirationProperties.PAGE_TIMEOUT_MIN_DEFAULT; public QueryManagementService(QueryProperties queryProperties, QueryParameters queryParameters, SecurityMarking securityMarking, - QueryLogicFactory queryLogicFactory, ResponseObjectFactory responseObjectFactory, QueryStorageCache queryStorageCache, AuditClient auditClient, LockManager lockManager) { + QueryLogicFactory queryLogicFactory, ResponseObjectFactory responseObjectFactory, QueryStorageCache queryStorageCache, + AuditClient auditClient, LockManager lockManager) { this.queryProperties = queryProperties; this.queryParameters = queryParameters; this.securityMarking = securityMarking; @@ -66,12 +67,12 @@ public QueryManagementService(QueryProperties queryProperties, QueryParameters q this.auditClient = auditClient; this.lockManager = lockManager; } - + /** * Defines a datawave query. * - * Validates the query parameters using the base validation, query logic-specific validation, and markings validation. - * If the parameters are valid, the query will be stored in the query storage cache where it can be acted upon in a subsequent call. + * Validates the query parameters using the base validation, query logic-specific validation, and markings validation. If the parameters are valid, the + * query will be stored in the query storage cache where it can be acted upon in a subsequent call. * * @param queryLogicName * @param parameters @@ -110,12 +111,12 @@ public TaskKey define(String queryLogicName, MultiValueMap parame throw new BadRequestQueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR, e); } } - + /** * Creates a datawave query. * - * Validates the query parameters using the base validation, query logic-specific validation, and markings validation. - * If the parameters are valid, the query will be stored in the query storage cache where it can be acted upon in a subsequent call. + * Validates the query parameters using the base validation, query logic-specific validation, and markings validation. If the parameters are valid, the + * query will be stored in the query storage cache where it can be acted upon in a subsequent call. * * @param queryLogicName * @param parameters @@ -126,18 +127,18 @@ public TaskKey define(String queryLogicName, MultiValueMap parame public TaskKey create(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { // validate query and get a query logic QueryLogic queryLogic = validateQuery(queryLogicName, parameters, currentUser); - + String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); log.trace(userId + " has authorizations " + currentUser.getPrimaryUser().getAuths()); - + // set some audit parameters which are used internally String userDn = currentUser.getPrimaryUser().getDn().subjectDN(); setInternalAuditParameters(queryLogicName, userDn, parameters); - + // send an audit record to the auditor Query query = createQuery(queryLogicName, parameters, userDn, currentUser.getDNs()); audit(query, queryLogic, parameters, currentUser); - + try { // persist the query w/ query id in the query storage cache // @formatter:off @@ -146,21 +147,21 @@ public TaskKey create(String queryLogicName, MultiValueMap parame query, getMaxConcurrentTasks(queryLogic)); // @formatter:on - + // TODO: JWO: Figure out how to make query tracing work with our new architecture. Datawave issue #1155 - + // TODO: JWO: Figure out how to make query metrics work with our new architecture. Datawave issue #1156 - + return taskKey; } catch (Exception e) { log.error("Unknown error storing query", e); throw new BadRequestQueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR, e); } } - + protected void audit(Query query, QueryLogic queryLogic, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { Auditor.AuditType auditType = queryLogic.getAuditType(query); - + parameters.add(PrivateAuditConstants.AUDIT_TYPE, auditType.name()); if (auditType != Auditor.AuditType.NONE) { // audit the query before execution @@ -173,12 +174,12 @@ protected void audit(Query query, QueryLogic queryLogic, MultiValueMap queryLogic, MultiValueMap Date: Thu, 6 May 2021 10:25:47 -0400 Subject: [PATCH 040/218] minor fixes --- .../hazelcast/HazelcastLockManager.java | 4 +- .../zookeeper/ZookeeperLockManager.java | 4 +- .../lock/local/LocalLockManager.java | 4 +- .../microservice/query/QueryController.java | 15 +------ .../query/QueryManagementService.java | 41 +++++++++++++++---- 5 files changed, 38 insertions(+), 30 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLockManager.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLockManager.java index 8973faaa..da3fe0f1 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLockManager.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLockManager.java @@ -7,12 +7,10 @@ import datawave.microservice.lock.LockManager; import datawave.microservice.lock.Semaphore; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; -@Component("lockManager") -@ConditionalOnMissingBean(name = "lockManager") +@Component("distributedLockManager") @ConditionalOnBean(HazelcastInstance.class) @ConditionalOnProperty(name = "datawave.lock.type", havingValue = "hazelcast") public class HazelcastLockManager implements LockManager { diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLockManager.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLockManager.java index 11bbb267..a5bda2ae 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLockManager.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLockManager.java @@ -8,12 +8,10 @@ import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreV2; import org.apache.curator.framework.recipes.shared.SharedCount; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; -@Component("lockManager") -@ConditionalOnMissingBean(name = "lockManager") +@Component("distributedLockManager") @ConditionalOnBean(CuratorFramework.class) @ConditionalOnProperty(name = "datawave.lock.type", havingValue = "zookeeper") public class ZookeeperLockManager implements LockManager { diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLockManager.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLockManager.java index cc54f257..67738a5d 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLockManager.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLockManager.java @@ -3,14 +3,12 @@ import datawave.microservice.lock.Lock; import datawave.microservice.lock.LockManager; import datawave.microservice.lock.Semaphore; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import java.util.concurrent.locks.ReentrantLock; -@Component("lockManager") -@ConditionalOnMissingBean(name = "lockManager") +@Component("distributedLockManager") @ConditionalOnProperty(name = "datawave.lock.type", havingValue = "local") public class LocalLockManager implements LockManager { @Override diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 467f3953..33326783 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -44,7 +44,7 @@ public GenericResponse define(@PathVariable(name = "queryLogic") String @Timed(name = "dw.query.createQuery", absolute = true) @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE) - @RequestMapping(path = "{queryLogic}/define", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + @RequestMapping(path = "{queryLogic}/create", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public GenericResponse create(@PathVariable(name = "queryLogic") String queryLogic, @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { @@ -61,19 +61,8 @@ public GenericResponse create(@PathVariable(name = "queryLogic") String @RequestMapping(path = "{queryId}/next", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public BaseQueryResponse next(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { - // need to find the query in the query storage cache, and then check for results on the results queue - - // figure out what user this is - - // check to see if we're already handling a next call for this user/query - - // check to see if the query id exists in the query cache - - // make sure that this is the caller's query - - // get the next set of results - // return the results to the user + queryManagementService.next(); return null; } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index a86a8be1..0b6f1720 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -8,6 +8,7 @@ import datawave.microservice.common.audit.PrivateAuditConstants; import datawave.microservice.common.storage.QueryPool; import datawave.microservice.common.storage.QueryStorageCache; +import datawave.microservice.common.storage.QueryTask; import datawave.microservice.common.storage.TaskKey; import datawave.microservice.lock.LockManager; import datawave.microservice.query.config.QueryExpirationProperties; @@ -30,7 +31,9 @@ import org.springframework.stereotype.Service; import org.springframework.util.MultiValueMap; +import java.io.IOException; import java.text.MessageFormat; +import java.text.ParseException; import java.util.List; @Service @@ -39,17 +42,17 @@ public class QueryManagementService { private static final ObjectMapper mapper = new ObjectMapper(); - private QueryProperties queryProperties; + private final QueryProperties queryProperties; // Note: QueryParameters need to be request scoped - private QueryParameters queryParameters; + private final QueryParameters queryParameters; // Note: SecurityMarking needs to be request scoped - private SecurityMarking securityMarking; - private QueryLogicFactory queryLogicFactory; - private ResponseObjectFactory responseObjectFactory; - private QueryStorageCache queryStorageCache; - private AuditClient auditClient; - private LockManager lockManager; + private final SecurityMarking securityMarking; + private final QueryLogicFactory queryLogicFactory; + private final ResponseObjectFactory responseObjectFactory; + private final QueryStorageCache queryStorageCache; + private final AuditClient auditClient; + private final LockManager lockManager; // TODO: JWO: Pull these from configuration instead private final int PAGE_TIMEOUT_MIN = 1; @@ -159,6 +162,28 @@ public TaskKey create(String queryLogicName, MultiValueMap parame } } + public void next() throws IOException, InterruptedException, ParseException { + // does the query exist? + QueryTask queryTask = queryStorageCache.getTask(null, 0); + Query query = queryTask.getQueryCheckpoint().getPropertiesAsQuery(); + + // if it exists, can we gaet a lock on this next call? + + // need to find the query in the query storage cache, and then check for results on the results queue + + // figure out what user this is + + // check to see if we're already handling a next call for this user/query + + // check to see if the query id exists in the query cache + + // make sure that this is the caller's query + + // get the next set of results + + // return the results to the user + } + protected void audit(Query query, QueryLogic queryLogic, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { Auditor.AuditType auditType = queryLogic.getAuditType(query); From 38437e956edff7a76bb7c44490ee211ebe5bfdee Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 6 May 2021 11:38:41 -0400 Subject: [PATCH 041/218] Trying to flesh out the next call... --- .../microservice/query/QueryController.java | 2 +- .../query/QueryManagementService.java | 81 +++++++++++++++---- .../QueryMetricsEnrichmentFilterAdvice.java | 2 - .../microservice/query/QueryServiceTest.java | 32 +++++++- 4 files changed, 96 insertions(+), 21 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 33326783..db8534c4 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -62,7 +62,7 @@ public GenericResponse create(@PathVariable(name = "queryLogic") String "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public BaseQueryResponse next(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { - queryManagementService.next(); + queryManagementService.next(queryId, currentUser); return null; } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 0b6f1720..05ef889d 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -7,9 +7,12 @@ import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.common.audit.PrivateAuditConstants; import datawave.microservice.common.storage.QueryPool; +import datawave.microservice.common.storage.QueryQueueListener; +import datawave.microservice.common.storage.QueryQueueManager; import datawave.microservice.common.storage.QueryStorageCache; import datawave.microservice.common.storage.QueryTask; import datawave.microservice.common.storage.TaskKey; +import datawave.microservice.lock.Lock; import datawave.microservice.lock.LockManager; import datawave.microservice.query.config.QueryExpirationProperties; import datawave.microservice.query.config.QueryProperties; @@ -22,6 +25,8 @@ import datawave.webservice.query.Query; import datawave.webservice.query.exception.BadRequestQueryException; import datawave.webservice.query.exception.DatawaveErrorCode; +import datawave.webservice.query.exception.NotFoundQueryException; +import datawave.webservice.query.exception.PreConditionFailedQueryException; import datawave.webservice.query.exception.QueryException; import datawave.webservice.query.exception.UnauthorizedQueryException; import datawave.webservice.query.result.event.ResponseObjectFactory; @@ -32,6 +37,8 @@ import org.springframework.util.MultiValueMap; import java.io.IOException; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.text.MessageFormat; import java.text.ParseException; import java.util.List; @@ -51,24 +58,36 @@ public class QueryManagementService { private final QueryLogicFactory queryLogicFactory; private final ResponseObjectFactory responseObjectFactory; private final QueryStorageCache queryStorageCache; + private final QueryQueueManager queryQueueManager; private final AuditClient auditClient; private final LockManager lockManager; + private final String identifier; + // TODO: JWO: Pull these from configuration instead private final int PAGE_TIMEOUT_MIN = 1; private final int PAGE_TIMEOUT_MAX = QueryExpirationProperties.PAGE_TIMEOUT_MIN_DEFAULT; public QueryManagementService(QueryProperties queryProperties, QueryParameters queryParameters, SecurityMarking securityMarking, QueryLogicFactory queryLogicFactory, ResponseObjectFactory responseObjectFactory, QueryStorageCache queryStorageCache, - AuditClient auditClient, LockManager lockManager) { + QueryQueueManager queryQueueManager, AuditClient auditClient, LockManager lockManager) { this.queryProperties = queryProperties; this.queryParameters = queryParameters; this.securityMarking = securityMarking; this.queryLogicFactory = queryLogicFactory; this.responseObjectFactory = responseObjectFactory; this.queryStorageCache = queryStorageCache; + this.queryQueueManager = queryQueueManager; this.auditClient = auditClient; this.lockManager = lockManager; + + String hostname; + try { + hostname = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + hostname = "UNKNOWN"; + } + this.identifier = hostname; } /** @@ -162,26 +181,54 @@ public TaskKey create(String queryLogicName, MultiValueMap parame } } - public void next() throws IOException, InterruptedException, ParseException { + public void next(String queryId, ProxiedUserDetails currentUser) throws IOException, InterruptedException, ParseException, QueryException { // does the query exist? QueryTask queryTask = queryStorageCache.getTask(null, 0); - Query query = queryTask.getQueryCheckpoint().getPropertiesAsQuery(); - - // if it exists, can we gaet a lock on this next call? - - // need to find the query in the query storage cache, and then check for results on the results queue - - // figure out what user this is - - // check to see if we're already handling a next call for this user/query - - // check to see if the query id exists in the query cache - - // make sure that this is the caller's query + if (queryTask == null) { + throw new NotFoundQueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH, MessageFormat.format("{0}", queryId)); + } - // get the next set of results + // does the current user own this query? + String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + Query query = queryTask.getQueryCheckpoint().getPropertiesAsQuery(); + if (!query.getOwner().equals(userId)) { + throw new UnauthorizedQueryException(DatawaveErrorCode.QUERY_OWNER_MISMATCH, MessageFormat.format("{0} != {1}", userId, query.getOwner())); + } - // return the results to the user + Lock lock = lockManager.getLock(queryId); + try { + // try to get a lock on this call to prevent concurrent next calls + // we disallow concurrent next calls to prevent a single user from starving out other users + try { + lock.lock(); + } catch (Exception e) { + lock = null; + throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to acquire lock on query."); + } + + // if we made it this far, we're ready to start checking for results + // TODO: Figure out how to get results from the queue + QueryQueueListener listener = queryQueueManager.createListener(identifier, queryId); + + // TODO: I have no idea what I'm supposed to be receiving here... + Object something; + try { + something = listener.receive(); + } catch (Exception e) { + throw new PreConditionFailedQueryException(DatawaveErrorCode.QUERY_TIMEOUT_OR_SERVER_ERROR, e, MessageFormat.format("id = {0}", queryId)); + } + + // TODO: Where is the page number supposed to come from? It SHOULD come from the cache, right? + + } finally { + if (lock != null) { + try { + lock.unlock(); + } catch (Exception e) { + throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to release lock on query."); + } + } + } } protected void audit(Query query, QueryLogic queryLogic, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java index 6b78dabd..a83b7c22 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java @@ -1,6 +1,5 @@ package datawave.microservice.query.web.filter; -import datawave.microservice.common.storage.QueryCache; import datawave.microservice.common.storage.QueryState; import datawave.microservice.common.storage.QueryStorageCache; import datawave.microservice.query.logic.QueryLogicFactory; @@ -23,7 +22,6 @@ import java.util.Arrays; import java.util.List; import java.util.Objects; -import java.util.UUID; @ControllerAdvice public class QueryMetricsEnrichmentFilterAdvice extends BaseMethodStatsFilter implements ResponseBodyAdvice { diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java index 83ecc5a8..7ee00548 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java @@ -55,7 +55,7 @@ public void setup() { } @Test - public void testQuery() { + public void testDefineQuery() { Collection roles = Collections.singleton("AuthorizedUser"); DatawaveUser uathDWUser = new DatawaveUser(DN, USER, null, roles, null, System.currentTimeMillis()); ProxiedUserDetails authUser = new ProxiedUserDetails(Collections.singleton(uathDWUser), uathDWUser.getCreationTime()); @@ -84,6 +84,36 @@ public void testQuery() { } } + @Test + public void testCreateQuery() { + Collection roles = Collections.singleton("AuthorizedUser"); + DatawaveUser uathDWUser = new DatawaveUser(DN, USER, null, roles, null, System.currentTimeMillis()); + ProxiedUserDetails authUser = new ProxiedUserDetails(Collections.singleton(uathDWUser), uathDWUser.getCreationTime()); + + UriComponents uri = UriComponentsBuilder.newInstance().scheme("https").host("localhost").port(webServicePort).path("/query/v1/EventQuery/create") + .build(); + + MultiValueMap map = new LinkedMultiValueMap<>(); + map.set(DefaultQueryParameters.QUERY_STRING, "FIELD:SOME_VALUE"); + map.set(DefaultQueryParameters.QUERY_NAME, "The Greatest Query in the World - Tribute"); + map.set(DefaultQueryParameters.QUERY_PERSISTENCE, "PERSISTENT"); + map.set(DefaultQueryParameters.QUERY_AUTHORIZATIONS, "ALL"); + map.set(DefaultQueryParameters.QUERY_EXPIRATION, "20500101 000000.000"); + map.set(DefaultQueryParameters.QUERY_BEGIN, "20000101 000000.000"); + map.set(DefaultQueryParameters.QUERY_END, "20500101 000000.000"); + map.set(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, "ALL"); + + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + try { + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, String.class); + + System.out.println("done!"); + } finally { + assertTrue("", true); + } + } + @Configuration @Profile("QueryServiceTest") @ComponentScan(basePackages = "datawave.microservice") From 4f4ac65e08b6470c9dfc7d73cf46e6b250e209d0 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 12 May 2021 08:56:35 -0400 Subject: [PATCH 042/218] Added query service event handler and continued to work on next, close, cancel locking logic. --- .../query/config/QueryProperties.java | 112 +++++- .../microservice/query/QueryController.java | 23 +- .../query/QueryManagementService.java | 372 ++++++++++++++---- .../query/config/QueryServiceConfig.java | 15 + .../src/test/resources/config/bootstrap.yml | 2 +- 5 files changed, 448 insertions(+), 76 deletions(-) diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java index 94438f93..b8a0930b 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java @@ -5,16 +5,29 @@ import javax.annotation.Nonnegative; import javax.validation.Valid; import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Positive; +import javax.validation.constraints.PositiveOrZero; +import java.util.concurrent.TimeUnit; @Validated public class QueryProperties { @Valid private QueryExpirationProperties expiration; - @NotEmpty private String privilegedRole = "PrivilegedUser"; - + @Positive + private long resultQueueIntervalMillis = TimeUnit.SECONDS.toMillis(60); + // The amount of time to wait for the lock to be acquired + @PositiveOrZero + private long lockWaitTimeMillis = TimeUnit.SECONDS.toMillis(5); + // The amount of time that the lock will be held before being automatically released + @PositiveOrZero + private long lockLeaseTimeMillis = TimeUnit.SECONDS.toMillis(30); + private String executorServiceName = "executor"; + private int concurrentNextLimit = 1; + private ExecutorProperties nextExecutor = new ExecutorProperties(); private DefaultParameters defaultParams = new DefaultParameters(); public QueryExpirationProperties getExpiration() { @@ -33,6 +46,54 @@ public void setPrivilegedRole(String privilegedRole) { this.privilegedRole = privilegedRole; } + public long getResultQueueIntervalMillis() { + return resultQueueIntervalMillis; + } + + public void setResultQueueIntervalMillis(long resultQueueIntervalMillis) { + this.resultQueueIntervalMillis = resultQueueIntervalMillis; + } + + public long getLockWaitTimeMillis() { + return lockWaitTimeMillis; + } + + public void setLockWaitTimeMillis(long lockWaitTimeMillis) { + this.lockWaitTimeMillis = lockWaitTimeMillis; + } + + public long getLockLeaseTimeMillis() { + return lockLeaseTimeMillis; + } + + public void setLockLeaseTimeMillis(long lockLeaseTimeMillis) { + this.lockLeaseTimeMillis = lockLeaseTimeMillis; + } + + public String getExecutorServiceName() { + return executorServiceName; + } + + public void setExecutorServiceName(String executorServiceName) { + this.executorServiceName = executorServiceName; + } + + public int getConcurrentNextLimit() { + return concurrentNextLimit; + } + + public void setConcurrentNextLimit(int concurrentNextLimit) { + this.concurrentNextLimit = concurrentNextLimit; + } + + public ExecutorProperties getNextExecutor() { + return nextExecutor; + } + + public void setNextExecutor(ExecutorProperties nextExecutor) { + this.nextExecutor = nextExecutor; + } + public DefaultParameters getDefaultParams() { return defaultParams; } @@ -66,4 +127,51 @@ public void setMaxConcurrentTasks(int maxConcurrentTasks) { this.maxConcurrentTasks = maxConcurrentTasks; } } + + @Validated + public static class ExecutorProperties { + @PositiveOrZero + private int corePoolSize = 0; + + @Positive + private int maxPoolSize = 5; + + @PositiveOrZero + private int queueCapacity = 0; + + @NotNull + private String threadNamePrefix = "replayTask-"; + + public int getCorePoolSize() { + return corePoolSize; + } + + public void setCorePoolSize(int corePoolSize) { + this.corePoolSize = corePoolSize; + } + + public int getMaxPoolSize() { + return maxPoolSize; + } + + public void setMaxPoolSize(int maxPoolSize) { + this.maxPoolSize = maxPoolSize; + } + + public int getQueueCapacity() { + return queueCapacity; + } + + public void setQueueCapacity(int queueCapacity) { + this.queueCapacity = queueCapacity; + } + + public String getThreadNamePrefix() { + return threadNamePrefix; + } + + public void setThreadNamePrefix(String threadNamePrefix) { + this.threadNamePrefix = threadNamePrefix; + } + } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index db8534c4..5fb1d4f8 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -1,12 +1,12 @@ package datawave.microservice.query; import com.codahale.metrics.annotation.Timed; -import com.hazelcast.core.HazelcastInstance; import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.common.storage.TaskKey; import datawave.microservice.query.web.annotation.EnrichQueryMetrics; import datawave.webservice.result.BaseQueryResponse; import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.VoidResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; @@ -18,6 +18,8 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import java.util.List; + @RestController @RequestMapping(path = "/v1", produces = MediaType.APPLICATION_JSON_VALUE) public class QueryController { @@ -62,8 +64,25 @@ public GenericResponse create(@PathVariable(name = "queryLogic") String "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public BaseQueryResponse next(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { - queryManagementService.next(queryId, currentUser); + List resultObjects = queryManagementService.next(queryId, currentUser); return null; } + + @Timed(name = "dw.query.cancel", absolute = true) + @RequestMapping(path = "{queryId}/cancel", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse cancel(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + queryManagementService.cancel(queryId, currentUser); + return null; + } + + @Timed(name = "dw.query.close", absolute = true) + @RequestMapping(path = "{queryId}/close", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse close(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + queryManagementService.close(queryId, currentUser); + return null; + } + } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 05ef889d..437fcd42 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -9,15 +9,15 @@ import datawave.microservice.common.storage.QueryPool; import datawave.microservice.common.storage.QueryQueueListener; import datawave.microservice.common.storage.QueryQueueManager; +import datawave.microservice.common.storage.QueryStatus; import datawave.microservice.common.storage.QueryStorageCache; -import datawave.microservice.common.storage.QueryTask; import datawave.microservice.common.storage.TaskKey; -import datawave.microservice.lock.Lock; -import datawave.microservice.lock.LockManager; import datawave.microservice.query.config.QueryExpirationProperties; import datawave.microservice.query.config.QueryProperties; import datawave.microservice.query.logic.QueryLogic; import datawave.microservice.query.logic.QueryLogicFactory; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.remote.QueryRequestHandler; import datawave.microservice.query.util.QueryUtil; import datawave.security.util.ProxiedEntityUtils; import datawave.webservice.common.audit.AuditParameters; @@ -26,31 +26,44 @@ import datawave.webservice.query.exception.BadRequestQueryException; import datawave.webservice.query.exception.DatawaveErrorCode; import datawave.webservice.query.exception.NotFoundQueryException; -import datawave.webservice.query.exception.PreConditionFailedQueryException; import datawave.webservice.query.exception.QueryException; import datawave.webservice.query.exception.UnauthorizedQueryException; import datawave.webservice.query.result.event.ResponseObjectFactory; import datawave.webservice.query.util.QueryUncaughtExceptionHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.cloud.bus.BusProperties; +import org.springframework.cloud.bus.event.QueryRemoteRequestEvent; +import org.springframework.context.ApplicationContext; +import org.springframework.core.task.TaskRejectedException; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.stereotype.Service; +import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; -import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; import java.text.MessageFormat; -import java.text.ParseException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.UUID; +import java.util.concurrent.Future; + +import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CANCELED; +import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CLOSED; @Service -public class QueryManagementService { +public class QueryManagementService implements QueryRequestHandler { private final Logger log = LoggerFactory.getLogger(this.getClass()); private static final ObjectMapper mapper = new ObjectMapper(); private final QueryProperties queryProperties; + private final ApplicationContext appCtx; + private final BusProperties busProperties; + // Note: QueryParameters need to be request scoped private final QueryParameters queryParameters; // Note: SecurityMarking needs to be request scoped @@ -60,18 +73,22 @@ public class QueryManagementService { private final QueryStorageCache queryStorageCache; private final QueryQueueManager queryQueueManager; private final AuditClient auditClient; - private final LockManager lockManager; + private final ThreadPoolTaskExecutor nextExecutor; private final String identifier; + private MultiValueMap>> nextTaskMap = new LinkedMultiValueMap<>(); + // TODO: JWO: Pull these from configuration instead private final int PAGE_TIMEOUT_MIN = 1; private final int PAGE_TIMEOUT_MAX = QueryExpirationProperties.PAGE_TIMEOUT_MIN_DEFAULT; - public QueryManagementService(QueryProperties queryProperties, QueryParameters queryParameters, SecurityMarking securityMarking, - QueryLogicFactory queryLogicFactory, ResponseObjectFactory responseObjectFactory, QueryStorageCache queryStorageCache, - QueryQueueManager queryQueueManager, AuditClient auditClient, LockManager lockManager) { + public QueryManagementService(QueryProperties queryProperties, ApplicationContext appCtx, BusProperties busProperties, QueryParameters queryParameters, + SecurityMarking securityMarking, QueryLogicFactory queryLogicFactory, ResponseObjectFactory responseObjectFactory, + QueryStorageCache queryStorageCache, QueryQueueManager queryQueueManager, AuditClient auditClient, ThreadPoolTaskExecutor nextExecutor) { this.queryProperties = queryProperties; + this.appCtx = appCtx; + this.busProperties = busProperties; this.queryParameters = queryParameters; this.securityMarking = securityMarking; this.queryLogicFactory = queryLogicFactory; @@ -79,7 +96,7 @@ public QueryManagementService(QueryProperties queryProperties, QueryParameters q this.queryStorageCache = queryStorageCache; this.queryQueueManager = queryQueueManager; this.auditClient = auditClient; - this.lockManager = lockManager; + this.nextExecutor = nextExecutor; String hostname; try { @@ -147,87 +164,300 @@ public TaskKey define(String queryLogicName, MultiValueMap parame * @throws QueryException */ public TaskKey create(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { - // validate query and get a query logic - QueryLogic queryLogic = validateQuery(queryLogicName, parameters, currentUser); - - String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); - log.trace(userId + " has authorizations " + currentUser.getPrimaryUser().getAuths()); + try { + // validate query and get a query logic + QueryLogic queryLogic = validateQuery(queryLogicName, parameters, currentUser); + + String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + log.trace(userId + " has authorizations " + currentUser.getPrimaryUser().getAuths()); + + // set some audit parameters which are used internally + String userDn = currentUser.getPrimaryUser().getDn().subjectDN(); + setInternalAuditParameters(queryLogicName, userDn, parameters); + + // send an audit record to the auditor + Query query = createQuery(queryLogicName, parameters, userDn, currentUser.getDNs()); + audit(query, queryLogic, parameters, currentUser); + + try { + // persist the query w/ query id in the query storage cache + // @formatter:off + TaskKey taskKey = queryStorageCache.storeQuery( + new QueryPool(getPoolName()), + query, + getMaxConcurrentTasks(queryLogic)); + // @formatter:on + + // TODO: JWO: Figure out how to make query tracing work with our new architecture. Datawave issue #1155 + + // TODO: JWO: Figure out how to make query metrics work with our new architecture. Datawave issue #1156 + + return taskKey; + } catch (Exception e) { + log.error("Unknown error storing query", e); + throw new BadRequestQueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR, e); + } + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error creating query", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error creating query."); + } + } + + public List next(String queryId, ProxiedUserDetails currentUser) throws QueryException { + UUID queryUUID = UUID.fromString(queryId); + try { + // make sure the query is valid, and the user can act on it + validateRequest(queryId, currentUser); + + boolean decrementNext = false; + try { + // before we spin up a separate thread, make sure we are allowed to call next + incrementConcurrentNextCount(queryUUID); + decrementNext = true; + + Future> future; + try { + future = nextExecutor.submit(() -> nextCall(queryId)); + } catch (TaskRejectedException e) { + throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Next task rejected by the executor for query " + queryId); + } + + // add this future to the next task map + nextTaskMap.add(queryId, future); + + // wait for the results to be ready + List results = future.get(); + + // remote this future from the next task map + nextTaskMap.remove(queryId, future); + + return results; + } finally { + if (decrementNext) { + decrementConcurrentNextCount(queryUUID); + } + } + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error getting next page for query " + queryId, e); + throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Unknown error getting next page for query " + queryId); + } + } + + private void incrementConcurrentNextCount(UUID queryUUID) throws InterruptedException, QueryException { + if (queryStorageCache.tryLockQueryStatus(queryUUID, queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { + try { + // increment the concurrent next + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryUUID); + if (queryStatus != null && queryStatus.getConcurrentNextCount() < queryProperties.getConcurrentNextLimit()) { + queryStatus.setConcurrentNextCount(queryStatus.getConcurrentNextCount() + 1); + queryStorageCache.updateQueryStatus(queryStatus); + } else { + throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, + "Concurrent next call limit reached: " + queryProperties.getConcurrentNextLimit()); + } + } finally { + queryStorageCache.unlockQueryStatus(queryUUID); + } + } else { + throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to acquire lock on query."); + } + } + + private void decrementConcurrentNextCount(UUID queryUUID) throws InterruptedException, QueryException { + if (queryStorageCache.tryLockQueryStatus(queryUUID, queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { + try { + // increment the concurrent next + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryUUID); + if (queryStatus != null && queryStatus.getConcurrentNextCount() < queryProperties.getConcurrentNextLimit()) { + queryStatus.setConcurrentNextCount(queryStatus.getConcurrentNextCount() - 1); + queryStorageCache.updateQueryStatus(queryStatus); + } else { + throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, + "Concurrent next call limit reached: " + queryProperties.getConcurrentNextLimit()); + } + } finally { + queryStorageCache.unlockQueryStatus(queryUUID); + } + } else { + throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to acquire lock on query."); + } + } + + private List nextCall(String queryId) throws Exception { + List resultList = new ArrayList<>(); + QueryQueueListener resultListener = queryQueueManager.createListener(identifier, queryId); - // set some audit parameters which are used internally - String userDn = currentUser.getPrimaryUser().getDn().subjectDN(); - setInternalAuditParameters(queryLogicName, userDn, parameters); + // keep waiting for results until we're finished + while (!isFinished(queryId)) { + Object[] results = getObjectResults(resultListener.receive(queryProperties.getResultQueueIntervalMillis()).getPayload()); + if (results != null) { + resultList.addAll(Arrays.asList(results)); + } + } - // send an audit record to the auditor - Query query = createQuery(queryLogicName, parameters, userDn, currentUser.getDNs()); - audit(query, queryLogic, parameters, currentUser); + return resultList; + } + + private boolean isFinished(String queryId) { + // get the query stats from the cache + // TODO: It may be more efficient to broadcast a canceled query to all query services vs. hitting the cache each time + QueryStatus queryStatus = queryStorageCache.getQueryStatus(UUID.fromString(queryId)); + // conditions we need to check for + // 1) was this query canceled? + // 2) have we hit the user's results-per-page limit? + // 3) have we hit the query logic's results-per-page limit? + // 4) have we hit the query logic's bytes-per-page limit? + // 5) have we hit the max results (or the max results override)? + // 6) have we reached the "max work" limit? (i.e. next count + seek count) + // 7) are we going to timeout before getting a full page? if so, return partial results + return false; + } + + // TODO: This is totally bogus and should be removed once QueryQueueListener is updated to return objects + private Object[] getObjectResults(byte[] bytes) { + Object[] objects = null; + if (bytes != null) { + objects = new Object[bytes.length]; + for (int i = 0; i < bytes.length; i++) { + objects[i] = bytes[i]; + } + } + return objects; + } + + public void cancel(String queryId, ProxiedUserDetails currentUser) throws QueryException { try { - // persist the query w/ query id in the query storage cache - // @formatter:off - TaskKey taskKey = queryStorageCache.storeQuery( - new QueryPool(getPoolName()), - query, - getMaxConcurrentTasks(queryLogic)); - // @formatter:on + // make sure the query is valid, and the user can act on it + validateRequest(queryId, currentUser); - // TODO: JWO: Figure out how to make query tracing work with our new architecture. Datawave issue #1155 + // cancel the query + cancel(queryId, true); + } catch (QueryException e) { + throw e; + } catch (Exception e) { + QueryException queryException = new QueryException(DatawaveErrorCode.CANCELLATION_ERROR, e, "query_id: " + queryId); + log.error("Unable to cancel query with id " + queryId, queryException); + throw queryException; + } + } + + private void cancel(String queryId, boolean publishEvent) throws InterruptedException, QueryException { + UUID queryUUID = UUID.fromString(queryId); + + // if we have an active next call for this query locally, cancel it + List>> futures = nextTaskMap.get(queryId); + if (futures != null) { + for (Future> future : futures) { + future.cancel(true); + } - // TODO: JWO: Figure out how to make query metrics work with our new architecture. Datawave issue #1156 + // TODO: lock the cache entry and change state to canceled + // try to be nice and acquire the lock before updating the status + if (queryStorageCache.tryLockQueryStatus(queryUUID, queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { + try { + // update query state to CANCELED + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryUUID); + queryStatus.setQueryState(CANCELED); + queryStorageCache.updateQueryStatus(queryStatus); + } finally { + queryStorageCache.unlockQueryStatus(queryUUID); + } + } else { + // TODO: Instead of throwing an exception, do we want to be mean here and update the state to canceled without acquiring a lock? Probably yes... + throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to acquire lock on query"); + } + } else { + if (publishEvent) { + // broadcast the cancel request to all of the query services + appCtx.publishEvent(new QueryRemoteRequestEvent(this, busProperties.getId(), appCtx.getApplicationName(), QueryRequest.cancel(queryId))); + } + } + + if (publishEvent) { + // broadcast the cancel request to all of the executor services + appCtx.publishEvent( + new QueryRemoteRequestEvent(this, busProperties.getId(), queryProperties.getExecutorServiceName(), QueryRequest.cancel(queryId))); + } + } + + public void close(String queryId, ProxiedUserDetails currentUser) throws QueryException { + try { + // make sure the query is valid, and the user can act on it + validateRequest(queryId, currentUser); - return taskKey; + // close the query + close(queryId, true); + } catch (QueryException e) { + throw e; } catch (Exception e) { - log.error("Unknown error storing query", e); - throw new BadRequestQueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR, e); + QueryException queryException = new QueryException(DatawaveErrorCode.CLOSE_ERROR, e, "query_id: " + queryId); + log.error("Unable to close query with id " + queryId, queryException); + throw queryException; } } - public void next(String queryId, ProxiedUserDetails currentUser) throws IOException, InterruptedException, ParseException, QueryException { + private void close(String queryId, boolean publishEvent) throws InterruptedException, QueryException { + UUID queryUUID = UUID.fromString(queryId); + + if (queryStorageCache.tryLockQueryStatus(queryUUID, queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { + try { + // update query state to CLOSED + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryUUID); + queryStatus.setQueryState(CLOSED); + queryStorageCache.updateQueryStatus(queryStatus); + } finally { + queryStorageCache.unlockQueryStatus(queryUUID); + } + } else { + // TODO: Instead of throwing an exception, do we want to be mean here and update the state to canceled without acquiring a lock? Probably yes... + throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to acquire lock on query"); + } + + if (publishEvent) { + // broadcast the close request to all of the executor services + appCtx.publishEvent( + new QueryRemoteRequestEvent(this, busProperties.getId(), queryProperties.getExecutorServiceName(), QueryRequest.close(queryId))); + } + } + + private void validateRequest(String queryId, ProxiedUserDetails currentUser) throws QueryException { // does the query exist? - QueryTask queryTask = queryStorageCache.getTask(null, 0); - if (queryTask == null) { + QueryStatus queryStatus = queryStorageCache.getQueryStatus(UUID.fromString(queryId)); + if (queryStatus == null) { throw new NotFoundQueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH, MessageFormat.format("{0}", queryId)); } + // TODO: Check to see if this is an admin user // does the current user own this query? String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); - Query query = queryTask.getQueryCheckpoint().getPropertiesAsQuery(); + Query query = queryStatus.getQuery(); if (!query.getOwner().equals(userId)) { throw new UnauthorizedQueryException(DatawaveErrorCode.QUERY_OWNER_MISMATCH, MessageFormat.format("{0} != {1}", userId, query.getOwner())); } - - Lock lock = lockManager.getLock(queryId); + } + + @Override + public void handleRemoteRequest(QueryRequest queryRequest) { try { - // try to get a lock on this call to prevent concurrent next calls - // we disallow concurrent next calls to prevent a single user from starving out other users - try { - lock.lock(); - } catch (Exception e) { - lock = null; - throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to acquire lock on query."); - } - - // if we made it this far, we're ready to start checking for results - // TODO: Figure out how to get results from the queue - QueryQueueListener listener = queryQueueManager.createListener(identifier, queryId); - - // TODO: I have no idea what I'm supposed to be receiving here... - Object something; - try { - something = listener.receive(); - } catch (Exception e) { - throw new PreConditionFailedQueryException(DatawaveErrorCode.QUERY_TIMEOUT_OR_SERVER_ERROR, e, MessageFormat.format("id = {0}", queryId)); - } - - // TODO: Where is the page number supposed to come from? It SHOULD come from the cache, right? - - } finally { - if (lock != null) { - try { - lock.unlock(); - } catch (Exception e) { - throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to release lock on query."); - } + switch (queryRequest.getMethod()) { + case CANCEL: + log.trace("Received remote cancel request."); + cancel(queryRequest.getQueryId(), false); + break; + case CLOSE: + log.trace("Received remote close request."); + close(queryRequest.getQueryId(), false); + break; + default: + log.debug("Unknown remote query request method: {}", queryRequest.getMethod()); } + } catch (Exception e) { + log.error("Remote request failed:" + queryRequest); } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java index 4a7a9f89..383f26af 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java @@ -4,8 +4,10 @@ import datawave.microservice.query.QueryParameters; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.web.context.annotation.RequestScope; @Configuration @@ -23,4 +25,17 @@ public QueryParameters queryParameters() { public QueryProperties queryProperties() { return new QueryProperties(); } + + @RefreshScope + @Bean + public ThreadPoolTaskExecutor nextExecutor(QueryProperties queryProperties) { + QueryProperties.ExecutorProperties executorProperties = queryProperties.getNextExecutor(); + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(executorProperties.getCorePoolSize()); + executor.setMaxPoolSize(executorProperties.getMaxPoolSize()); + executor.setQueueCapacity(executorProperties.getQueueCapacity()); + executor.setThreadNamePrefix(executorProperties.getThreadNamePrefix()); + executor.initialize(); + return executor; + } } diff --git a/query-microservices/query-service/service/src/test/resources/config/bootstrap.yml b/query-microservices/query-service/service/src/test/resources/config/bootstrap.yml index 2eeba2b0..38f5a1eb 100644 --- a/query-microservices/query-service/service/src/test/resources/config/bootstrap.yml +++ b/query-microservices/query-service/service/src/test/resources/config/bootstrap.yml @@ -1,7 +1,7 @@ spring: cloud: bus: - enabled: false + enabled: true consul: enabled: false config: From f20408abdd2f35ae2d13d6f23043164c942d78f2 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Wed, 12 May 2021 13:44:32 +0000 Subject: [PATCH 043/218] Created a QueryStorageLock object to encapsolate the locking methods. --- .../query/QueryManagementService.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 437fcd42..5e949050 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -248,7 +248,7 @@ public List next(String queryId, ProxiedUserDetails currentUser) throws } private void incrementConcurrentNextCount(UUID queryUUID) throws InterruptedException, QueryException { - if (queryStorageCache.tryLockQueryStatus(queryUUID, queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { + if (queryStorageCache.getQueryStatusLock(queryUUID).tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { try { // increment the concurrent next QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryUUID); @@ -260,7 +260,7 @@ private void incrementConcurrentNextCount(UUID queryUUID) throws InterruptedExce "Concurrent next call limit reached: " + queryProperties.getConcurrentNextLimit()); } } finally { - queryStorageCache.unlockQueryStatus(queryUUID); + queryStorageCache.getQueryStatusLock(queryUUID).unlock(); } } else { throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to acquire lock on query."); @@ -268,7 +268,7 @@ private void incrementConcurrentNextCount(UUID queryUUID) throws InterruptedExce } private void decrementConcurrentNextCount(UUID queryUUID) throws InterruptedException, QueryException { - if (queryStorageCache.tryLockQueryStatus(queryUUID, queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { + if (queryStorageCache.getQueryStatusLock(queryUUID).tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { try { // increment the concurrent next QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryUUID); @@ -280,7 +280,7 @@ private void decrementConcurrentNextCount(UUID queryUUID) throws InterruptedExce "Concurrent next call limit reached: " + queryProperties.getConcurrentNextLimit()); } } finally { - queryStorageCache.unlockQueryStatus(queryUUID); + queryStorageCache.getQueryStatusLock(queryUUID).unlock(); } } else { throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to acquire lock on query."); @@ -358,14 +358,14 @@ private void cancel(String queryId, boolean publishEvent) throws InterruptedExce // TODO: lock the cache entry and change state to canceled // try to be nice and acquire the lock before updating the status - if (queryStorageCache.tryLockQueryStatus(queryUUID, queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { + if (queryStorageCache.getQueryStatusLock(queryUUID).tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { try { // update query state to CANCELED QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryUUID); queryStatus.setQueryState(CANCELED); queryStorageCache.updateQueryStatus(queryStatus); } finally { - queryStorageCache.unlockQueryStatus(queryUUID); + queryStorageCache.getQueryStatusLock(queryUUID).unlock(); } } else { // TODO: Instead of throwing an exception, do we want to be mean here and update the state to canceled without acquiring a lock? Probably yes... @@ -404,14 +404,14 @@ public void close(String queryId, ProxiedUserDetails currentUser) throws QueryEx private void close(String queryId, boolean publishEvent) throws InterruptedException, QueryException { UUID queryUUID = UUID.fromString(queryId); - if (queryStorageCache.tryLockQueryStatus(queryUUID, queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { + if (queryStorageCache.getQueryStatusLock(queryUUID).tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { try { // update query state to CLOSED QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryUUID); queryStatus.setQueryState(CLOSED); queryStorageCache.updateQueryStatus(queryStatus); } finally { - queryStorageCache.unlockQueryStatus(queryUUID); + queryStorageCache.getQueryStatusLock(queryUUID).unlock(); } } else { // TODO: Instead of throwing an exception, do we want to be mean here and update the state to canceled without acquiring a lock? Probably yes... From 6764867ef57d3c900045ef9daa2c2375711666d4 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Fri, 14 May 2021 12:40:00 +0000 Subject: [PATCH 044/218] Removed the query lock manager since we now have locking coupled with the hazelcast cache --- .../service/src/test/resources/config/application.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/query-microservices/query-service/service/src/test/resources/config/application.yml b/query-microservices/query-service/service/src/test/resources/config/application.yml index cf3c378a..59f3bd1f 100644 --- a/query-microservices/query-service/service/src/test/resources/config/application.yml +++ b/query-microservices/query-service/service/src/test/resources/config/application.yml @@ -123,8 +123,6 @@ datawave: storage: backend: LOCAL - lockManager: - LOCAL syncStorage: true sendNotifications: From ca37f1e67bb7c472662092a0d90defb3513a046d Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Fri, 14 May 2021 16:38:52 +0000 Subject: [PATCH 045/218] Added the query cache to the query starter. Created an initial directory structure for the query-executor. --- query-microservices/query-service/service/pom.xml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/query-microservices/query-service/service/pom.xml b/query-microservices/query-service/service/pom.xml index e315c156..f275cdfb 100644 --- a/query-microservices/query-service/service/pom.xml +++ b/query-microservices/query-service/service/pom.xml @@ -15,7 +15,6 @@ 3.2.1-SNAPSHOT 1.0-SNAPSHOT 1.0-SNAPSHOT - 1.0-SNAPSHOT 1.6.6-SNAPSHOT 1.3-SNAPSHOT 1.0-SNAPSHOT @@ -43,11 +42,6 @@ query-api ${version.microservice.query-api} - - gov.nsa.datawave.microservice - query-cache - ${version.microservice.query-cache} - gov.nsa.datawave.microservice spring-boot-starter-datawave @@ -79,10 +73,6 @@ gov.nsa.datawave.microservice query-api - - gov.nsa.datawave.microservice - query-cache - gov.nsa.datawave.microservice spring-boot-starter-datawave From f338d8ad54aaefb8f40f99262d3060880c55d88f Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Fri, 14 May 2021 19:27:48 +0000 Subject: [PATCH 046/218] Updated to pass the principals authorizations via the QueryStatus. Started filling out the query executor. --- .../query/QueryManagementService.java | 18 ++++++++++++++++-- .../microservice/query/QueryServiceTest.java | 8 ++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 5e949050..04cab245 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -19,6 +19,8 @@ import datawave.microservice.query.remote.QueryRequest; import datawave.microservice.query.remote.QueryRequestHandler; import datawave.microservice.query.util.QueryUtil; +import datawave.security.authorization.DatawavePrincipal; +import datawave.security.util.AuthorizationsUtil; import datawave.security.util.ProxiedEntityUtils; import datawave.webservice.common.audit.AuditParameters; import datawave.webservice.common.audit.Auditor; @@ -30,6 +32,7 @@ import datawave.webservice.query.exception.UnauthorizedQueryException; import datawave.webservice.query.result.event.ResponseObjectFactory; import datawave.webservice.query.util.QueryUncaughtExceptionHandler; +import org.apache.accumulo.core.security.Authorizations; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.cloud.bus.BusProperties; @@ -46,7 +49,10 @@ import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.concurrent.Future; @@ -130,13 +136,17 @@ public TaskKey define(String queryLogicName, MultiValueMap parame String userDn = currentUser.getPrimaryUser().getDn().subjectDN(); setInternalAuditParameters(queryLogicName, userDn, parameters); + // calculate the auths for this query + Set auths = AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), + new DatawavePrincipal(currentUser.getProxiedUsers())); + try { // persist the query w/ query id in the query storage cache // TODO: JWO: storeQuery assumes that this is a 'create' call, but this is a 'define' call. // @formatter:off TaskKey taskKey = queryStorageCache.storeQuery( new QueryPool(getPoolName()), - createQuery(queryLogicName, parameters, userDn, currentUser.getDNs()), + createQuery(queryLogicName, parameters, userDn, currentUser.getDNs()), auths, getMaxConcurrentTasks(queryLogic)); // @formatter:on @@ -179,12 +189,16 @@ public TaskKey create(String queryLogicName, MultiValueMap parame Query query = createQuery(queryLogicName, parameters, userDn, currentUser.getDNs()); audit(query, queryLogic, parameters, currentUser); + // calculate the auths for this query + Set auths = AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), + new DatawavePrincipal(currentUser.getProxiedUsers())); + try { // persist the query w/ query id in the query storage cache // @formatter:off TaskKey taskKey = queryStorageCache.storeQuery( new QueryPool(getPoolName()), - query, + query, auths, getMaxConcurrentTasks(queryLogic)); // @formatter:on diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java index 7ee00548..fdba990b 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java @@ -1,5 +1,7 @@ package datawave.microservice.query; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import datawave.marking.ColumnVisibilitySecurityMarking; import datawave.microservice.authorization.jwt.JWTRestTemplate; import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; @@ -57,7 +59,8 @@ public void setup() { @Test public void testDefineQuery() { Collection roles = Collections.singleton("AuthorizedUser"); - DatawaveUser uathDWUser = new DatawaveUser(DN, USER, null, roles, null, System.currentTimeMillis()); + Collection auths = Collections.singleton("ALL"); + DatawaveUser uathDWUser = new DatawaveUser(DN, USER, auths, roles, null, System.currentTimeMillis()); ProxiedUserDetails authUser = new ProxiedUserDetails(Collections.singleton(uathDWUser), uathDWUser.getCreationTime()); UriComponents uri = UriComponentsBuilder.newInstance().scheme("https").host("localhost").port(webServicePort).path("/query/v1/EventQuery/define") @@ -87,7 +90,8 @@ public void testDefineQuery() { @Test public void testCreateQuery() { Collection roles = Collections.singleton("AuthorizedUser"); - DatawaveUser uathDWUser = new DatawaveUser(DN, USER, null, roles, null, System.currentTimeMillis()); + Collection auths = Collections.singleton("ALL"); + DatawaveUser uathDWUser = new DatawaveUser(DN, USER, auths, roles, null, System.currentTimeMillis()); ProxiedUserDetails authUser = new ProxiedUserDetails(Collections.singleton(uathDWUser), uathDWUser.getCreationTime()); UriComponents uri = UriComponentsBuilder.newInstance().scheme("https").host("localhost").port(webServicePort).path("/query/v1/EventQuery/create") From 94fe563d82c417f75e6268d72922e16cfd5efe94 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 14 May 2021 20:20:53 -0400 Subject: [PATCH 047/218] Updated query storage cache to use stop creating it's own hazelcast instance, and start using the cluster instance. Also updated query storage cache to use as much spring-provided configuration as possible vs using custom configuration. --- .../src/test/resources/config/application.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/query-microservices/query-service/service/src/test/resources/config/application.yml b/query-microservices/query-service/service/src/test/resources/config/application.yml index 59f3bd1f..1d1daeab 100644 --- a/query-microservices/query-service/service/src/test/resources/config/application.yml +++ b/query-microservices/query-service/service/src/test/resources/config/application.yml @@ -92,6 +92,11 @@ warehouse: ivaratorCacheScanTimeoutMinutes: 60 modelName: 'DATAWAVE' +query: + storage: + syncStorage: true + sendNotifications: true + datawave: lock: type: local @@ -120,13 +125,6 @@ datawave: definedView: "LuceneUUIDEventQuery" - fieldName: "PAGE_TITLE" definedView: "LuceneUUIDEventQuery" - storage: - backend: - LOCAL - syncStorage: - true - sendNotifications: - true logic: factory: enabled: true From 3dc1b0f046e36aabff3e28787d218e62a0b43a8a Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Tue, 18 May 2021 16:41:20 -0400 Subject: [PATCH 048/218] Various updates: - Updated Query Json Serialization to be polymorphic. - Updated Query Json Serialization to serialize the query as a string. - Created a ProxiedUserEntity-specific AuthorizationsUtil to avoid usage of DatawavePrincipal. - Updated all query microservices to use the spring-boot provided log4j dependencies. - Moved Local query storage cache implementation to test and created a test jar. - Renamed the query cache dependency to query-storage to be consistent with the api and parent modules. - Renamed various application launcher classes to be consistent with our existing microservices. - Removed Autowired injection for components where a constructor would suffice. - Updated QueryTaskNotification to have it be published via the spring cloud bus. - Many other various changes/improvements. --- .../query-service/service/pom.xml | 38 ++++++------------- .../query/QueryManagementService.java | 29 +++++--------- .../query/config/QueryServiceConfig.java | 1 + .../microservice/query/QueryServiceTest.java | 2 - 4 files changed, 21 insertions(+), 49 deletions(-) diff --git a/query-microservices/query-service/service/pom.xml b/query-microservices/query-service/service/pom.xml index f275cdfb..4d640aa3 100644 --- a/query-microservices/query-service/service/pom.xml +++ b/query-microservices/query-service/service/pom.xml @@ -12,41 +12,17 @@ DATAWAVE Query Microservice datawave.microservice.query.QueryService - 3.2.1-SNAPSHOT - 1.0-SNAPSHOT 1.0-SNAPSHOT - 1.6.6-SNAPSHOT 1.3-SNAPSHOT 1.0-SNAPSHOT - - gov.nsa.datawave - datawave-query-core - ${version.datawave} - - - * - org.slf4j - - - - - gov.nsa.datawave.microservice - query - ${version.microservice.query} - gov.nsa.datawave.microservice query-api ${version.microservice.query-api} - - gov.nsa.datawave.microservice - spring-boot-starter-datawave - ${version.microservice.starter} - gov.nsa.datawave.microservice spring-boot-starter-datawave-audit @@ -58,9 +34,11 @@ ${version.microservice.starter-query} - gov.nsa.datawave.webservices - datawave-ws-client - ${version.datawave} + gov.nsa.datawave.microservice + spring-boot-starter-datawave-query + ${version.microservice.starter-query} + pom + import @@ -85,6 +63,12 @@ gov.nsa.datawave.microservice spring-boot-starter-datawave-query + + gov.nsa.datawave.microservice + query-storage + test-jar + test + junit junit diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 04cab245..ef5c0155 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -5,6 +5,7 @@ import datawave.marking.SecurityMarking; import datawave.microservice.audit.AuditClient; import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.authorization.util.AuthorizationsUtil; import datawave.microservice.common.audit.PrivateAuditConstants; import datawave.microservice.common.storage.QueryPool; import datawave.microservice.common.storage.QueryQueueListener; @@ -19,8 +20,6 @@ import datawave.microservice.query.remote.QueryRequest; import datawave.microservice.query.remote.QueryRequestHandler; import datawave.microservice.query.util.QueryUtil; -import datawave.security.authorization.DatawavePrincipal; -import datawave.security.util.AuthorizationsUtil; import datawave.security.util.ProxiedEntityUtils; import datawave.webservice.common.audit.AuditParameters; import datawave.webservice.common.audit.Auditor; @@ -32,11 +31,10 @@ import datawave.webservice.query.exception.UnauthorizedQueryException; import datawave.webservice.query.result.event.ResponseObjectFactory; import datawave.webservice.query.util.QueryUncaughtExceptionHandler; -import org.apache.accumulo.core.security.Authorizations; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.cloud.bus.BusProperties; -import org.springframework.cloud.bus.event.QueryRemoteRequestEvent; +import org.springframework.cloud.bus.event.RemoteQueryRequestEvent; import org.springframework.context.ApplicationContext; import org.springframework.core.task.TaskRejectedException; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; @@ -49,10 +47,7 @@ import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.UUID; import java.util.concurrent.Future; @@ -136,17 +131,14 @@ public TaskKey define(String queryLogicName, MultiValueMap parame String userDn = currentUser.getPrimaryUser().getDn().subjectDN(); setInternalAuditParameters(queryLogicName, userDn, parameters); - // calculate the auths for this query - Set auths = AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), - new DatawavePrincipal(currentUser.getProxiedUsers())); - try { // persist the query w/ query id in the query storage cache // TODO: JWO: storeQuery assumes that this is a 'create' call, but this is a 'define' call. // @formatter:off TaskKey taskKey = queryStorageCache.storeQuery( new QueryPool(getPoolName()), - createQuery(queryLogicName, parameters, userDn, currentUser.getDNs()), auths, + createQuery(queryLogicName, parameters, userDn, currentUser.getDNs()), + AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), currentUser), getMaxConcurrentTasks(queryLogic)); // @formatter:on @@ -189,16 +181,13 @@ public TaskKey create(String queryLogicName, MultiValueMap parame Query query = createQuery(queryLogicName, parameters, userDn, currentUser.getDNs()); audit(query, queryLogic, parameters, currentUser); - // calculate the auths for this query - Set auths = AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), - new DatawavePrincipal(currentUser.getProxiedUsers())); - try { // persist the query w/ query id in the query storage cache // @formatter:off TaskKey taskKey = queryStorageCache.storeQuery( new QueryPool(getPoolName()), - query, auths, + query, + AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), currentUser), getMaxConcurrentTasks(queryLogic)); // @formatter:on @@ -388,14 +377,14 @@ private void cancel(String queryId, boolean publishEvent) throws InterruptedExce } else { if (publishEvent) { // broadcast the cancel request to all of the query services - appCtx.publishEvent(new QueryRemoteRequestEvent(this, busProperties.getId(), appCtx.getApplicationName(), QueryRequest.cancel(queryId))); + appCtx.publishEvent(new RemoteQueryRequestEvent(this, busProperties.getId(), appCtx.getApplicationName(), QueryRequest.cancel(queryId))); } } if (publishEvent) { // broadcast the cancel request to all of the executor services appCtx.publishEvent( - new QueryRemoteRequestEvent(this, busProperties.getId(), queryProperties.getExecutorServiceName(), QueryRequest.cancel(queryId))); + new RemoteQueryRequestEvent(this, busProperties.getId(), queryProperties.getExecutorServiceName(), QueryRequest.cancel(queryId))); } } @@ -435,7 +424,7 @@ private void close(String queryId, boolean publishEvent) throws InterruptedExcep if (publishEvent) { // broadcast the close request to all of the executor services appCtx.publishEvent( - new QueryRemoteRequestEvent(this, busProperties.getId(), queryProperties.getExecutorServiceName(), QueryRequest.close(queryId))); + new RemoteQueryRequestEvent(this, busProperties.getId(), queryProperties.getExecutorServiceName(), QueryRequest.close(queryId))); } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java index 383f26af..179e4687 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java @@ -21,6 +21,7 @@ public QueryParameters queryParameters() { } @Bean + @ConditionalOnMissingBean(QueryProperties.class) @ConfigurationProperties("datawave.query") public QueryProperties queryProperties() { return new QueryProperties(); diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java index fdba990b..790b9d61 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java @@ -1,7 +1,5 @@ package datawave.microservice.query; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import datawave.marking.ColumnVisibilitySecurityMarking; import datawave.microservice.authorization.jwt.JWTRestTemplate; import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; From 3dd0a19f4842df0c3ea94886dbb92822cd56c7ae Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 19 May 2021 13:41:00 -0400 Subject: [PATCH 049/218] Reworked some of the conditional annotations for query storage, and updated QueryStorageCacheTest to work with an external rabbitmq --- .../service/src/test/resources/config/application.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/query-microservices/query-service/service/src/test/resources/config/application.yml b/query-microservices/query-service/service/src/test/resources/config/application.yml index 1d1daeab..60b828df 100644 --- a/query-microservices/query-service/service/src/test/resources/config/application.yml +++ b/query-microservices/query-service/service/src/test/resources/config/application.yml @@ -98,9 +98,6 @@ query: sendNotifications: true datawave: - lock: - type: local - metadata: all-metadata-auths: - PRIVATE,PUBLIC From a6f5ac0be7523cc76a9b563ff796fc671c4381a4 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 19 May 2021 14:19:37 -0400 Subject: [PATCH 050/218] Added separate create and define query calls for the query storage cache. --- .../datawave/microservice/query/QueryManagementService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index ef5c0155..46e6dfc8 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -135,7 +135,7 @@ public TaskKey define(String queryLogicName, MultiValueMap parame // persist the query w/ query id in the query storage cache // TODO: JWO: storeQuery assumes that this is a 'create' call, but this is a 'define' call. // @formatter:off - TaskKey taskKey = queryStorageCache.storeQuery( + TaskKey taskKey = queryStorageCache.defineQuery( new QueryPool(getPoolName()), createQuery(queryLogicName, parameters, userDn, currentUser.getDNs()), AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), currentUser), @@ -184,7 +184,7 @@ public TaskKey create(String queryLogicName, MultiValueMap parame try { // persist the query w/ query id in the query storage cache // @formatter:off - TaskKey taskKey = queryStorageCache.storeQuery( + TaskKey taskKey = queryStorageCache.createQuery( new QueryPool(getPoolName()), query, AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), currentUser), From 6f64685cc1e493409e93c5a56f9eddcc62e4ca56 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Fri, 21 May 2021 17:53:34 +0000 Subject: [PATCH 051/218] Kafka and RabbitMq working with query storage cache test. Changed result queues to pass a Result object. --- .../query/QueryManagementService.java | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 46e6dfc8..901f391b 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -12,6 +12,7 @@ import datawave.microservice.common.storage.QueryQueueManager; import datawave.microservice.common.storage.QueryStatus; import datawave.microservice.common.storage.QueryStorageCache; +import datawave.microservice.common.storage.Result; import datawave.microservice.common.storage.TaskKey; import datawave.microservice.query.config.QueryExpirationProperties; import datawave.microservice.query.config.QueryProperties; @@ -296,7 +297,7 @@ private List nextCall(String queryId) throws Exception { // keep waiting for results until we're finished while (!isFinished(queryId)) { - Object[] results = getObjectResults(resultListener.receive(queryProperties.getResultQueueIntervalMillis()).getPayload()); + Object[] results = resultListener.receive(queryProperties.getResultQueueIntervalMillis()).getPayload().getPayloadObject(); if (results != null) { resultList.addAll(Arrays.asList(results)); } @@ -321,18 +322,6 @@ private boolean isFinished(String queryId) { return false; } - // TODO: This is totally bogus and should be removed once QueryQueueListener is updated to return objects - private Object[] getObjectResults(byte[] bytes) { - Object[] objects = null; - if (bytes != null) { - objects = new Object[bytes.length]; - for (int i = 0; i < bytes.length; i++) { - objects[i] = bytes[i]; - } - } - return objects; - } - public void cancel(String queryId, ProxiedUserDetails currentUser) throws QueryException { try { // make sure the query is valid, and the user can act on it From 7995adecebf112651e00c0a68eafa7f1adba4dd8 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Fri, 21 May 2021 18:53:46 +0000 Subject: [PATCH 052/218] formatting --- .../datawave/microservice/query/QueryManagementService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 901f391b..68bd4fa8 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -297,7 +297,7 @@ private List nextCall(String queryId) throws Exception { // keep waiting for results until we're finished while (!isFinished(queryId)) { - Object[] results = resultListener.receive(queryProperties.getResultQueueIntervalMillis()).getPayload().getPayloadObject(); + Object[] results = resultListener.receive(queryProperties.getResultQueueIntervalMillis()).getPayload().getPayload(); if (results != null) { resultList.addAll(Arrays.asList(results)); } From 35dc981a86b833faf256910599242ec839fdc433 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 21 May 2021 13:53:46 -0400 Subject: [PATCH 053/218] Added dev-messaging docker-compose configuration. Updated QueryStorageCacheTest to run with each implementation (excluding rabbitmq). Added convenience methods to KafkaQueryQueueManager and added methods which can check to ensure that groups and topics are created. --- .../microservice/query/QueryManagementService.java | 13 ++++++++----- .../query/config/QueryServiceConfig.java | 8 ++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 68bd4fa8..47b2a1f7 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -12,7 +12,7 @@ import datawave.microservice.common.storage.QueryQueueManager; import datawave.microservice.common.storage.QueryStatus; import datawave.microservice.common.storage.QueryStorageCache; -import datawave.microservice.common.storage.Result; +import datawave.microservice.common.storage.QueryStorageLock; import datawave.microservice.common.storage.TaskKey; import datawave.microservice.query.config.QueryExpirationProperties; import datawave.microservice.query.config.QueryProperties; @@ -70,6 +70,7 @@ public class QueryManagementService implements QueryRequestHandler { private final QueryParameters queryParameters; // Note: SecurityMarking needs to be request scoped private final SecurityMarking securityMarking; + private final QueryLogicFactory queryLogicFactory; private final ResponseObjectFactory responseObjectFactory; private final QueryStorageCache queryStorageCache; @@ -272,7 +273,8 @@ private void incrementConcurrentNextCount(UUID queryUUID) throws InterruptedExce } private void decrementConcurrentNextCount(UUID queryUUID) throws InterruptedException, QueryException { - if (queryStorageCache.getQueryStatusLock(queryUUID).tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { + QueryStorageLock statusLock = queryStorageCache.getQueryStatusLock(queryUUID); + if (statusLock.tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { try { // increment the concurrent next QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryUUID); @@ -284,7 +286,7 @@ private void decrementConcurrentNextCount(UUID queryUUID) throws InterruptedExce "Concurrent next call limit reached: " + queryProperties.getConcurrentNextLimit()); } } finally { - queryStorageCache.getQueryStatusLock(queryUUID).unlock(); + statusLock.unlock(); } } else { throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to acquire lock on query."); @@ -350,14 +352,15 @@ private void cancel(String queryId, boolean publishEvent) throws InterruptedExce // TODO: lock the cache entry and change state to canceled // try to be nice and acquire the lock before updating the status - if (queryStorageCache.getQueryStatusLock(queryUUID).tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { + QueryStorageLock statusLock = queryStorageCache.getQueryStatusLock(queryUUID); + if (statusLock.tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { try { // update query state to CANCELED QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryUUID); queryStatus.setQueryState(CANCELED); queryStorageCache.updateQueryStatus(queryStatus); } finally { - queryStorageCache.getQueryStatusLock(queryUUID).unlock(); + statusLock.unlock(); } } else { // TODO: Instead of throwing an exception, do we want to be mean here and update the state to canceled without acquiring a lock? Probably yes... diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java index 179e4687..8060a316 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java @@ -1,5 +1,6 @@ package datawave.microservice.query.config; +import datawave.marking.MarkingFunctions; import datawave.microservice.query.DefaultQueryParameters; import datawave.microservice.query.QueryParameters; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -20,6 +21,13 @@ public QueryParameters queryParameters() { return new DefaultQueryParameters(); } + @Bean + @ConditionalOnMissingBean + @RequestScope + public MarkingFunctions markingFunctions() { + return new MarkingFunctions.Default(); + } + @Bean @ConditionalOnMissingBean(QueryProperties.class) @ConfigurationProperties("datawave.query") From 3f30b35d1d4dc9f138cea30ddb798ffc5ad9f9b1 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 27 May 2021 16:20:38 -0400 Subject: [PATCH 054/218] Implemented the query next call. --- .../config/QueryExpirationProperties.java | 95 ++++--- .../query/config/QueryProperties.java | 23 +- .../microservice/query/QueryController.java | 7 +- .../query/QueryManagementService.java | 264 ++++++++++++------ .../query/config/QueryServiceConfig.java | 10 +- .../microservice/query/runner/NextCall.java | 234 ++++++++++++++++ 6 files changed, 503 insertions(+), 130 deletions(-) create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java index 5225ad62..9dcefa04 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java @@ -1,75 +1,94 @@ package datawave.microservice.query.config; +import java.util.concurrent.TimeUnit; + public class QueryExpirationProperties { - public static final int PAGE_TIMEOUT_MIN_DEFAULT = 60; - public static final int IDLE_TIME_MIN_DEFAULT = 15; + private long idleTimeout = 15; + private TimeUnit idleTimeUnit = TimeUnit.MINUTES; + private long callTimeout = 60; + private TimeUnit callTimeUnit = TimeUnit.MINUTES; + private long shortCircuitCheckTime = callTimeout / 2; + private TimeUnit shortCircuitCheckTimeUnit = TimeUnit.MINUTES; + private long shortCircuitTimeout = Math.round(0.97 * callTimeout); + private TimeUnit shortCircuitTimeUnit = TimeUnit.MINUTES; + + public long getIdleTimeout() { + return idleTimeout; + } + + public long getIdleTimeoutMillis() { + return idleTimeUnit.toMillis(idleTimeout); + } + + public void setIdleTimeout(long idleTimeout) { + this.idleTimeout = idleTimeout; + } - private long idleTimeMinutes = IDLE_TIME_MIN_DEFAULT; - private long callTimeMinutes = PAGE_TIMEOUT_MIN_DEFAULT; - private long pageSizeShortCircuitCheckTimeMinutes = PAGE_TIMEOUT_MIN_DEFAULT / 2; - private long pageShortCircuitTimeoutMinutes = Math.round(0.97 * PAGE_TIMEOUT_MIN_DEFAULT); + public TimeUnit getIdleTimeUnit() { + return idleTimeUnit; + } - public long getIdleTimeMinutes() { - return idleTimeMinutes; + public void setIdleTimeUnit(TimeUnit idleTimeUnit) { + this.idleTimeUnit = idleTimeUnit; } - public long getIdleTimeInMS() { - return idleTimeMinutes * 60 * 1000; + public long getCallTimeout() { + return callTimeout; } - public void setIdleTime(long idleTimeMinutes) { - this.idleTimeMinutes = idleTimeMinutes; + public long getCallTimeoutMillis() { + return callTimeUnit.toMillis(callTimeout); } - public void setIdleTimeMinutes(long idleTimeMinutes) { - this.idleTimeMinutes = idleTimeMinutes; + public void setCallTimeout(long callTimeout) { + this.callTimeout = callTimeout; } - public long getCallTimeMinutes() { - return callTimeMinutes; + public TimeUnit getCallTimeUnit() { + return callTimeUnit; } - public long getCallTimeInMS() { - return callTimeMinutes * 60 * 1000; + public void setCallTimeUnit(TimeUnit callTimeUnit) { + this.callTimeUnit = callTimeUnit; } - public void setCallTime(long callTimeMinutes) { - this.callTimeMinutes = callTimeMinutes; + public long getShortCircuitCheckTime() { + return shortCircuitCheckTime; } - public void setCallTimeMinutes(long callTimeMinutes) { - this.callTimeMinutes = callTimeMinutes; + public long getShortCircuitCheckTimeMillis() { + return shortCircuitCheckTimeUnit.toMillis(shortCircuitCheckTime); } - public float getPageSizeShortCircuitCheckTimeMinutes() { - return pageSizeShortCircuitCheckTimeMinutes; + public void setShortCircuitCheckTime(long shortCircuitCheckTime) { + this.shortCircuitCheckTime = shortCircuitCheckTime; } - public long getPageSizeShortCircuitCheckTimeInMS() { - return pageSizeShortCircuitCheckTimeMinutes * 60 * 1000; + public TimeUnit getShortCircuitCheckTimeUnit() { + return shortCircuitCheckTimeUnit; } - public void setPageSizeShortCircuitCheckTime(long pageSizeShortCircuitCheckTimeMinutes) { - this.pageSizeShortCircuitCheckTimeMinutes = pageSizeShortCircuitCheckTimeMinutes; + public void setShortCircuitCheckTimeUnit(TimeUnit shortCircuitCheckTimeUnit) { + this.shortCircuitCheckTimeUnit = shortCircuitCheckTimeUnit; } - public void setPageSizeShortCircuitCheckTimeMinutes(long pageSizeShortCircuitCheckTimeMinutes) { - this.pageSizeShortCircuitCheckTimeMinutes = pageSizeShortCircuitCheckTimeMinutes; + public long getShortCircuitTimeout() { + return shortCircuitTimeout; } - public long getPageShortCircuitTimeoutMinutes() { - return pageShortCircuitTimeoutMinutes; + public long getShortCircuitTimeoutMillis() { + return shortCircuitTimeUnit.toMillis(shortCircuitTimeout); } - public long getPageShortCircuitTimeoutInMS() { - return pageShortCircuitTimeoutMinutes * 60 * 1000; + public void setShortCircuitTimeout(long shortCircuitTimeout) { + this.shortCircuitTimeout = shortCircuitTimeout; } - public void setPageShortCircuitTimeout(long pageShortCircuitTimeoutMinutes) { - this.pageShortCircuitTimeoutMinutes = pageShortCircuitTimeoutMinutes; + public TimeUnit getShortCircuitTimeUnit() { + return shortCircuitTimeUnit; } - public void setPageShortCircuitTimeoutMinutes(long pageShortCircuitTimeoutMinutes) { - this.pageShortCircuitTimeoutMinutes = pageShortCircuitTimeoutMinutes; + public void setShortCircuitTimeUnit(TimeUnit shortCircuitTimeUnit) { + this.shortCircuitTimeUnit = shortCircuitTimeUnit; } } diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java index b8a0930b..65fff008 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java @@ -18,7 +18,8 @@ public class QueryProperties { @NotEmpty private String privilegedRole = "PrivilegedUser"; @Positive - private long resultQueueIntervalMillis = TimeUnit.SECONDS.toMillis(60); + private long resultPollRate = TimeUnit.SECONDS.toMillis(30); + private TimeUnit resultPollRateTimeUnit = TimeUnit.MILLISECONDS; // The amount of time to wait for the lock to be acquired @PositiveOrZero private long lockWaitTimeMillis = TimeUnit.SECONDS.toMillis(5); @@ -46,12 +47,24 @@ public void setPrivilegedRole(String privilegedRole) { this.privilegedRole = privilegedRole; } - public long getResultQueueIntervalMillis() { - return resultQueueIntervalMillis; + public long getResultPollRate() { + return resultPollRate; } - public void setResultQueueIntervalMillis(long resultQueueIntervalMillis) { - this.resultQueueIntervalMillis = resultQueueIntervalMillis; + public void setResultPollRate(long resultPollRate) { + this.resultPollRate = resultPollRate; + } + + public TimeUnit getResultPollRateTimeUnit() { + return resultPollRateTimeUnit; + } + + public void setResultPollRateTimeUnit(TimeUnit resultPollRateTimeUnit) { + this.resultPollRateTimeUnit = resultPollRateTimeUnit; + } + + public long getResultPollRateMillis() { + return resultPollRateTimeUnit.toMillis(resultPollRate); } public long getLockWaitTimeMillis() { diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 5fb1d4f8..50acf9a3 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -18,8 +18,6 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import java.util.List; - @RestController @RequestMapping(path = "/v1", produces = MediaType.APPLICATION_JSON_VALUE) public class QueryController { @@ -63,10 +61,7 @@ public GenericResponse create(@PathVariable(name = "queryLogic") String @RequestMapping(path = "{queryId}/next", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public BaseQueryResponse next(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { - - List resultObjects = queryManagementService.next(queryId, currentUser); - - return null; + return queryManagementService.next(queryId, currentUser); } @Timed(name = "dw.query.cancel", absolute = true) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 47b2a1f7..e504f53c 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -8,30 +8,33 @@ import datawave.microservice.authorization.util.AuthorizationsUtil; import datawave.microservice.common.audit.PrivateAuditConstants; import datawave.microservice.common.storage.QueryPool; -import datawave.microservice.common.storage.QueryQueueListener; import datawave.microservice.common.storage.QueryQueueManager; import datawave.microservice.common.storage.QueryStatus; import datawave.microservice.common.storage.QueryStorageCache; import datawave.microservice.common.storage.QueryStorageLock; import datawave.microservice.common.storage.TaskKey; -import datawave.microservice.query.config.QueryExpirationProperties; import datawave.microservice.query.config.QueryProperties; import datawave.microservice.query.logic.QueryLogic; import datawave.microservice.query.logic.QueryLogicFactory; import datawave.microservice.query.remote.QueryRequest; import datawave.microservice.query.remote.QueryRequestHandler; +import datawave.microservice.query.runner.NextCall; import datawave.microservice.query.util.QueryUtil; import datawave.security.util.ProxiedEntityUtils; import datawave.webservice.common.audit.AuditParameters; import datawave.webservice.common.audit.Auditor; import datawave.webservice.query.Query; +import datawave.webservice.query.cache.QueryMetricFactory; +import datawave.webservice.query.cache.ResultsPage; import datawave.webservice.query.exception.BadRequestQueryException; import datawave.webservice.query.exception.DatawaveErrorCode; +import datawave.webservice.query.exception.NoResultsQueryException; import datawave.webservice.query.exception.NotFoundQueryException; import datawave.webservice.query.exception.QueryException; import datawave.webservice.query.exception.UnauthorizedQueryException; import datawave.webservice.query.result.event.ResponseObjectFactory; import datawave.webservice.query.util.QueryUncaughtExceptionHandler; +import datawave.webservice.result.BaseQueryResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.cloud.bus.BusProperties; @@ -46,11 +49,9 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Arrays; +import java.util.Date; import java.util.List; import java.util.UUID; -import java.util.concurrent.Future; import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CANCELED; import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CLOSED; @@ -72,34 +73,37 @@ public class QueryManagementService implements QueryRequestHandler { private final SecurityMarking securityMarking; private final QueryLogicFactory queryLogicFactory; + private final QueryMetricFactory queryMetricFactory; private final ResponseObjectFactory responseObjectFactory; private final QueryStorageCache queryStorageCache; private final QueryQueueManager queryQueueManager; private final AuditClient auditClient; - private final ThreadPoolTaskExecutor nextExecutor; private final String identifier; - private MultiValueMap>> nextTaskMap = new LinkedMultiValueMap<>(); + private final ThreadPoolTaskExecutor nextCallExecutor; + private final MultiValueMap nextCallMap = new LinkedMultiValueMap<>(); // TODO: JWO: Pull these from configuration instead private final int PAGE_TIMEOUT_MIN = 1; - private final int PAGE_TIMEOUT_MAX = QueryExpirationProperties.PAGE_TIMEOUT_MIN_DEFAULT; + private final int PAGE_TIMEOUT_MAX = 60; public QueryManagementService(QueryProperties queryProperties, ApplicationContext appCtx, BusProperties busProperties, QueryParameters queryParameters, - SecurityMarking securityMarking, QueryLogicFactory queryLogicFactory, ResponseObjectFactory responseObjectFactory, - QueryStorageCache queryStorageCache, QueryQueueManager queryQueueManager, AuditClient auditClient, ThreadPoolTaskExecutor nextExecutor) { + SecurityMarking securityMarking, QueryLogicFactory queryLogicFactory, QueryMetricFactory queryMetricFactory, + ResponseObjectFactory responseObjectFactory, QueryStorageCache queryStorageCache, QueryQueueManager queryQueueManager, + AuditClient auditClient, ThreadPoolTaskExecutor nextCallExecutor) { this.queryProperties = queryProperties; this.appCtx = appCtx; this.busProperties = busProperties; this.queryParameters = queryParameters; this.securityMarking = securityMarking; this.queryLogicFactory = queryLogicFactory; + this.queryMetricFactory = queryMetricFactory; this.responseObjectFactory = responseObjectFactory; this.queryStorageCache = queryStorageCache; this.queryQueueManager = queryQueueManager; this.auditClient = auditClient; - this.nextExecutor = nextExecutor; + this.nextCallExecutor = nextCallExecutor; String hostname; try { @@ -210,37 +214,64 @@ public TaskKey create(String queryLogicName, MultiValueMap parame } } - public List next(String queryId, ProxiedUserDetails currentUser) throws QueryException { + /** + * Gets the next page of results from the query object. If the object is no longer alive, meaning that the current session has expired, then this fail. The + * response object type is dynamic, see the listQueryLogic operation to determine what the response type object will be. + * + * @param queryId + * @param currentUser + * + * @return + * @throws QueryException + */ + public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) throws QueryException { UUID queryUUID = UUID.fromString(queryId); try { // make sure the query is valid, and the user can act on it validateRequest(queryId, currentUser); - boolean decrementNext = false; + // before we spin up a separate thread, make sure we are allowed to call next + QueryStatus queryStatus = incrementConcurrentNextCount(queryUUID); + + // TODO: Publish a next notification? + + // get the query logic + String queryLogicName = queryStatus.getQuery().getQueryLogicName(); + QueryLogic queryLogic = queryLogicFactory.getQueryLogic(queryStatus.getQuery().getQueryLogicName(), currentUser.getPrimaryUser().getRoles()); + + long pageNumber = -1L; + NextCall nextCall = new NextCall(queryProperties, queryQueueManager, queryStorageCache, queryMetricFactory, queryId, queryLogic, identifier); + nextCallMap.add(queryId, nextCall); try { - // before we spin up a separate thread, make sure we are allowed to call next - incrementConcurrentNextCount(queryUUID); - decrementNext = true; - - Future> future; - try { - future = nextExecutor.submit(() -> nextCall(queryId)); - } catch (TaskRejectedException e) { - throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Next task rejected by the executor for query " + queryId); - } - - // add this future to the next task map - nextTaskMap.add(queryId, future); + // submit the next call to the executor + nextCall.setFuture(nextCallExecutor.submit(nextCall)); // wait for the results to be ready - List results = future.get(); + ResultsPage resultsPage = nextCall.getFuture().get(); - // remote this future from the next task map - nextTaskMap.remove(queryId, future); - - return results; + // format the response + if (!resultsPage.getResults().isEmpty()) { + BaseQueryResponse response = queryLogic.getTransformer(queryStatus.getQuery()).createResponse(resultsPage); + + // after all of our work is done, perform our final query status update for this next call + pageNumber = updateStatusAndGetPageNumber(queryUUID, resultsPage.getResults().size()); + + response.setHasResults(true); + response.setPageNumber(pageNumber); + response.setLogicName(queryLogicName); + response.setQueryId(queryId); + return response; + } else { + throw new NoResultsQueryException(DatawaveErrorCode.NO_QUERY_RESULTS_FOUND, MessageFormat.format("{0}", queryId)); + } + } catch (TaskRejectedException e) { + throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Next task rejected by the executor for query " + queryId); } finally { - if (decrementNext) { + // remove this next call from the map, and decrement the next count for this query + nextCallMap.get(queryId).remove(nextCall); + + // update query status if we failed + if (pageNumber > 0) { decrementConcurrentNextCount(queryUUID); } } @@ -252,38 +283,58 @@ public List next(String queryId, ProxiedUserDetails currentUser) throws } } - private void incrementConcurrentNextCount(UUID queryUUID) throws InterruptedException, QueryException { - if (queryStorageCache.getQueryStatusLock(queryUUID).tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { + private QueryStatus incrementConcurrentNextCount(UUID queryUUID) throws InterruptedException, QueryException { + QueryStatus queryStatus; + QueryStorageLock statusLock = queryStorageCache.getQueryStatusLock(queryUUID); + if (statusLock.tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { try { // increment the concurrent next - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryUUID); - if (queryStatus != null && queryStatus.getConcurrentNextCount() < queryProperties.getConcurrentNextLimit()) { - queryStatus.setConcurrentNextCount(queryStatus.getConcurrentNextCount() + 1); - queryStorageCache.updateQueryStatus(queryStatus); + queryStatus = queryStorageCache.getQueryStatus(queryUUID); + if (queryStatus != null) { + if (queryStatus.getConcurrentNextCount() < queryProperties.getConcurrentNextLimit()) { + queryStatus.setConcurrentNextCount(queryStatus.getConcurrentNextCount() + 1); + + // update the last used datetime for the query + queryStatus.setLastUsed(new Date()); + + queryStorageCache.updateQueryStatus(queryStatus); + } else { + throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, + "Concurrent next call limit reached: " + queryProperties.getConcurrentNextLimit()); + } } else { - throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, - "Concurrent next call limit reached: " + queryProperties.getConcurrentNextLimit()); + throw new QueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH, "Unable to find query status in cache."); } } finally { - queryStorageCache.getQueryStatusLock(queryUUID).unlock(); + statusLock.unlock(); } } else { throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to acquire lock on query."); } + return queryStatus; } - private void decrementConcurrentNextCount(UUID queryUUID) throws InterruptedException, QueryException { + private QueryStatus decrementConcurrentNextCount(UUID queryUUID) throws InterruptedException, QueryException { + QueryStatus queryStatus; QueryStorageLock statusLock = queryStorageCache.getQueryStatusLock(queryUUID); if (statusLock.tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { try { // increment the concurrent next - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryUUID); - if (queryStatus != null && queryStatus.getConcurrentNextCount() < queryProperties.getConcurrentNextLimit()) { - queryStatus.setConcurrentNextCount(queryStatus.getConcurrentNextCount() - 1); - queryStorageCache.updateQueryStatus(queryStatus); + queryStatus = queryStorageCache.getQueryStatus(queryUUID); + if (queryStatus != null) { + if (queryStatus.getConcurrentNextCount() > 0) { + queryStatus.setConcurrentNextCount(queryStatus.getConcurrentNextCount() - 1); + + // update the last used datetime for the query + queryStatus.setLastUsed(new Date()); + + queryStorageCache.updateQueryStatus(queryStatus); + } else { + throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, + "Concurrent next count can't be decremented: " + queryStatus.getConcurrentNextCount()); + } } else { - throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, - "Concurrent next call limit reached: " + queryProperties.getConcurrentNextLimit()); + throw new QueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH, "Unable to find query status in cache."); } } finally { statusLock.unlock(); @@ -291,39 +342,63 @@ private void decrementConcurrentNextCount(UUID queryUUID) throws InterruptedExce } else { throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to acquire lock on query."); } + return queryStatus; } - private List nextCall(String queryId) throws Exception { - List resultList = new ArrayList<>(); - QueryQueueListener resultListener = queryQueueManager.createListener(identifier, queryId); - - // keep waiting for results until we're finished - while (!isFinished(queryId)) { - Object[] results = resultListener.receive(queryProperties.getResultQueueIntervalMillis()).getPayload().getPayload(); - if (results != null) { - resultList.addAll(Arrays.asList(results)); + private long updateStatusAndGetPageNumber(UUID queryUUID, int numResults) throws InterruptedException, QueryException { + long pageNumber; + QueryStorageLock statusLock = queryStorageCache.getQueryStatusLock(queryUUID); + if (statusLock.tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { + try { + // increment the concurrent next + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryUUID); + if (queryStatus != null) { + // decrement the concurrent next count + if (queryStatus.getConcurrentNextCount() > 0) { + queryStatus.setConcurrentNextCount(queryStatus.getConcurrentNextCount() - 1); + + // update the last used datetime for the query + queryStatus.setLastUsed(new Date()); + + queryStorageCache.updateQueryStatus(queryStatus); + } else { + // TODO: Exception, or error? + throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, + "Concurrent next count can't be decremented: " + queryStatus.getConcurrentNextCount()); + } + + // get and increment the page number + queryStatus.setLastPageNumber(queryStatus.getLastPageNumber() + 1); + pageNumber = queryStatus.getLastPageNumber(); + + // update the number of results returned + queryStatus.setNumResultsReturned(queryStatus.getNumResultsReturned() + numResults); + + // update the last used datetime for the query + queryStatus.setLastUsed(new Date()); + + queryStorageCache.updateQueryStatus(queryStatus); + } else { + throw new QueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH, "Unable to find query status in cache."); + } + } finally { + statusLock.unlock(); } + } else { + throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to acquire lock on query."); } - - return resultList; - } - - private boolean isFinished(String queryId) { - // get the query stats from the cache - // TODO: It may be more efficient to broadcast a canceled query to all query services vs. hitting the cache each time - QueryStatus queryStatus = queryStorageCache.getQueryStatus(UUID.fromString(queryId)); - - // conditions we need to check for - // 1) was this query canceled? - // 2) have we hit the user's results-per-page limit? - // 3) have we hit the query logic's results-per-page limit? - // 4) have we hit the query logic's bytes-per-page limit? - // 5) have we hit the max results (or the max results override)? - // 6) have we reached the "max work" limit? (i.e. next count + seek count) - // 7) are we going to timeout before getting a full page? if so, return partial results - return false; + return pageNumber; } + /** + * Releases the resources associated with this query. Any currently running calls to 'next' on the query will be stopped. Calls to 'next' after a 'cancel' + * will start over at page 1. + * + * @param queryId + * @param currentUser + * + * @throws QueryException + */ public void cancel(String queryId, ProxiedUserDetails currentUser) throws QueryException { try { // make sure the query is valid, and the user can act on it @@ -340,15 +415,24 @@ public void cancel(String queryId, ProxiedUserDetails currentUser) throws QueryE } } + /** + * + * Once the cancel request is validated, this method is called to actually cancel the query. + * + * @param queryId + * The id of the query to cancel + * @param publishEvent + * If true, the cancel request will be published on the bus + * @throws InterruptedException + * @throws QueryException + */ private void cancel(String queryId, boolean publishEvent) throws InterruptedException, QueryException { UUID queryUUID = UUID.fromString(queryId); // if we have an active next call for this query locally, cancel it - List>> futures = nextTaskMap.get(queryId); - if (futures != null) { - for (Future> future : futures) { - future.cancel(true); - } + List nextCalls = nextCallMap.get(queryId); + if (nextCalls != null) { + nextCalls.forEach(NextCall::cancel); // TODO: lock the cache entry and change state to canceled // try to be nice and acquire the lock before updating the status @@ -380,6 +464,15 @@ private void cancel(String queryId, boolean publishEvent) throws InterruptedExce } } + /** + * Releases the resources associated with this query. Any currently running calls to 'next' on the query will continue until they finish. Calls to 'next' + * after a 'close' will start over at page 1. + * + * @param queryId + * @param currentUser + * + * @throws QueryException + */ public void close(String queryId, ProxiedUserDetails currentUser) throws QueryException { try { // make sure the query is valid, and the user can act on it @@ -396,6 +489,17 @@ public void close(String queryId, ProxiedUserDetails currentUser) throws QueryEx } } + /** + * + * Once the close request is validated, this method is called to actually close the query. + * + * @param queryId + * The id of the query to close + * @param publishEvent + * If true, the close request will be published on the bus + * @throws InterruptedException + * @throws QueryException + */ private void close(String queryId, boolean publishEvent) throws InterruptedException, QueryException { UUID queryUUID = UUID.fromString(queryId); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java index 8060a316..cc0a3dfb 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java @@ -3,6 +3,8 @@ import datawave.marking.MarkingFunctions; import datawave.microservice.query.DefaultQueryParameters; import datawave.microservice.query.QueryParameters; +import datawave.webservice.query.cache.QueryMetricFactory; +import datawave.webservice.query.cache.QueryMetricFactoryImpl; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.cloud.context.config.annotation.RefreshScope; @@ -35,9 +37,15 @@ public QueryProperties queryProperties() { return new QueryProperties(); } + @Bean + @ConditionalOnMissingBean(type = "QueryMetricFactory") + public QueryMetricFactory queryMetricFactory() { + return new QueryMetricFactoryImpl(); + } + @RefreshScope @Bean - public ThreadPoolTaskExecutor nextExecutor(QueryProperties queryProperties) { + public ThreadPoolTaskExecutor nextCallExecutor(QueryProperties queryProperties) { QueryProperties.ExecutorProperties executorProperties = queryProperties.getNextExecutor(); ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); executor.setCorePoolSize(executorProperties.getCorePoolSize()); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java new file mode 100644 index 00000000..8ffa2ccf --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -0,0 +1,234 @@ +package datawave.microservice.query.runner; + +import datawave.microservice.common.storage.QueryQueueListener; +import datawave.microservice.common.storage.QueryQueueManager; +import datawave.microservice.common.storage.QueryStatus; +import datawave.microservice.common.storage.QueryStorageCache; +import datawave.microservice.common.storage.Result; +import datawave.microservice.query.config.QueryExpirationProperties; +import datawave.microservice.query.config.QueryProperties; +import datawave.microservice.query.logic.QueryLogic; +import datawave.webservice.query.cache.QueryMetricFactory; +import datawave.webservice.query.cache.ResultsPage; +import datawave.webservice.query.data.ObjectSizeOf; +import datawave.webservice.query.exception.QueryException; +import datawave.webservice.query.metric.BaseQueryMetric; +import datawave.webservice.query.metric.QueryMetric; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.messaging.Message; + +import java.util.LinkedList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.Future; + +public class NextCall implements Callable> { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + + private final QueryProperties queryProperties; + private final QueryQueueManager queryQueueManager; + private final QueryStorageCache queryStorageCache; + private final String queryId; + private final String identifier; + + private volatile boolean canceled = false; + private volatile Future> future = null; + + private final int userResultsPerPage; + private final boolean maxResultsOverridden; + private final long maxResultsOverride; + private final long maxResults; + private final int logicResultsPerPage; + private final long logicBytesPerPage; + private final long logicMaxWork; + private final long maxResultsPerPage; + + private final List results = new LinkedList<>(); + private long pageSizeBytes; + private long startTimeMillis; + private ResultsPage.Status status = ResultsPage.Status.COMPLETE; + + private final BaseQueryMetric metric; + + public NextCall(QueryProperties queryProperties, QueryQueueManager queryQueueManager, QueryStorageCache queryStorageCache, + QueryMetricFactory queryMetricFactory, String queryId, QueryLogic queryLogic, String identifier) throws QueryException { + this.queryProperties = queryProperties; + this.queryQueueManager = queryQueueManager; + this.queryStorageCache = queryStorageCache; + this.queryId = queryId; + this.identifier = identifier; + + QueryStatus status = getQueryStatus(); + this.userResultsPerPage = status.getQuery().getPagesize(); + this.maxResultsOverridden = status.getQuery().isMaxResultsOverridden(); + this.maxResultsOverride = status.getQuery().getMaxResultsOverride(); + + this.logicResultsPerPage = queryLogic.getMaxPageSize(); + this.logicBytesPerPage = queryLogic.getPageByteTrigger(); + this.logicMaxWork = queryLogic.getMaxWork(); + + this.maxResultsPerPage = Math.min(userResultsPerPage, logicResultsPerPage); + + this.maxResults = queryLogic.getResultLimit(status.getQuery().getDnList()); + if (this.maxResults != queryLogic.getMaxResults()) { + log.info("Maximum results set to " + this.maxResults + " instead of default " + queryLogic.getMaxResults() + ", user " + + status.getQuery().getUserDN() + " has a DN configured with a different limit"); + } + + this.metric = queryMetricFactory.createMetric(); + } + + @Override + public ResultsPage call() throws Exception { + startTimeMillis = System.currentTimeMillis(); + + QueryQueueListener resultListener = queryQueueManager.createListener(identifier, queryId); + + // keep waiting for results until we're finished + while (!isFinished(queryId)) { + Message message = resultListener.receive(queryProperties.getResultPollRateMillis()); + if (message != null) { + + // TODO: In the past, if we got a null result we would mark the next call as finished + // Should we continue to do that, or something else? + Object result = message.getPayload().getPayload(); + if (result != null) { + results.add(result); + + if (logicBytesPerPage > 0) { + pageSizeBytes += ObjectSizeOf.Sizer.getObjectSize(result); + } + } else { + log.debug("Null result encountered, no more results"); + break; + } + } + } + + return new ResultsPage<>(results, status); + } + + private boolean isFinished(String queryId) { + boolean finished = false; + + // 1) was this query canceled? + if (canceled) { + log.info("Query [{}]: query cancelled, aborting next call", queryId); + finished = true; + } + + // 2) have we hit the user's results-per-page limit? + if (!finished && results.size() >= userResultsPerPage) { + log.info("Query [{}]: user requested max page size has been reached, aborting next call", queryId); + finished = true; + } + + // 3) have we hit the query logic's results-per-page limit? + if (!finished && results.size() >= logicResultsPerPage) { + log.info("Query [{}]: query logic max page size has been reached, aborting next call", queryId); + finished = true; + } + + // 4) have we hit the query logic's bytes-per-page limit? + if (!finished && pageSizeBytes >= logicBytesPerPage) { + log.info("Query [{}]: query logic max page byte size has been reached, aborting next call", queryId); + finished = true; + status = ResultsPage.Status.PARTIAL; + } + + // 5) have we hit the max results (or the max results override)? + if (!finished) { + long numResultsReturned = getQueryStatus().getNumResultsReturned(); + long numResults = numResultsReturned + results.size(); + if (this.maxResultsOverridden) { + if (maxResultsOverride >= 0 && numResults >= maxResultsOverride) { + log.info("Query [{}]: max results override has been reached, aborting next call", queryId); + // TODO: Figure out query metrics + metric.setLifecycle(QueryMetric.Lifecycle.MAXRESULTS); + finished = true; + } + } else if (maxResults >= 0 && numResults >= maxResults) { + log.info("Query [{}]: logic max results has been reached, aborting next call", queryId); + // TODO: Figure out query metrics + metric.setLifecycle(QueryMetric.Lifecycle.MAXRESULTS); + finished = true; + } + } + + // TODO: Do I need to pull query metrics to get the next/seek count? + // This used to come from the query logic transform iterator + // 6) have we reached the "max work" limit? (i.e. next count + seek count) + if (!finished && logicMaxWork >= 0 && (metric.getNextCount() + metric.getSeekCount()) >= logicMaxWork) { + log.info("Query [{}]: logic max work has been reached, aborting next call", queryId); + // TODO: Figure out query metrics + metric.setLifecycle(BaseQueryMetric.Lifecycle.MAXWORK); + finished = true; + } + + // 7) are we going to timeout before getting a full page? if so, return partial results + if (!finished && timeout()) { + log.info("Query [{}]: logic max expire before page is full, returning existing results: {} of {} results in {}ms", queryId, results.size(), + maxResultsPerPage, callTimeMillis()); + finished = true; + status = ResultsPage.Status.PARTIAL; + } + + return finished; + } + + protected boolean timeout() { + boolean timeout = false; + + // only return prematurely if we have at least 1 result + if (!results.isEmpty()) { + long callTimeMillis = callTimeMillis(); + + QueryExpirationProperties expiration = queryProperties.getExpiration(); + + // if after the page size short circuit check time + if (callTimeMillis >= expiration.getShortCircuitCheckTimeMillis()) { + float percentTimeComplete = (float) callTimeMillis / (float) (expiration.getCallTimeout()); + float percentResultsComplete = (float) results.size() / (float) maxResultsPerPage; + // if the percent results complete is less than the percent time complete, then break out + if (percentResultsComplete < percentTimeComplete) { + timeout = true; + } + } + + // if after the page short circuit timeout, then break out + if (callTimeMillis >= expiration.getShortCircuitTimeoutMillis()) { + timeout = true; + } + } + + return timeout; + } + + private long callTimeMillis() { + return System.currentTimeMillis() - startTimeMillis; + } + + // TODO: We may want to control the rate that we pull query status from the cache + // Perhaps by adding update interval properties. + private QueryStatus getQueryStatus() { + return queryStorageCache.getQueryStatus(UUID.fromString(queryId)); + } + + public void cancel() { + this.canceled = true; + } + + public boolean isCanceled() { + return canceled; + } + + public Future> getFuture() { + return future; + } + + public void setFuture(Future> future) { + this.future = future; + } +} From 73db631741c904c1dfe77499a201c2bd7cff63fa Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 28 May 2021 12:08:23 -0400 Subject: [PATCH 055/218] Updated next call to use a builder, and to control cache hit rate via a configuration property. Also, split out properties classes and added validation where applicable. --- .../config/DefaultParameterProperties.java | 31 ++++ .../query/config/NextCallProperties.java | 80 +++++++++ .../config/QueryExpirationProperties.java | 13 ++ .../query/config/QueryLogicProperties.java | 19 -- .../query/config/QueryProperties.java | 166 +++++------------- .../ThreadPoolTaskExecutorProperties.java | 62 +++++++ .../query/QueryManagementService.java | 17 +- .../query/config/QueryServiceConfig.java | 2 +- .../microservice/query/runner/NextCall.java | 118 ++++++++++--- 9 files changed, 339 insertions(+), 169 deletions(-) create mode 100644 query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/DefaultParameterProperties.java create mode 100644 query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/NextCallProperties.java delete mode 100644 query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryLogicProperties.java create mode 100644 query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/ThreadPoolTaskExecutorProperties.java diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/DefaultParameterProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/DefaultParameterProperties.java new file mode 100644 index 00000000..4516ae8b --- /dev/null +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/DefaultParameterProperties.java @@ -0,0 +1,31 @@ +package datawave.microservice.query.config; + +import org.springframework.validation.annotation.Validated; + +import javax.annotation.Nonnegative; +import javax.validation.constraints.NotEmpty; + +@Validated +public class DefaultParameterProperties { + @NotEmpty + private String pool = "unassigned"; + @Nonnegative + private int maxConcurrentTasks = 10; + + public String getPool() { + return pool; + } + + public void setPool(String pool) { + this.pool = pool; + } + + public int getMaxConcurrentTasks() { + return maxConcurrentTasks; + } + + public void setMaxConcurrentTasks(int maxConcurrentTasks) { + this.maxConcurrentTasks = maxConcurrentTasks; + } + +} diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/NextCallProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/NextCallProperties.java new file mode 100644 index 00000000..52deda82 --- /dev/null +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/NextCallProperties.java @@ -0,0 +1,80 @@ +package datawave.microservice.query.config; + +import org.springframework.validation.annotation.Validated; + +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Positive; +import javax.validation.constraints.PositiveOrZero; +import java.util.concurrent.TimeUnit; + +@Validated +public class NextCallProperties { + @PositiveOrZero + private long resultPollInterval = TimeUnit.SECONDS.toMillis(30); + @NotNull + private TimeUnit resultPollTimeUnit = TimeUnit.MILLISECONDS; + @Positive + private int concurrency = 1; + @PositiveOrZero + private long statusUpdateInterval = TimeUnit.SECONDS.toMillis(30); + @NotNull + private TimeUnit statusUpdateTimeUnit = TimeUnit.MILLISECONDS; + + private ThreadPoolTaskExecutorProperties executor = new ThreadPoolTaskExecutorProperties(10, 100, 100, "nextCall-"); + + public long getResultPollInterval() { + return resultPollInterval; + } + + public long getResultPollIntervalMillis() { + return resultPollTimeUnit.toMillis(resultPollInterval); + } + + public void setResultPollInterval(long resultPollInterval) { + this.resultPollInterval = resultPollInterval; + } + + public TimeUnit getResultPollTimeUnit() { + return resultPollTimeUnit; + } + + public void setResultPollTimeUnit(TimeUnit resultPollTimeUnit) { + this.resultPollTimeUnit = resultPollTimeUnit; + } + + public int getConcurrency() { + return concurrency; + } + + public void setConcurrency(int concurrency) { + this.concurrency = concurrency; + } + + public long getStatusUpdateInterval() { + return statusUpdateInterval; + } + + public long getStatusUpdateIntervalMillis() { + return statusUpdateTimeUnit.toMillis(statusUpdateInterval); + } + + public void setStatusUpdateInterval(long statusUpdateInterval) { + this.statusUpdateInterval = statusUpdateInterval; + } + + public TimeUnit getStatusUpdateTimeUnit() { + return statusUpdateTimeUnit; + } + + public void setStatusUpdateTimeUnit(TimeUnit statusUpdateTimeUnit) { + this.statusUpdateTimeUnit = statusUpdateTimeUnit; + } + + public ThreadPoolTaskExecutorProperties getExecutor() { + return executor; + } + + public void setExecutor(ThreadPoolTaskExecutorProperties executor) { + this.executor = executor; + } +} diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java index 9dcefa04..8ed9bf8d 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java @@ -1,15 +1,28 @@ package datawave.microservice.query.config; +import org.springframework.validation.annotation.Validated; + +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Positive; import java.util.concurrent.TimeUnit; +@Validated public class QueryExpirationProperties { + @Positive private long idleTimeout = 15; + @NotNull private TimeUnit idleTimeUnit = TimeUnit.MINUTES; + @Positive private long callTimeout = 60; + @NotNull private TimeUnit callTimeUnit = TimeUnit.MINUTES; + @Positive private long shortCircuitCheckTime = callTimeout / 2; + @NotNull private TimeUnit shortCircuitCheckTimeUnit = TimeUnit.MINUTES; + @Positive private long shortCircuitTimeout = Math.round(0.97 * callTimeout); + @NotNull private TimeUnit shortCircuitTimeUnit = TimeUnit.MINUTES; public long getIdleTimeout() { diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryLogicProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryLogicProperties.java deleted file mode 100644 index 4fdaae87..00000000 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryLogicProperties.java +++ /dev/null @@ -1,19 +0,0 @@ -package datawave.microservice.query.config; - -import org.springframework.validation.annotation.Validated; - -import javax.validation.Valid; - -@Validated -public class QueryLogicProperties { - @Valid - private boolean metricsEnabled = false; - - public boolean isMetricsEnabled() { - return metricsEnabled; - } - - public void setMetricsEnabled(boolean metricsEnabled) { - this.metricsEnabled = metricsEnabled; - } -} diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java index 65fff008..d73919a5 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java @@ -2,42 +2,31 @@ import org.springframework.validation.annotation.Validated; -import javax.annotation.Nonnegative; -import javax.validation.Valid; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; -import javax.validation.constraints.Positive; import javax.validation.constraints.PositiveOrZero; import java.util.concurrent.TimeUnit; @Validated public class QueryProperties { - - @Valid - private QueryExpirationProperties expiration; @NotEmpty private String privilegedRole = "PrivilegedUser"; - @Positive - private long resultPollRate = TimeUnit.SECONDS.toMillis(30); - private TimeUnit resultPollRateTimeUnit = TimeUnit.MILLISECONDS; // The amount of time to wait for the lock to be acquired @PositiveOrZero - private long lockWaitTimeMillis = TimeUnit.SECONDS.toMillis(5); + private long lockWaitTime = TimeUnit.SECONDS.toMillis(5); + @NotNull + private TimeUnit lockWaitTimeUnit = TimeUnit.MILLISECONDS; // The amount of time that the lock will be held before being automatically released @PositiveOrZero - private long lockLeaseTimeMillis = TimeUnit.SECONDS.toMillis(30); + private long lockLeaseTime = TimeUnit.SECONDS.toMillis(30); + @NotNull + private TimeUnit lockLeaseTimeUnit = TimeUnit.MILLISECONDS; + @NotEmpty private String executorServiceName = "executor"; - private int concurrentNextLimit = 1; - private ExecutorProperties nextExecutor = new ExecutorProperties(); - private DefaultParameters defaultParams = new DefaultParameters(); - - public QueryExpirationProperties getExpiration() { - return expiration; - } - public void setExpiration(QueryExpirationProperties expiration) { - this.expiration = expiration; - } + private QueryExpirationProperties expiration; + private NextCallProperties nextCall = new NextCallProperties(); + private DefaultParameterProperties defaultParams = new DefaultParameterProperties(); public String getPrivilegedRole() { return privilegedRole; @@ -47,40 +36,46 @@ public void setPrivilegedRole(String privilegedRole) { this.privilegedRole = privilegedRole; } - public long getResultPollRate() { - return resultPollRate; + public long getLockWaitTime() { + return lockWaitTime; } - public void setResultPollRate(long resultPollRate) { - this.resultPollRate = resultPollRate; + public long getLockWaitTimeMillis() { + return lockWaitTimeUnit.toMillis(lockWaitTime); } - public TimeUnit getResultPollRateTimeUnit() { - return resultPollRateTimeUnit; + public void setLockWaitTime(long lockWaitTime) { + this.lockWaitTime = lockWaitTime; } - public void setResultPollRateTimeUnit(TimeUnit resultPollRateTimeUnit) { - this.resultPollRateTimeUnit = resultPollRateTimeUnit; + public TimeUnit getLockWaitTimeUnit() { + return lockWaitTimeUnit; } - public long getResultPollRateMillis() { - return resultPollRateTimeUnit.toMillis(resultPollRate); + public QueryProperties setLockWaitTimeUnit(TimeUnit lockWaitTimeUnit) { + this.lockWaitTimeUnit = lockWaitTimeUnit; + return this; } - public long getLockWaitTimeMillis() { - return lockWaitTimeMillis; + public long getLockLeaseTime() { + return lockLeaseTime; } - public void setLockWaitTimeMillis(long lockWaitTimeMillis) { - this.lockWaitTimeMillis = lockWaitTimeMillis; + public long getLockLeaseTimeMillis() { + return lockLeaseTimeUnit.toMillis(lockLeaseTime); } - public long getLockLeaseTimeMillis() { - return lockLeaseTimeMillis; + public void setLockLeaseTime(long lockLeaseTime) { + this.lockLeaseTime = lockLeaseTime; } - public void setLockLeaseTimeMillis(long lockLeaseTimeMillis) { - this.lockLeaseTimeMillis = lockLeaseTimeMillis; + public TimeUnit getLockLeaseTimeUnit() { + return lockLeaseTimeUnit; + } + + public QueryProperties setLockLeaseTimeUnit(TimeUnit lockLeaseTimeUnit) { + this.lockLeaseTimeUnit = lockLeaseTimeUnit; + return this; } public String getExecutorServiceName() { @@ -91,100 +86,27 @@ public void setExecutorServiceName(String executorServiceName) { this.executorServiceName = executorServiceName; } - public int getConcurrentNextLimit() { - return concurrentNextLimit; + public QueryExpirationProperties getExpiration() { + return expiration; } - public void setConcurrentNextLimit(int concurrentNextLimit) { - this.concurrentNextLimit = concurrentNextLimit; + public void setExpiration(QueryExpirationProperties expiration) { + this.expiration = expiration; } - public ExecutorProperties getNextExecutor() { - return nextExecutor; + public NextCallProperties getNextCall() { + return nextCall; } - public void setNextExecutor(ExecutorProperties nextExecutor) { - this.nextExecutor = nextExecutor; + public void setNextCall(NextCallProperties nextCall) { + this.nextCall = nextCall; } - public DefaultParameters getDefaultParams() { + public DefaultParameterProperties getDefaultParams() { return defaultParams; } - public void setDefaultParams(DefaultParameters defaultParams) { + public void setDefaultParams(DefaultParameterProperties defaultParams) { this.defaultParams = defaultParams; } - - @Validated - public static class DefaultParameters { - - @NotEmpty - private String pool = "unassigned"; - - @Nonnegative - private int maxConcurrentTasks = 10; - - public String getPool() { - return pool; - } - - public void setPool(String pool) { - this.pool = pool; - } - - public int getMaxConcurrentTasks() { - return maxConcurrentTasks; - } - - public void setMaxConcurrentTasks(int maxConcurrentTasks) { - this.maxConcurrentTasks = maxConcurrentTasks; - } - } - - @Validated - public static class ExecutorProperties { - @PositiveOrZero - private int corePoolSize = 0; - - @Positive - private int maxPoolSize = 5; - - @PositiveOrZero - private int queueCapacity = 0; - - @NotNull - private String threadNamePrefix = "replayTask-"; - - public int getCorePoolSize() { - return corePoolSize; - } - - public void setCorePoolSize(int corePoolSize) { - this.corePoolSize = corePoolSize; - } - - public int getMaxPoolSize() { - return maxPoolSize; - } - - public void setMaxPoolSize(int maxPoolSize) { - this.maxPoolSize = maxPoolSize; - } - - public int getQueueCapacity() { - return queueCapacity; - } - - public void setQueueCapacity(int queueCapacity) { - this.queueCapacity = queueCapacity; - } - - public String getThreadNamePrefix() { - return threadNamePrefix; - } - - public void setThreadNamePrefix(String threadNamePrefix) { - this.threadNamePrefix = threadNamePrefix; - } - } } diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/ThreadPoolTaskExecutorProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/ThreadPoolTaskExecutorProperties.java new file mode 100644 index 00000000..23b4aa74 --- /dev/null +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/ThreadPoolTaskExecutorProperties.java @@ -0,0 +1,62 @@ +package datawave.microservice.query.config; + +import org.springframework.validation.annotation.Validated; + +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Positive; +import javax.validation.constraints.PositiveOrZero; + +@Validated +public class ThreadPoolTaskExecutorProperties { + @PositiveOrZero + private int corePoolSize = 0; + @Positive + private int maxPoolSize = 5; + @PositiveOrZero + private int queueCapacity = 0; + @NotNull + private String threadNamePrefix = ""; + + public ThreadPoolTaskExecutorProperties() { + + } + + public ThreadPoolTaskExecutorProperties(int corePoolSize, int maxPoolSize, int queueCapacity, String threadNamePrefix) { + this.corePoolSize = corePoolSize; + this.maxPoolSize = maxPoolSize; + this.queueCapacity = queueCapacity; + this.threadNamePrefix = threadNamePrefix; + } + + public int getCorePoolSize() { + return corePoolSize; + } + + public void setCorePoolSize(int corePoolSize) { + this.corePoolSize = corePoolSize; + } + + public int getMaxPoolSize() { + return maxPoolSize; + } + + public void setMaxPoolSize(int maxPoolSize) { + this.maxPoolSize = maxPoolSize; + } + + public int getQueueCapacity() { + return queueCapacity; + } + + public void setQueueCapacity(int queueCapacity) { + this.queueCapacity = queueCapacity; + } + + public String getThreadNamePrefix() { + return threadNamePrefix; + } + + public void setThreadNamePrefix(String threadNamePrefix) { + this.threadNamePrefix = threadNamePrefix; + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index e504f53c..291b30fc 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -239,8 +239,19 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th String queryLogicName = queryStatus.getQuery().getQueryLogicName(); QueryLogic queryLogic = queryLogicFactory.getQueryLogic(queryStatus.getQuery().getQueryLogicName(), currentUser.getPrimaryUser().getRoles()); + // @formatter:off + NextCall nextCall = new NextCall.Builder() + .setQueryProperties(queryProperties) + .setQueryQueueManager(queryQueueManager) + .setQueryStorageCache(queryStorageCache) + .setQueryMetricFactory(queryMetricFactory) + .setQueryId(queryId) + .setQueryLogic(queryLogic) + .setIdentifier(identifier) + .build(); + // @formatter:on + long pageNumber = -1L; - NextCall nextCall = new NextCall(queryProperties, queryQueueManager, queryStorageCache, queryMetricFactory, queryId, queryLogic, identifier); nextCallMap.add(queryId, nextCall); try { // submit the next call to the executor @@ -291,7 +302,7 @@ private QueryStatus incrementConcurrentNextCount(UUID queryUUID) throws Interrup // increment the concurrent next queryStatus = queryStorageCache.getQueryStatus(queryUUID); if (queryStatus != null) { - if (queryStatus.getConcurrentNextCount() < queryProperties.getConcurrentNextLimit()) { + if (queryStatus.getConcurrentNextCount() < queryProperties.getNextCall().getConcurrency()) { queryStatus.setConcurrentNextCount(queryStatus.getConcurrentNextCount() + 1); // update the last used datetime for the query @@ -300,7 +311,7 @@ private QueryStatus incrementConcurrentNextCount(UUID queryUUID) throws Interrup queryStorageCache.updateQueryStatus(queryStatus); } else { throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, - "Concurrent next call limit reached: " + queryProperties.getConcurrentNextLimit()); + "Concurrent next call limit reached: " + queryProperties.getNextCall().getConcurrency()); } } else { throw new QueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH, "Unable to find query status in cache."); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java index cc0a3dfb..777bb989 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java @@ -46,7 +46,7 @@ public QueryMetricFactory queryMetricFactory() { @RefreshScope @Bean public ThreadPoolTaskExecutor nextCallExecutor(QueryProperties queryProperties) { - QueryProperties.ExecutorProperties executorProperties = queryProperties.getNextExecutor(); + ThreadPoolTaskExecutorProperties executorProperties = queryProperties.getNextCall().getExecutor(); ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); executor.setCorePoolSize(executorProperties.getCorePoolSize()); executor.setMaxPoolSize(executorProperties.getMaxPoolSize()); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 8ffa2ccf..ed4f9330 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -5,6 +5,7 @@ import datawave.microservice.common.storage.QueryStatus; import datawave.microservice.common.storage.QueryStorageCache; import datawave.microservice.common.storage.Result; +import datawave.microservice.query.config.NextCallProperties; import datawave.microservice.query.config.QueryExpirationProperties; import datawave.microservice.query.config.QueryProperties; import datawave.microservice.query.logic.QueryLogic; @@ -27,7 +28,8 @@ public class NextCall implements Callable> { private final Logger log = LoggerFactory.getLogger(this.getClass()); - private final QueryProperties queryProperties; + private final NextCallProperties nextCallProperties; + private final QueryExpirationProperties expirationProperties; private final QueryQueueManager queryQueueManager; private final QueryStorageCache queryStorageCache; private final String queryId; @@ -50,34 +52,37 @@ public class NextCall implements Callable> { private long startTimeMillis; private ResultsPage.Status status = ResultsPage.Status.COMPLETE; + private long lastStatusUpdateTime = 0L; + private QueryStatus queryStatus; + private final BaseQueryMetric metric; - public NextCall(QueryProperties queryProperties, QueryQueueManager queryQueueManager, QueryStorageCache queryStorageCache, - QueryMetricFactory queryMetricFactory, String queryId, QueryLogic queryLogic, String identifier) throws QueryException { - this.queryProperties = queryProperties; - this.queryQueueManager = queryQueueManager; - this.queryStorageCache = queryStorageCache; - this.queryId = queryId; - this.identifier = identifier; + private NextCall(Builder builder) throws QueryException { + this.nextCallProperties = builder.nextCallProperties; + this.expirationProperties = builder.expirationProperties; + this.queryQueueManager = builder.queryQueueManager; + this.queryStorageCache = builder.queryStorageCache; + this.queryId = builder.queryId; + this.identifier = builder.identifier; QueryStatus status = getQueryStatus(); this.userResultsPerPage = status.getQuery().getPagesize(); this.maxResultsOverridden = status.getQuery().isMaxResultsOverridden(); this.maxResultsOverride = status.getQuery().getMaxResultsOverride(); - this.logicResultsPerPage = queryLogic.getMaxPageSize(); - this.logicBytesPerPage = queryLogic.getPageByteTrigger(); - this.logicMaxWork = queryLogic.getMaxWork(); + this.logicResultsPerPage = builder.queryLogic.getMaxPageSize(); + this.logicBytesPerPage = builder.queryLogic.getPageByteTrigger(); + this.logicMaxWork = builder.queryLogic.getMaxWork(); this.maxResultsPerPage = Math.min(userResultsPerPage, logicResultsPerPage); - this.maxResults = queryLogic.getResultLimit(status.getQuery().getDnList()); - if (this.maxResults != queryLogic.getMaxResults()) { - log.info("Maximum results set to " + this.maxResults + " instead of default " + queryLogic.getMaxResults() + ", user " + this.maxResults = builder.queryLogic.getResultLimit(status.getQuery().getDnList()); + if (this.maxResults != builder.queryLogic.getMaxResults()) { + log.info("Maximum results set to " + this.maxResults + " instead of default " + builder.queryLogic.getMaxResults() + ", user " + status.getQuery().getUserDN() + " has a DN configured with a different limit"); } - this.metric = queryMetricFactory.createMetric(); + this.metric = builder.queryMetricFactory.createMetric(); } @Override @@ -88,7 +93,7 @@ public ResultsPage call() throws Exception { // keep waiting for results until we're finished while (!isFinished(queryId)) { - Message message = resultListener.receive(queryProperties.getResultPollRateMillis()); + Message message = resultListener.receive(nextCallProperties.getResultPollIntervalMillis()); if (message != null) { // TODO: In the past, if we got a null result we would mark the next call as finished @@ -185,11 +190,9 @@ protected boolean timeout() { if (!results.isEmpty()) { long callTimeMillis = callTimeMillis(); - QueryExpirationProperties expiration = queryProperties.getExpiration(); - // if after the page size short circuit check time - if (callTimeMillis >= expiration.getShortCircuitCheckTimeMillis()) { - float percentTimeComplete = (float) callTimeMillis / (float) (expiration.getCallTimeout()); + if (callTimeMillis >= expirationProperties.getShortCircuitCheckTimeMillis()) { + float percentTimeComplete = (float) callTimeMillis / (float) (expirationProperties.getCallTimeout()); float percentResultsComplete = (float) results.size() / (float) maxResultsPerPage; // if the percent results complete is less than the percent time complete, then break out if (percentResultsComplete < percentTimeComplete) { @@ -198,7 +201,7 @@ protected boolean timeout() { } // if after the page short circuit timeout, then break out - if (callTimeMillis >= expiration.getShortCircuitTimeoutMillis()) { + if (callTimeMillis >= expirationProperties.getShortCircuitTimeoutMillis()) { timeout = true; } } @@ -210,10 +213,16 @@ private long callTimeMillis() { return System.currentTimeMillis() - startTimeMillis; } - // TODO: We may want to control the rate that we pull query status from the cache - // Perhaps by adding update interval properties. private QueryStatus getQueryStatus() { - return queryStorageCache.getQueryStatus(UUID.fromString(queryId)); + if (queryStatus == null || isQueryStatusExpired()) { + lastStatusUpdateTime = System.currentTimeMillis(); + queryStatus = queryStorageCache.getQueryStatus(UUID.fromString(queryId)); + } + return queryStatus; + } + + private boolean isQueryStatusExpired() { + return (System.currentTimeMillis() - lastStatusUpdateTime) > nextCallProperties.getStatusUpdateIntervalMillis(); } public void cancel() { @@ -231,4 +240,65 @@ public Future> getFuture() { public void setFuture(Future> future) { this.future = future; } + + public static class Builder { + private NextCallProperties nextCallProperties; + private QueryExpirationProperties expirationProperties; + private QueryQueueManager queryQueueManager; + private QueryStorageCache queryStorageCache; + private QueryMetricFactory queryMetricFactory; + private String queryId; + private QueryLogic queryLogic; + private String identifier; + + public Builder setQueryProperties(QueryProperties queryProperties) { + this.nextCallProperties = queryProperties.getNextCall(); + this.expirationProperties = queryProperties.getExpiration(); + return this; + } + + public Builder setNextCallProperties(NextCallProperties nextCallProperties) { + this.nextCallProperties = nextCallProperties; + return this; + } + + public Builder setExpirationProperties(QueryExpirationProperties expirationProperties) { + this.expirationProperties = expirationProperties; + return this; + } + + public Builder setQueryQueueManager(QueryQueueManager queryQueueManager) { + this.queryQueueManager = queryQueueManager; + return this; + } + + public Builder setQueryStorageCache(QueryStorageCache queryStorageCache) { + this.queryStorageCache = queryStorageCache; + return this; + } + + public Builder setQueryMetricFactory(QueryMetricFactory queryMetricFactory) { + this.queryMetricFactory = queryMetricFactory; + return this; + } + + public Builder setQueryId(String queryId) { + this.queryId = queryId; + return this; + } + + public Builder setQueryLogic(QueryLogic queryLogic) { + this.queryLogic = queryLogic; + return this; + } + + public Builder setIdentifier(String identifier) { + this.identifier = identifier; + return this; + } + + public NextCall build() throws QueryException { + return new NextCall(this); + } + } } From ccdf63ccfd072eec80d0b0269efacba3b2fce2c0 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 28 May 2021 22:39:33 -0400 Subject: [PATCH 056/218] Stubbed out some query monitoring logic which can be used to periodically monitor the status of the system. --- .../query/config/QueryProperties.java | 3 +- .../query/QueryManagementService.java | 11 ++- .../query/monitor/MonitorTask.java | 52 ++++++++++ .../query/monitor/QueryMonitor.java | 67 +++++++++++++ .../query/monitor/cache/MonitorStatus.java | 13 +++ .../monitor/cache/MonitorStatusCache.java | 80 +++++++++++++++ .../query/monitor/config/MonitorConfig.java | 37 +++++++ .../monitor/config/MonitorProperties.java | 97 +++++++++++++++++++ 8 files changed, 358 insertions(+), 2 deletions(-) create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java index d73919a5..5aa67a8a 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java @@ -4,6 +4,7 @@ import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; +import javax.validation.constraints.Positive; import javax.validation.constraints.PositiveOrZero; import java.util.concurrent.TimeUnit; @@ -17,7 +18,7 @@ public class QueryProperties { @NotNull private TimeUnit lockWaitTimeUnit = TimeUnit.MILLISECONDS; // The amount of time that the lock will be held before being automatically released - @PositiveOrZero + @Positive private long lockLeaseTime = TimeUnit.SECONDS.toMillis(30); @NotNull private TimeUnit lockLeaseTimeUnit = TimeUnit.MILLISECONDS; diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 291b30fc..b1200f40 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -52,6 +52,7 @@ import java.util.Date; import java.util.List; import java.util.UUID; +import java.util.concurrent.ExecutionException; import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CANCELED; import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CLOSED; @@ -240,7 +241,7 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th QueryLogic queryLogic = queryLogicFactory.getQueryLogic(queryStatus.getQuery().getQueryLogicName(), currentUser.getPrimaryUser().getRoles()); // @formatter:off - NextCall nextCall = new NextCall.Builder() + final NextCall nextCall = new NextCall.Builder() .setQueryProperties(queryProperties) .setQueryQueueManager(queryQueueManager) .setQueryStorageCache(queryStorageCache) @@ -277,6 +278,14 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th } } catch (TaskRejectedException e) { throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Next task rejected by the executor for query " + queryId); + } catch (ExecutionException e) { + // try to unwrap the execution exception + Throwable cause = e.getCause(); + if (cause instanceof Exception) { + throw (Exception) e.getCause(); + } else { + throw e; + } } finally { // remove this next call from the map, and decrement the next count for this query nextCallMap.get(queryId).remove(nextCall); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java new file mode 100644 index 00000000..137e753f --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java @@ -0,0 +1,52 @@ +package datawave.microservice.query.monitor; + +import datawave.microservice.query.monitor.cache.MonitorStatus; +import datawave.microservice.query.monitor.cache.MonitorStatusCache; +import datawave.microservice.query.monitor.config.MonitorProperties; + +import java.util.concurrent.Callable; + +public class MonitorTask implements Callable { + private final MonitorProperties monitorProperties; + private final MonitorStatusCache monitorStatusCache; + + public MonitorTask(MonitorProperties monitorProperties, MonitorStatusCache monitorStatusCache) { + this.monitorProperties = monitorProperties; + this.monitorStatusCache = monitorStatusCache; + } + + @Override + public Void call() throws Exception { + if (tryLock()) { + boolean success = false; + MonitorStatus monitorStatus = null; + try { + monitorStatus = monitorStatusCache.getStatus(); + if (isMonitorIntervalExpired()) { + // TODO: perform idle checks, etc. + + success = true; + } + } finally { + if (success && monitorStatus != null) { + monitorStatus.setLastChecked(System.currentTimeMillis()); + } + unlock(); + } + } + return null; + } + + private boolean tryLock() throws InterruptedException { + return monitorStatusCache.tryLock(monitorProperties.getLockWaitTime(), monitorProperties.getLockWaitTimeUnit(), monitorProperties.getLockLeaseTime(), + monitorProperties.getLockLeaseTimeUnit()); + } + + private void unlock() { + monitorStatusCache.unlock(); + } + + private boolean isMonitorIntervalExpired() { + return (System.currentTimeMillis() - monitorStatusCache.getStatus().getLastCheckedMillis()) > monitorProperties.getMonitorIntervalMillis(); + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java new file mode 100644 index 00000000..62de49f3 --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java @@ -0,0 +1,67 @@ +package datawave.microservice.query.monitor; + +import datawave.microservice.query.monitor.cache.MonitorStatusCache; +import datawave.microservice.query.monitor.config.MonitorProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +@Component +@ConditionalOnProperty(name = "datawave.query.monitor.enabled", havingValue = "true", matchIfMissing = true) +public class QueryMonitor { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + + private final MonitorProperties monitorProperties; + private final MonitorStatusCache monitorStatusCache; + private final ExecutorService executor = Executors.newSingleThreadExecutor(); + + private long taskStartTime; + private Future taskFuture; + + public QueryMonitor(MonitorProperties monitorProperties, MonitorStatusCache monitorStatusCache) { + this.monitorProperties = monitorProperties; + this.monitorStatusCache = monitorStatusCache; + } + + // this runs in a separate thread every 30 seconds (by default) + @Scheduled(cron = "${datawave.query.monitor.scheduler-crontab:*/30 * * * * ?}") + public void damageReport() { + // perform some upkeep + if (taskFuture != null) { + if (taskFuture.isDone()) { + try { + taskFuture.get(); + } catch (InterruptedException e) { + log.warn("Query Monitor task was interrupted"); + } catch (ExecutionException e) { + log.error("Query Monitor task failed", e.getCause()); + } + taskFuture = null; + } else if (isTaskLeaseExpired()) { + // if the lease has expired for the future, cancel it and wait for next scheduled task + taskFuture.cancel(true); + } + } + + // schedule a new monitor task if the previous one has finished/expired + if (taskFuture != null && isMonitorIntervalExpired()) { + taskStartTime = System.currentTimeMillis(); + taskFuture = executor.submit(new MonitorTask(monitorProperties, monitorStatusCache)); + } + } + + private boolean isTaskLeaseExpired() { + return (System.currentTimeMillis() - taskStartTime) > monitorProperties.getMonitorIntervalMillis(); + } + + private boolean isMonitorIntervalExpired() { + return (System.currentTimeMillis() - monitorStatusCache.getStatus().getLastCheckedMillis()) > monitorProperties.getMonitorIntervalMillis(); + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java new file mode 100644 index 00000000..5ad29d2f --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java @@ -0,0 +1,13 @@ +package datawave.microservice.query.monitor.cache; + +public class MonitorStatus { + private long lastCheckedMillis; + + public long getLastCheckedMillis() { + return lastCheckedMillis; + } + + public void setLastChecked(long lastCheckedMillis) { + this.lastCheckedMillis = lastCheckedMillis; + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java new file mode 100644 index 00000000..914f607e --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java @@ -0,0 +1,80 @@ +package datawave.microservice.query.monitor.cache; + +import datawave.microservice.cached.LockableCacheInspector; +import org.springframework.cache.annotation.CacheConfig; +import org.springframework.cache.annotation.CacheEvict; +import org.springframework.cache.annotation.CachePut; + +import java.util.concurrent.TimeUnit; + +@CacheConfig(cacheNames = MonitorStatusCache.CACHE_NAME) +public class MonitorStatusCache { + private final LockableCacheInspector cacheInspector; + + public static final String CACHE_NAME = "QueryMonitorCache"; + public static final String CACHE_KEY = "MonitorStatus"; + + public MonitorStatusCache(LockableCacheInspector cacheInspector) { + this.cacheInspector = cacheInspector; + } + + /** + * Get the query monitor status + * + * @return + */ + public MonitorStatus getStatus() { + return cacheInspector.list(CACHE_NAME, MonitorStatus.class, CACHE_KEY); + } + + /** + * Store the query monitor status + * + * @param monitorStatus + * @return + */ + @CachePut(key = CACHE_KEY) + public MonitorStatus setStatus(MonitorStatus monitorStatus) { + return monitorStatus; + } + + /** + * Deletes the query monitor status + */ + @CacheEvict(key = CACHE_KEY) + public void deleteStatus() { + + } + + public void lock() { + cacheInspector.lock(CACHE_NAME, CACHE_KEY); + } + + public void lock(long leaseTime, TimeUnit leaseTimeUnit) { + cacheInspector.lock(CACHE_NAME, CACHE_KEY, leaseTime, leaseTimeUnit); + } + + public boolean tryLock() { + return cacheInspector.tryLock(CACHE_NAME, CACHE_KEY); + } + + public boolean tryLock(long waitTime, TimeUnit waitTimeUnit) throws InterruptedException { + return cacheInspector.tryLock(CACHE_NAME, CACHE_KEY, waitTime, waitTimeUnit); + } + + public boolean tryLock(long waitTime, TimeUnit waitTimeUnit, long leaseTime, TimeUnit leaseTimeUnit) throws InterruptedException { + return cacheInspector.tryLock(CACHE_NAME, CACHE_KEY, waitTime, waitTimeUnit, leaseTime, leaseTimeUnit); + } + + public void unlock() { + cacheInspector.unlock(CACHE_NAME, CACHE_KEY); + } + + public void forceUnlock() { + cacheInspector.forceUnlock(CACHE_NAME, CACHE_KEY); + } + + public boolean isLocked() { + return cacheInspector.isLocked(CACHE_NAME, CACHE_KEY); + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java new file mode 100644 index 00000000..206d5a56 --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java @@ -0,0 +1,37 @@ +package datawave.microservice.query.monitor.config; + +import com.hazelcast.spring.cache.HazelcastCacheManager; +import datawave.microservice.cached.CacheInspector; +import datawave.microservice.cached.LockableCacheInspector; +import datawave.microservice.cached.LockableHazelcastCacheInspector; +import datawave.microservice.cached.UniversalLockableCacheInspector; +import datawave.microservice.query.monitor.cache.MonitorStatusCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cache.CacheManager; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableScheduling; + +@EnableCaching +@EnableScheduling +@Configuration +@ConditionalOnProperty(name = "datawave.query.monitor.enabled", havingValue = "true", matchIfMissing = true) +@EnableConfigurationProperties(MonitorProperties.class) +public class MonitorConfig { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + + @Bean + public MonitorStatusCache monitorStatusCache(CacheInspector cacheInspector, CacheManager cacheManager) { + log.debug("Using " + cacheManager.getClass() + " for caching"); + LockableCacheInspector lockableCacheInspector = null; + if (cacheManager instanceof HazelcastCacheManager) + lockableCacheInspector = new LockableHazelcastCacheInspector(cacheManager); + else + lockableCacheInspector = new UniversalLockableCacheInspector(cacheInspector); + return new MonitorStatusCache(lockableCacheInspector); + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java new file mode 100644 index 00000000..d655e24d --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java @@ -0,0 +1,97 @@ +package datawave.microservice.query.monitor.config; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.validation.annotation.Validated; + +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Positive; +import javax.validation.constraints.PositiveOrZero; +import java.util.concurrent.TimeUnit; + +@Validated +@ConfigurationProperties(prefix = "datawave.query.monitor") +public class MonitorProperties { + private String schedulerCrontab; + @PositiveOrZero + private long monitorInterval = TimeUnit.MILLISECONDS.toMillis(30); + @NotNull + private TimeUnit monitorIntervalTimeUnit = TimeUnit.MILLISECONDS; + // The amount of time to wait for the monitor lock to be acquired + @PositiveOrZero + private long lockWaitTime = 0; + @NotNull + private TimeUnit lockWaitTimeUnit = TimeUnit.MILLISECONDS; + // The amount of time that the monitor lock will be held before being automatically released + @Positive + private long lockLeaseTime = TimeUnit.MINUTES.toMillis(1); + @NotNull + private TimeUnit lockLeaseTimeUnit = TimeUnit.MILLISECONDS; + + public String getSchedulerCrontab() { + return schedulerCrontab; + } + + public void setSchedulerCrontab(String schedulerCrontab) { + this.schedulerCrontab = schedulerCrontab; + } + + public long getMonitorInterval() { + return monitorInterval; + } + + public long getMonitorIntervalMillis() { + return monitorIntervalTimeUnit.toMillis(monitorInterval); + } + + public void setMonitorInterval(long monitorInterval) { + this.monitorInterval = monitorInterval; + } + + public TimeUnit getMonitorIntervalTimeUnit() { + return monitorIntervalTimeUnit; + } + + public void setMonitorIntervalTimeUnit(TimeUnit monitorIntervalTimeUnit) { + this.monitorIntervalTimeUnit = monitorIntervalTimeUnit; + } + + public long getLockWaitTime() { + return lockWaitTime; + } + + public long getLockWaitTimeMillis() { + return lockWaitTime; + } + + public void setLockWaitTime(long lockWaitTime) { + this.lockWaitTime = lockWaitTime; + } + + public TimeUnit getLockWaitTimeUnit() { + return lockWaitTimeUnit; + } + + public void setLockWaitTimeUnit(TimeUnit lockWaitTimeUnit) { + this.lockWaitTimeUnit = lockWaitTimeUnit; + } + + public long getLockLeaseTime() { + return lockLeaseTime; + } + + public long getLockLeaseTimeMillis() { + return lockLeaseTimeUnit.toMillis(lockLeaseTime); + } + + public void setLockLeaseTime(long lockLeaseTime) { + this.lockLeaseTime = lockLeaseTime; + } + + public TimeUnit getLockLeaseTimeUnit() { + return lockLeaseTimeUnit; + } + + public void setLockLeaseTimeUnit(TimeUnit lockLeaseTimeUnit) { + this.lockLeaseTimeUnit = lockLeaseTimeUnit; + } +} From e60e87d6aca27367b336179a59769afa1ab315a8 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 3 Jun 2021 16:29:35 -0400 Subject: [PATCH 057/218] Fleshed out the query monitoring logic and added additional query request types. --- .../query/QueryManagementService.java | 159 ++++++++++++++---- .../query/monitor/MonitorTask.java | 80 ++++++++- .../query/monitor/QueryMonitor.java | 16 +- .../query/monitor/cache/MonitorStatus.java | 4 + .../monitor/config/MonitorProperties.java | 25 +++ .../microservice/query/runner/NextCall.java | 52 ++++-- 6 files changed, 284 insertions(+), 52 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index b1200f40..b25c0feb 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -31,7 +31,9 @@ import datawave.webservice.query.exception.NoResultsQueryException; import datawave.webservice.query.exception.NotFoundQueryException; import datawave.webservice.query.exception.QueryException; +import datawave.webservice.query.exception.TimeoutQueryException; import datawave.webservice.query.exception.UnauthorizedQueryException; +import datawave.webservice.query.metric.BaseQueryMetric; import datawave.webservice.query.result.event.ResponseObjectFactory; import datawave.webservice.query.util.QueryUncaughtExceptionHandler; import datawave.webservice.result.BaseQueryResponse; @@ -49,13 +51,14 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.text.MessageFormat; -import java.util.Date; +import java.util.Arrays; import java.util.List; import java.util.UUID; import java.util.concurrent.ExecutionException; import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CANCELED; import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CLOSED; +import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CREATED; @Service public class QueryManagementService implements QueryRequestHandler { @@ -198,6 +201,9 @@ public TaskKey create(String queryLogicName, MultiValueMap parame getMaxConcurrentTasks(queryLogic)); // @formatter:on + // publish a create event to the executor pool + publishExecutorEvent(QueryRequest.create(taskKey.getQueryId().toString()), getPoolName()); + // TODO: JWO: Figure out how to make query tracing work with our new architecture. Datawave issue #1155 // TODO: JWO: Figure out how to make query metrics work with our new architecture. Datawave issue #1156 @@ -234,7 +240,8 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th // before we spin up a separate thread, make sure we are allowed to call next QueryStatus queryStatus = incrementConcurrentNextCount(queryUUID); - // TODO: Publish a next notification? + // publish a next event to the executor pool + publishNextEvent(queryId, queryStatus.getQueryKey().getQueryPool().getName()); // get the query logic String queryLogicName = queryStatus.getQuery().getQueryLogicName(); @@ -274,7 +281,11 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th response.setQueryId(queryId); return response; } else { - throw new NoResultsQueryException(DatawaveErrorCode.NO_QUERY_RESULTS_FOUND, MessageFormat.format("{0}", queryId)); + if (nextCall.getMetric().getLifecycle() == BaseQueryMetric.Lifecycle.NEXTTIMEOUT) { + throw new TimeoutQueryException(DatawaveErrorCode.QUERY_TIMEOUT, MessageFormat.format("{0}", queryId)); + } else { + throw new NoResultsQueryException(DatawaveErrorCode.NO_QUERY_RESULTS_FOUND, MessageFormat.format("{0}", queryId)); + } } } catch (TaskRejectedException e) { throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Next task rejected by the executor for query " + queryId); @@ -308,14 +319,17 @@ private QueryStatus incrementConcurrentNextCount(UUID queryUUID) throws Interrup QueryStorageLock statusLock = queryStorageCache.getQueryStatusLock(queryUUID); if (statusLock.tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { try { - // increment the concurrent next queryStatus = queryStorageCache.getQueryStatus(queryUUID); - if (queryStatus != null) { + + // we can only call next on a created query + if (queryStatus != null && queryStatus.getQueryState() == CREATED) { + + // increment the concurrent next count if (queryStatus.getConcurrentNextCount() < queryProperties.getNextCall().getConcurrency()) { queryStatus.setConcurrentNextCount(queryStatus.getConcurrentNextCount() + 1); // update the last used datetime for the query - queryStatus.setLastUsed(new Date()); + queryStatus.setLastUsedMillis(System.currentTimeMillis()); queryStorageCache.updateQueryStatus(queryStatus); } else { @@ -339,14 +353,15 @@ private QueryStatus decrementConcurrentNextCount(UUID queryUUID) throws Interrup QueryStorageLock statusLock = queryStorageCache.getQueryStatusLock(queryUUID); if (statusLock.tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { try { - // increment the concurrent next queryStatus = queryStorageCache.getQueryStatus(queryUUID); if (queryStatus != null) { + + // decrement the concurrent next count if (queryStatus.getConcurrentNextCount() > 0) { queryStatus.setConcurrentNextCount(queryStatus.getConcurrentNextCount() - 1); // update the last used datetime for the query - queryStatus.setLastUsed(new Date()); + queryStatus.setLastUsedMillis(System.currentTimeMillis()); queryStorageCache.updateQueryStatus(queryStatus); } else { @@ -370,15 +385,15 @@ private long updateStatusAndGetPageNumber(UUID queryUUID, int numResults) throws QueryStorageLock statusLock = queryStorageCache.getQueryStatusLock(queryUUID); if (statusLock.tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { try { - // increment the concurrent next QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryUUID); if (queryStatus != null) { + // decrement the concurrent next count if (queryStatus.getConcurrentNextCount() > 0) { queryStatus.setConcurrentNextCount(queryStatus.getConcurrentNextCount() - 1); // update the last used datetime for the query - queryStatus.setLastUsed(new Date()); + queryStatus.setLastUsedMillis(System.currentTimeMillis()); queryStorageCache.updateQueryStatus(queryStatus); } else { @@ -395,7 +410,7 @@ private long updateStatusAndGetPageNumber(UUID queryUUID, int numResults) throws queryStatus.setNumResultsReturned(queryStatus.getNumResultsReturned() + numResults); // update the last used datetime for the query - queryStatus.setLastUsed(new Date()); + queryStatus.setLastUsedMillis(System.currentTimeMillis()); queryStorageCache.updateQueryStatus(queryStatus); } else { @@ -422,10 +437,10 @@ private long updateStatusAndGetPageNumber(UUID queryUUID, int numResults) throws public void cancel(String queryId, ProxiedUserDetails currentUser) throws QueryException { try { // make sure the query is valid, and the user can act on it - validateRequest(queryId, currentUser); + QueryStatus queryStatus = validateRequest(queryId, currentUser); // cancel the query - cancel(queryId, true); + cancel(queryId, queryStatus.getQueryKey().getQueryPool().getName(), true); } catch (QueryException e) { throw e; } catch (Exception e) { @@ -446,7 +461,24 @@ public void cancel(String queryId, ProxiedUserDetails currentUser) throws QueryE * @throws InterruptedException * @throws QueryException */ - private void cancel(String queryId, boolean publishEvent) throws InterruptedException, QueryException { + public void cancel(String queryId, boolean publishEvent) throws InterruptedException, QueryException { + cancel(queryId, null, publishEvent); + } + + /** + * + * Once the cancel request is validated, this method is called to actually cancel the query. + * + * @param queryId + * The id of the query to cancel + * @param poolName + * The pool name for this query + * @param publishEvent + * If true, the cancel request will be published on the bus + * @throws InterruptedException + * @throws QueryException + */ + public void cancel(String queryId, String poolName, boolean publishEvent) throws InterruptedException, QueryException { UUID queryUUID = UUID.fromString(queryId); // if we have an active next call for this query locally, cancel it @@ -463,6 +495,10 @@ private void cancel(String queryId, boolean publishEvent) throws InterruptedExce QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryUUID); queryStatus.setQueryState(CANCELED); queryStorageCache.updateQueryStatus(queryStatus); + + if (poolName == null) { + poolName = queryStatus.getQueryKey().getQueryPool().getName(); + } } finally { statusLock.unlock(); } @@ -470,17 +506,21 @@ private void cancel(String queryId, boolean publishEvent) throws InterruptedExce // TODO: Instead of throwing an exception, do we want to be mean here and update the state to canceled without acquiring a lock? Probably yes... throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to acquire lock on query"); } - } else { - if (publishEvent) { - // broadcast the cancel request to all of the query services - appCtx.publishEvent(new RemoteQueryRequestEvent(this, busProperties.getId(), appCtx.getApplicationName(), QueryRequest.cancel(queryId))); - } } if (publishEvent) { - // broadcast the cancel request to all of the executor services - appCtx.publishEvent( - new RemoteQueryRequestEvent(this, busProperties.getId(), queryProperties.getExecutorServiceName(), QueryRequest.cancel(queryId))); + QueryRequest cancelRequest = QueryRequest.cancel(queryId); + + // publish a cancel event to all of the query services + publishSelfEvent(cancelRequest); + + // get the pool name if we don't have it already + if (poolName == null) { + poolName = queryStorageCache.getQueryStatus(queryUUID).getQueryKey().getQueryPool().getName(); + } + + // publish a cancel event to the executor pool + publishExecutorEvent(cancelRequest, poolName); } } @@ -496,10 +536,10 @@ private void cancel(String queryId, boolean publishEvent) throws InterruptedExce public void close(String queryId, ProxiedUserDetails currentUser) throws QueryException { try { // make sure the query is valid, and the user can act on it - validateRequest(queryId, currentUser); + QueryStatus queryStatus = validateRequest(queryId, currentUser); // close the query - close(queryId, true); + close(queryId, queryStatus.getQueryKey().getQueryPool().getName(), true); } catch (QueryException e) { throw e; } catch (Exception e) { @@ -520,17 +560,39 @@ public void close(String queryId, ProxiedUserDetails currentUser) throws QueryEx * @throws InterruptedException * @throws QueryException */ - private void close(String queryId, boolean publishEvent) throws InterruptedException, QueryException { + public void close(String queryId, boolean publishEvent) throws InterruptedException, QueryException { + close(queryId, null, publishEvent); + } + + /** + * + * Once the close request is validated, this method is called to actually close the query. + * + * @param queryId + * The id of the query to close + * @param poolName + * The pool name for this query + * @param publishEvent + * If true, the close request will be published on the bus + * @throws InterruptedException + * @throws QueryException + */ + public void close(String queryId, String poolName, boolean publishEvent) throws InterruptedException, QueryException { UUID queryUUID = UUID.fromString(queryId); - if (queryStorageCache.getQueryStatusLock(queryUUID).tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { + QueryStorageLock statusLock = queryStorageCache.getQueryStatusLock(queryUUID); + if (statusLock.tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { try { // update query state to CLOSED QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryUUID); queryStatus.setQueryState(CLOSED); queryStorageCache.updateQueryStatus(queryStatus); + + if (poolName == null) { + poolName = queryStatus.getQueryKey().getQueryPool().getName(); + } } finally { - queryStorageCache.getQueryStatusLock(queryUUID).unlock(); + statusLock.unlock(); } } else { // TODO: Instead of throwing an exception, do we want to be mean here and update the state to canceled without acquiring a lock? Probably yes... @@ -538,13 +600,16 @@ private void close(String queryId, boolean publishEvent) throws InterruptedExcep } if (publishEvent) { - // broadcast the close request to all of the executor services - appCtx.publishEvent( - new RemoteQueryRequestEvent(this, busProperties.getId(), queryProperties.getExecutorServiceName(), QueryRequest.close(queryId))); + if (poolName == null) { + poolName = queryStorageCache.getQueryStatus(queryUUID).getQueryKey().getQueryPool().getName(); + } + + // publish a close event to the executor pool + publishExecutorEvent(QueryRequest.close(queryId), poolName); } } - private void validateRequest(String queryId, ProxiedUserDetails currentUser) throws QueryException { + private QueryStatus validateRequest(String queryId, ProxiedUserDetails currentUser) throws QueryException { // does the query exist? QueryStatus queryStatus = queryStorageCache.getQueryStatus(UUID.fromString(queryId)); if (queryStatus == null) { @@ -558,6 +623,8 @@ private void validateRequest(String queryId, ProxiedUserDetails currentUser) thr if (!query.getOwner().equals(userId)) { throw new UnauthorizedQueryException(DatawaveErrorCode.QUERY_OWNER_MISMATCH, MessageFormat.format("{0} != {1}", userId, query.getOwner())); } + + return queryStatus; } @Override @@ -626,6 +693,36 @@ protected String getPoolName() { return (queryParameters.getPool() != null) ? queryParameters.getPool() : queryProperties.getDefaultParams().getPool(); } + protected String getPooledExecutorName(String poolName) { + return String.join("-", Arrays.asList(queryProperties.getExecutorServiceName(), poolName)); + } + + public void publishNextEvent(String queryId, String queryPool) { + publishExecutorEvent(QueryRequest.next(queryId), queryPool); + } + + private void publishExecutorEvent(QueryRequest queryRequest, String queryPool) { + // @formatter:off + appCtx.publishEvent( + new RemoteQueryRequestEvent( + this, + busProperties.getId(), + getPooledExecutorName(queryPool), + queryRequest)); + // @formatter:on + } + + private void publishSelfEvent(QueryRequest queryRequest) { + // @formatter:off + appCtx.publishEvent( + new RemoteQueryRequestEvent( + this, + busProperties.getId(), + appCtx.getApplicationName(), + queryRequest)); + // @formatter:on + } + protected int getMaxConcurrentTasks(QueryLogic queryLogic) { // if there's an override, use it if (queryParameters.isMaxConcurrentTasksOverridden()) { diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java index 137e753f..4bfab83f 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java @@ -1,18 +1,36 @@ package datawave.microservice.query.monitor; +import datawave.microservice.common.storage.QueryStatus; +import datawave.microservice.common.storage.QueryStorageCache; +import datawave.microservice.query.QueryManagementService; +import datawave.microservice.query.config.QueryExpirationProperties; import datawave.microservice.query.monitor.cache.MonitorStatus; import datawave.microservice.query.monitor.cache.MonitorStatusCache; import datawave.microservice.query.monitor.config.MonitorProperties; +import datawave.webservice.query.exception.QueryException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.UUID; import java.util.concurrent.Callable; public class MonitorTask implements Callable { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + private final MonitorProperties monitorProperties; + private final QueryExpirationProperties expirationProperties; private final MonitorStatusCache monitorStatusCache; + private final QueryStorageCache queryStorageCache; + private final QueryManagementService queryManagementService; - public MonitorTask(MonitorProperties monitorProperties, MonitorStatusCache monitorStatusCache) { + public MonitorTask(MonitorProperties monitorProperties, QueryExpirationProperties expirationProperties, MonitorStatusCache monitorStatusCache, + QueryStorageCache queryStorageCache, QueryManagementService queryManagementService) { this.monitorProperties = monitorProperties; + this.expirationProperties = expirationProperties; this.monitorStatusCache = monitorStatusCache; + this.queryStorageCache = queryStorageCache; + this.queryManagementService = queryManagementService; } @Override @@ -21,15 +39,16 @@ public Void call() throws Exception { boolean success = false; MonitorStatus monitorStatus = null; try { + long currentTimeMillis = System.currentTimeMillis(); monitorStatus = monitorStatusCache.getStatus(); - if (isMonitorIntervalExpired()) { - // TODO: perform idle checks, etc. - + if (monitorStatus.isExpired(currentTimeMillis, monitorProperties.getMonitorIntervalMillis())) { + monitor(currentTimeMillis); success = true; } } finally { - if (success && monitorStatus != null) { + if (success) { monitorStatus.setLastChecked(System.currentTimeMillis()); + monitorStatusCache.setStatus(monitorStatus); } unlock(); } @@ -37,6 +56,53 @@ public Void call() throws Exception { return null; } + // TODO: Check for the following conditions + // 1) Is query progress idle? If so, poke the query + // 2) Is the user idle? If so, close the query + // 3) Are there any other conditions that we should check for? + private void monitor(long currentTimeMillis) { + for (QueryStatus status : queryStorageCache.getQueryStatus()) { + String queryId = status.getQueryKey().getQueryId().toString(); + if (status.getQueryState() == QueryStatus.QUERY_STATE.DEFINED || status.getQueryState() == QueryStatus.QUERY_STATE.CREATED) { + if (status.isUserIdle(currentTimeMillis, expirationProperties.getIdleTimeoutMillis())) { + // if the user hasn't interacted with the query in a while, cancel it + cancelQuery(queryId); + } else if (status.isProgressIdle(currentTimeMillis, expirationProperties.getIdleTimeoutMillis())) { + // if progress hasn't been made for the query in a while, apply the shock paddles + defibrilateQuery(queryId, status.getQueryKey().getQueryPool().getName()); + } + } else { + if (status.isInactive(currentTimeMillis, monitorProperties.getInactiveQueryTimeToLiveMillis())) { + // if the query has been inactive for too long, evict it + deleteQuery(UUID.fromString(queryId)); + } + } + } + } + + private void cancelQuery(String queryId) { + try { + queryManagementService.cancel(queryId, true); + } catch (InterruptedException e) { + log.error("Interrupted while trying to cancel idle query: " + queryId, e); + } catch (QueryException e) { + log.error("Encountered error while trying to cancel idle query: " + queryId, e); + } + } + + private void defibrilateQuery(String queryId, String queryPool) { + // publish a next event to the executor pool + queryManagementService.publishNextEvent(queryId, queryPool); + } + + private void deleteQuery(UUID queryUUID) { + try { + queryStorageCache.deleteQuery(queryUUID); + } catch (IOException e) { + log.error("Encountered error while trying to evict inactive query: " + queryUUID.toString(), e); + } + } + private boolean tryLock() throws InterruptedException { return monitorStatusCache.tryLock(monitorProperties.getLockWaitTime(), monitorProperties.getLockWaitTimeUnit(), monitorProperties.getLockLeaseTime(), monitorProperties.getLockLeaseTimeUnit()); @@ -45,8 +111,4 @@ private boolean tryLock() throws InterruptedException { private void unlock() { monitorStatusCache.unlock(); } - - private boolean isMonitorIntervalExpired() { - return (System.currentTimeMillis() - monitorStatusCache.getStatus().getLastCheckedMillis()) > monitorProperties.getMonitorIntervalMillis(); - } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java index 62de49f3..d627ccfd 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java @@ -1,5 +1,9 @@ package datawave.microservice.query.monitor; +import datawave.microservice.common.storage.QueryStorageCache; +import datawave.microservice.query.QueryManagementService; +import datawave.microservice.query.config.QueryExpirationProperties; +import datawave.microservice.query.config.QueryProperties; import datawave.microservice.query.monitor.cache.MonitorStatusCache; import datawave.microservice.query.monitor.config.MonitorProperties; import org.slf4j.Logger; @@ -19,15 +23,22 @@ public class QueryMonitor { private final Logger log = LoggerFactory.getLogger(this.getClass()); private final MonitorProperties monitorProperties; + private final QueryExpirationProperties expirationProperties; private final MonitorStatusCache monitorStatusCache; + private final QueryStorageCache queryStorageCache; + private final QueryManagementService queryManagementService; private final ExecutorService executor = Executors.newSingleThreadExecutor(); private long taskStartTime; private Future taskFuture; - public QueryMonitor(MonitorProperties monitorProperties, MonitorStatusCache monitorStatusCache) { + public QueryMonitor(MonitorProperties monitorProperties, QueryProperties queryProperties, MonitorStatusCache monitorStatusCache, + QueryStorageCache queryStorageCache, QueryManagementService queryManagementService) { this.monitorProperties = monitorProperties; + this.expirationProperties = queryProperties.getExpiration(); this.monitorStatusCache = monitorStatusCache; + this.queryStorageCache = queryStorageCache; + this.queryManagementService = queryManagementService; } // this runs in a separate thread every 30 seconds (by default) @@ -53,7 +64,8 @@ public void damageReport() { // schedule a new monitor task if the previous one has finished/expired if (taskFuture != null && isMonitorIntervalExpired()) { taskStartTime = System.currentTimeMillis(); - taskFuture = executor.submit(new MonitorTask(monitorProperties, monitorStatusCache)); + taskFuture = executor + .submit(new MonitorTask(monitorProperties, expirationProperties, monitorStatusCache, queryStorageCache, queryManagementService)); } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java index 5ad29d2f..8a701fee 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java @@ -10,4 +10,8 @@ public long getLastCheckedMillis() { public void setLastChecked(long lastCheckedMillis) { this.lastCheckedMillis = lastCheckedMillis; } + + public boolean isExpired(long currentTimeMillis, long expirationTimeoutMillis) { + return (currentTimeMillis - lastCheckedMillis) >= expirationTimeoutMillis; + } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java index d655e24d..9147a960 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java @@ -26,6 +26,11 @@ public class MonitorProperties { private long lockLeaseTime = TimeUnit.MINUTES.toMillis(1); @NotNull private TimeUnit lockLeaseTimeUnit = TimeUnit.MILLISECONDS; + // The amount of time that an inactive query should remain in the query cache + @PositiveOrZero + private long inactiveQueryTimeToLive = 24; + @NotNull + private TimeUnit inactiveQueryTimeUnit = TimeUnit.HOURS; public String getSchedulerCrontab() { return schedulerCrontab; @@ -94,4 +99,24 @@ public TimeUnit getLockLeaseTimeUnit() { public void setLockLeaseTimeUnit(TimeUnit lockLeaseTimeUnit) { this.lockLeaseTimeUnit = lockLeaseTimeUnit; } + + public long getInactiveQueryTimeToLive() { + return inactiveQueryTimeToLive; + } + + public long getInactiveQueryTimeToLiveMillis() { + return inactiveQueryTimeUnit.toMillis(inactiveQueryTimeToLive); + } + + public void setInactiveQueryTimeToLive(long inactiveQueryTimeToLive) { + this.inactiveQueryTimeToLive = inactiveQueryTimeToLive; + } + + public TimeUnit getInactiveQueryTimeUnit() { + return inactiveQueryTimeUnit; + } + + public void setInactiveQueryTimeUnit(TimeUnit inactiveQueryTimeUnit) { + this.inactiveQueryTimeUnit = inactiveQueryTimeUnit; + } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index ed4f9330..0af575fe 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -117,30 +117,36 @@ public ResultsPage call() throws Exception { private boolean isFinished(String queryId) { boolean finished = false; + long callTimeMillis = System.currentTimeMillis() - startTimeMillis; // 1) was this query canceled? if (canceled) { log.info("Query [{}]: query cancelled, aborting next call", queryId); + finished = true; } // 2) have we hit the user's results-per-page limit? if (!finished && results.size() >= userResultsPerPage) { log.info("Query [{}]: user requested max page size has been reached, aborting next call", queryId); + finished = true; } // 3) have we hit the query logic's results-per-page limit? if (!finished && results.size() >= logicResultsPerPage) { log.info("Query [{}]: query logic max page size has been reached, aborting next call", queryId); + finished = true; } // 4) have we hit the query logic's bytes-per-page limit? if (!finished && pageSizeBytes >= logicBytesPerPage) { log.info("Query [{}]: query logic max page byte size has been reached, aborting next call", queryId); - finished = true; + status = ResultsPage.Status.PARTIAL; + + finished = true; } // 5) have we hit the max results (or the max results override)? @@ -150,14 +156,18 @@ private boolean isFinished(String queryId) { if (this.maxResultsOverridden) { if (maxResultsOverride >= 0 && numResults >= maxResultsOverride) { log.info("Query [{}]: max results override has been reached, aborting next call", queryId); + // TODO: Figure out query metrics metric.setLifecycle(QueryMetric.Lifecycle.MAXRESULTS); + finished = true; } } else if (maxResults >= 0 && numResults >= maxResults) { log.info("Query [{}]: logic max results has been reached, aborting next call", queryId); + // TODO: Figure out query metrics metric.setLifecycle(QueryMetric.Lifecycle.MAXRESULTS); + finished = true; } } @@ -166,30 +176,48 @@ private boolean isFinished(String queryId) { // This used to come from the query logic transform iterator // 6) have we reached the "max work" limit? (i.e. next count + seek count) if (!finished && logicMaxWork >= 0 && (metric.getNextCount() + metric.getSeekCount()) >= logicMaxWork) { - log.info("Query [{}]: logic max work has been reached, aborting next call", queryId); + log.info("Query [{}]: logic max work has been reached, aborting next call", queryId); + // TODO: Figure out query metrics metric.setLifecycle(BaseQueryMetric.Lifecycle.MAXWORK); + finished = true; } // 7) are we going to timeout before getting a full page? if so, return partial results - if (!finished && timeout()) { + if (!finished && shortCircuitTimeout(callTimeMillis)) { log.info("Query [{}]: logic max expire before page is full, returning existing results: {} of {} results in {}ms", queryId, results.size(), - maxResultsPerPage, callTimeMillis()); - finished = true; + maxResultsPerPage, callTimeMillis); + status = ResultsPage.Status.PARTIAL; + + finished = true; + + } + + // 8) have we been in this next call too long? if so, return + if (!finished && callExpiredTimeout(callTimeMillis)) { + log.info("Query [{}]: max call time reached, returning existing results: {} of {} results in {}ms", queryId, results.size(), maxResultsPerPage, + callTimeMillis); + + if (!results.isEmpty()) { + status = ResultsPage.Status.PARTIAL; + } + + // TODO: Figure out query metrics + metric.setLifecycle(BaseQueryMetric.Lifecycle.NEXTTIMEOUT); + + finished = true; } return finished; } - protected boolean timeout() { + private boolean shortCircuitTimeout(long callTimeMillis) { boolean timeout = false; // only return prematurely if we have at least 1 result if (!results.isEmpty()) { - long callTimeMillis = callTimeMillis(); - // if after the page size short circuit check time if (callTimeMillis >= expirationProperties.getShortCircuitCheckTimeMillis()) { float percentTimeComplete = (float) callTimeMillis / (float) (expirationProperties.getCallTimeout()); @@ -209,8 +237,8 @@ protected boolean timeout() { return timeout; } - private long callTimeMillis() { - return System.currentTimeMillis() - startTimeMillis; + private boolean callExpiredTimeout(long callTimeMillis) { + return callTimeMillis >= expirationProperties.getCallTimeoutMillis(); } private QueryStatus getQueryStatus() { @@ -241,6 +269,10 @@ public void setFuture(Future> future) { this.future = future; } + public BaseQueryMetric getMetric() { + return metric; + } + public static class Builder { private NextCallProperties nextCallProperties; private QueryExpirationProperties expirationProperties; From 5d8f3ee7ede4cc51d11437e3ba16f693b0af65df Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 3 Jun 2021 17:09:13 -0400 Subject: [PATCH 058/218] formatting --- .../microservice/query/monitor/QueryMonitor.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java index d627ccfd..8894eeaa 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java @@ -43,7 +43,7 @@ public QueryMonitor(MonitorProperties monitorProperties, QueryProperties queryPr // this runs in a separate thread every 30 seconds (by default) @Scheduled(cron = "${datawave.query.monitor.scheduler-crontab:*/30 * * * * ?}") - public void damageReport() { + public void monitorTaskScheduler() { // perform some upkeep if (taskFuture != null) { if (taskFuture.isDone()) { @@ -64,8 +64,15 @@ public void damageReport() { // schedule a new monitor task if the previous one has finished/expired if (taskFuture != null && isMonitorIntervalExpired()) { taskStartTime = System.currentTimeMillis(); - taskFuture = executor - .submit(new MonitorTask(monitorProperties, expirationProperties, monitorStatusCache, queryStorageCache, queryManagementService)); + // @formatter:off + taskFuture = executor.submit( + new MonitorTask( + monitorProperties, + expirationProperties, + monitorStatusCache, + queryStorageCache, + queryManagementService)); + // @formatter:on } } From 4d423133e824dc223643507ae15a2534e90eb4d7 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 4 Jun 2021 12:02:56 -0400 Subject: [PATCH 059/218] Added a QueryStatus Update Helper which automatically handles locking/unlocking of the query status during updates. --- query-microservices/query-service/README.md | 72 +++--- .../query/QueryManagementService.java | 237 ++++-------------- .../microservice/query/runner/NextCall.java | 10 +- .../query/status/QueryStatusUpdateHelper.java | 75 ++++++ .../query/status/StatusUpdater.java | 13 + 5 files changed, 169 insertions(+), 238 deletions(-) create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/StatusUpdater.java diff --git a/query-microservices/query-service/README.md b/query-microservices/query-service/README.md index c8241095..93f95764 100644 --- a/query-microservices/query-service/README.md +++ b/query-microservices/query-service/README.md @@ -11,42 +11,42 @@ query capabilities. ### User API -| Method | Operation | Description | Path Param | Request Body | Response Body | -|:--------------|:-------------------------------------|:----------------------------------------------------------------------------------|:-------------------|:---------------------|:-----------------------------------------| -| `GET` | /listQueryLogic | List QueryLogic types that are currently available | N/A | N/A | [QueryLogicResponse] | -| `POST` | /{queryLogic}/define | Define a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | -| `POST` | /{queryLogic}/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | -| `POST` | /{queryLogic}/plan | Generate a query plan using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | -| `POST` | /{queryLogic}/predict | Generate a query prediction using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | -| `POST` | /{queryLogic}/async/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | -| `PUT` | /{id}/reset | Resets the specified query | [QueryId] | N/A | [VoidResponse] | -| `POST` | /{queryLogic}/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | -| `POST` | /{queryLogic}/async/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | -| `GET` | /lookupContentUUID/{uuidType}/{uuid} | Returns content associated with the given UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] or [StreamingOutput] | -| `POST` | /lookupContentUUID | Returns content associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] or [StreamingOutput] | -| `GET` | /lookupUUID/{uuidType}/{uuid} | Returns event associated with the given batch of UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] or [StreamingOutput] | -| `POST` | /lookupUUID | Returns event(s) associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] or [StreamingOutput] | -| `GET` | /{id}/plan | Returns the plan for the specified query | [QueryId] | N/A | [GenericResponse] | -| `GET` | /{id}/predictions | Returns the predictions for the specified query | [QueryId] | N/A | [GenericResponse] | -| `GET` | /{id}/async/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | -| `GET` | /{id}/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | -| `PUT` `POST` | /{id}/close | Closes the specified query | [QueryId] | N/A | [VoidResponse] | -| `PUT` `POST` | /{id}/adminClose | Closes the specified query | [QueryId] | N/A | [VoidResponse] | -| `PUT` `POST` | /{id}/cancel | Cancels the specified query | [QueryId] | N/A | [VoidResponse] | -| `PUT` `POST` | /{id}/adminCancel | Cancels the specified query | [QueryId] | N/A | [VoidResponse] | -| `GET` | /listAll | Returns a list of queries associated with the current user | N/A | N/A | [QueryImplListResponse] | -| `GET` | /{id} | Returns query info for the specified query | [QueryId] | N/A | [QueryImplListResponse] | -| `GET` | /list | Returns a list of queries associated with the current user with given name | N/A | [Name] | [QueryImplListResponse] | -| `DELETE` | /{id}/remove | Remove (delete) the specified query | [QueryId] | N/A | [VoidResponse] | -| `POST` | /{id}/duplicate | Duplicates the specified query | [QueryId] | [QueryParameters] | [GenericResponse] | -| `PUT` `POST` | /{id}/update | Updates the specified query | [QueryId] | [QueryParameters] | [GenericResponse] | -| `GET` | /{id}/listAll | Returns a list of queries associated with the specified user | [UserId] | N/A | [QueryImplListResponse] | -| `POST` | /purgeQueryCache | Purges the cache of query objects | N/A | N/A | [VoidResponse] | -| `GET` | /enableTracing | Enables tracing for queries which match the given criteria | N/A | [QueryRegex], [User] | [VoidResponse] | -| `GET` | /disableTracing | Disables tracing for queries which match the given criteria | N/A | [QueryRegex], [User] | [VoidResponse] | -| `GET` | /disableAllTracing | Disables tracing for all queries | N/A | N/A | [VoidResponse] | -| `POST` | /{logicName}/execute | Create a query using the specified query logic and params, and stream the results | [QueryLogicName] | [QueryParameters] | [StreamingOutput] | -| `POST` | /{logicName}/async/execute | Create a query using the specified query logic and params, and stream the results | [QueryLogicName] | [QueryParameters] | [StreamingOutput] | +| Done? | Method | Operation | Description | Path Param | Request Body | Response Body | +|:--------|:--------------|:-------------------------------------|:----------------------------------------------------------------------------------|:-------------------|:---------------------|:-----------------------------------------| +| | `GET` | /listQueryLogic | List QueryLogic types that are currently available | N/A | N/A | [QueryLogicResponse] | +| ✓ | `POST` | /{queryLogic}/define | Define a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| ✓ | `POST` | /{queryLogic}/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| | `POST` | /{queryLogic}/plan | Generate a query plan using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| | `POST` | /{queryLogic}/predict | Generate a query prediction using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| | `POST` | /{queryLogic}/async/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| | `PUT` | /{id}/reset | Resets the specified query | [QueryId] | N/A | [VoidResponse] | +| | `POST` | /{queryLogic}/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | +| | `POST` | /{queryLogic}/async/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | +| | `GET` | /lookupContentUUID/{uuidType}/{uuid} | Returns content associated with the given UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] or [StreamingOutput] | +| | `POST` | /lookupContentUUID | Returns content associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] or [StreamingOutput] | +| | `GET` | /lookupUUID/{uuidType}/{uuid} | Returns event associated with the given batch of UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] or [StreamingOutput] | +| | `POST` | /lookupUUID | Returns event(s) associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] or [StreamingOutput] | +| | `GET` | /{id}/plan | Returns the plan for the specified query | [QueryId] | N/A | [GenericResponse] | +| | `GET` | /{id}/predictions | Returns the predictions for the specified query | [QueryId] | N/A | [GenericResponse] | +| | `GET` | /{id}/async/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | +| ✓ | `GET` | /{id}/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | +| ✓ | `PUT` `POST` | /{id}/close | Closes the specified query | [QueryId] | N/A | [VoidResponse] | +| | `PUT` `POST` | /{id}/adminClose | Closes the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | `PUT` `POST` | /{id}/cancel | Cancels the specified query | [QueryId] | N/A | [VoidResponse] | +| | `PUT` `POST` | /{id}/adminCancel | Cancels the specified query | [QueryId] | N/A | [VoidResponse] | +| | `GET` | /listAll | Returns a list of queries associated with the current user | N/A | N/A | [QueryImplListResponse] | +| | `GET` | /{id} | Returns query info for the specified query | [QueryId] | N/A | [QueryImplListResponse] | +| | `GET` | /list | Returns a list of queries associated with the current user with given name | N/A | [Name] | [QueryImplListResponse] | +| | `DELETE` | /{id}/remove | Remove (delete) the specified query | [QueryId] | N/A | [VoidResponse] | +| | `POST` | /{id}/duplicate | Duplicates the specified query | [QueryId] | [QueryParameters] | [GenericResponse] | +| | `PUT` `POST` | /{id}/update | Updates the specified query | [QueryId] | [QueryParameters] | [GenericResponse] | +| | `GET` | /{id}/listAll | Returns a list of queries associated with the specified user | [UserId] | N/A | [QueryImplListResponse] | +| | `POST` | /purgeQueryCache | Purges the cache of query objects | N/A | N/A | [VoidResponse] | +| | `GET` | /enableTracing | Enables tracing for queries which match the given criteria | N/A | [QueryRegex], [User] | [VoidResponse] | +| | `GET` | /disableTracing | Disables tracing for queries which match the given criteria | N/A | [QueryRegex], [User] | [VoidResponse] | +| | `GET` | /disableAllTracing | Disables tracing for all queries | N/A | N/A | [VoidResponse] | +| | `POST` | /{logicName}/execute | Create a query using the specified query logic and params, and stream the results | [QueryLogicName] | [QueryParameters] | [StreamingOutput] | +| | `POST` | /{logicName}/async/execute | Create a query using the specified query logic and params, and stream the results | [QueryLogicName] | [QueryParameters] | [StreamingOutput] | --- diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index b25c0feb..5022583d 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -11,7 +11,6 @@ import datawave.microservice.common.storage.QueryQueueManager; import datawave.microservice.common.storage.QueryStatus; import datawave.microservice.common.storage.QueryStorageCache; -import datawave.microservice.common.storage.QueryStorageLock; import datawave.microservice.common.storage.TaskKey; import datawave.microservice.query.config.QueryProperties; import datawave.microservice.query.logic.QueryLogic; @@ -19,6 +18,7 @@ import datawave.microservice.query.remote.QueryRequest; import datawave.microservice.query.remote.QueryRequestHandler; import datawave.microservice.query.runner.NextCall; +import datawave.microservice.query.status.QueryStatusUpdateHelper; import datawave.microservice.query.util.QueryUtil; import datawave.security.util.ProxiedEntityUtils; import datawave.webservice.common.audit.AuditParameters; @@ -58,7 +58,6 @@ import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CANCELED; import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CLOSED; -import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CREATED; @Service public class QueryManagementService implements QueryRequestHandler { @@ -82,10 +81,10 @@ public class QueryManagementService implements QueryRequestHandler { private final QueryStorageCache queryStorageCache; private final QueryQueueManager queryQueueManager; private final AuditClient auditClient; - + private final ThreadPoolTaskExecutor nextCallExecutor; private final String identifier; - private final ThreadPoolTaskExecutor nextCallExecutor; + private final QueryStatusUpdateHelper queryStatusUpdateHelper; private final MultiValueMap nextCallMap = new LinkedMultiValueMap<>(); // TODO: JWO: Pull these from configuration instead @@ -116,11 +115,13 @@ public QueryManagementService(QueryProperties queryProperties, ApplicationContex hostname = "UNKNOWN"; } this.identifier = hostname; + + this.queryStatusUpdateHelper = new QueryStatusUpdateHelper(this.queryProperties, this.queryStorageCache); } /** * Defines a datawave query. - * + *

* Validates the query parameters using the base validation, query logic-specific validation, and markings validation. If the parameters are valid, the * query will be stored in the query storage cache where it can be acted upon in a subsequent call. * @@ -165,7 +166,7 @@ public TaskKey define(String queryLogicName, MultiValueMap parame /** * Creates a datawave query. - * + *

* Validates the query parameters using the base validation, query logic-specific validation, and markings validation. If the parameters are valid, the * query will be stored in the query storage cache where it can be acted upon in a subsequent call. * @@ -227,7 +228,6 @@ public TaskKey create(String queryLogicName, MultiValueMap parame * * @param queryId * @param currentUser - * * @return * @throws QueryException */ @@ -238,7 +238,7 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th validateRequest(queryId, currentUser); // before we spin up a separate thread, make sure we are allowed to call next - QueryStatus queryStatus = incrementConcurrentNextCount(queryUUID); + QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(queryUUID, queryStatusUpdateHelper::claimConcurrentNext); // publish a next event to the executor pool publishNextEvent(queryId, queryStatus.getQueryKey().getQueryPool().getName()); @@ -259,7 +259,7 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th .build(); // @formatter:on - long pageNumber = -1L; + boolean success = false; nextCallMap.add(queryId, nextCall); try { // submit the next call to the executor @@ -273,12 +273,18 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th BaseQueryResponse response = queryLogic.getTransformer(queryStatus.getQuery()).createResponse(resultsPage); // after all of our work is done, perform our final query status update for this next call - pageNumber = updateStatusAndGetPageNumber(queryUUID, resultsPage.getResults().size()); + queryStatus = queryStatusUpdateHelper.lockedUpdate(queryUUID, status -> { + queryStatusUpdateHelper.releaseConcurrentNext(status); + status.setLastPageNumber(status.getLastPageNumber() + 1); + status.setNumResultsReturned(status.getNumResultsReturned() + resultsPage.getResults().size()); + }); response.setHasResults(true); - response.setPageNumber(pageNumber); + response.setPageNumber(queryStatus.getLastPageNumber()); response.setLogicName(queryLogicName); response.setQueryId(queryId); + + success = true; return response; } else { if (nextCall.getMetric().getLifecycle() == BaseQueryMetric.Lifecycle.NEXTTIMEOUT) { @@ -302,8 +308,8 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th nextCallMap.get(queryId).remove(nextCall); // update query status if we failed - if (pageNumber > 0) { - decrementConcurrentNextCount(queryUUID); + if (!success) { + queryStatusUpdateHelper.lockedUpdate(queryUUID, queryStatusUpdateHelper::releaseConcurrentNext); } } } catch (QueryException e) { @@ -314,124 +320,12 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th } } - private QueryStatus incrementConcurrentNextCount(UUID queryUUID) throws InterruptedException, QueryException { - QueryStatus queryStatus; - QueryStorageLock statusLock = queryStorageCache.getQueryStatusLock(queryUUID); - if (statusLock.tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { - try { - queryStatus = queryStorageCache.getQueryStatus(queryUUID); - - // we can only call next on a created query - if (queryStatus != null && queryStatus.getQueryState() == CREATED) { - - // increment the concurrent next count - if (queryStatus.getConcurrentNextCount() < queryProperties.getNextCall().getConcurrency()) { - queryStatus.setConcurrentNextCount(queryStatus.getConcurrentNextCount() + 1); - - // update the last used datetime for the query - queryStatus.setLastUsedMillis(System.currentTimeMillis()); - - queryStorageCache.updateQueryStatus(queryStatus); - } else { - throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, - "Concurrent next call limit reached: " + queryProperties.getNextCall().getConcurrency()); - } - } else { - throw new QueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH, "Unable to find query status in cache."); - } - } finally { - statusLock.unlock(); - } - } else { - throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to acquire lock on query."); - } - return queryStatus; - } - - private QueryStatus decrementConcurrentNextCount(UUID queryUUID) throws InterruptedException, QueryException { - QueryStatus queryStatus; - QueryStorageLock statusLock = queryStorageCache.getQueryStatusLock(queryUUID); - if (statusLock.tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { - try { - queryStatus = queryStorageCache.getQueryStatus(queryUUID); - if (queryStatus != null) { - - // decrement the concurrent next count - if (queryStatus.getConcurrentNextCount() > 0) { - queryStatus.setConcurrentNextCount(queryStatus.getConcurrentNextCount() - 1); - - // update the last used datetime for the query - queryStatus.setLastUsedMillis(System.currentTimeMillis()); - - queryStorageCache.updateQueryStatus(queryStatus); - } else { - throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, - "Concurrent next count can't be decremented: " + queryStatus.getConcurrentNextCount()); - } - } else { - throw new QueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH, "Unable to find query status in cache."); - } - } finally { - statusLock.unlock(); - } - } else { - throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to acquire lock on query."); - } - return queryStatus; - } - - private long updateStatusAndGetPageNumber(UUID queryUUID, int numResults) throws InterruptedException, QueryException { - long pageNumber; - QueryStorageLock statusLock = queryStorageCache.getQueryStatusLock(queryUUID); - if (statusLock.tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { - try { - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryUUID); - if (queryStatus != null) { - - // decrement the concurrent next count - if (queryStatus.getConcurrentNextCount() > 0) { - queryStatus.setConcurrentNextCount(queryStatus.getConcurrentNextCount() - 1); - - // update the last used datetime for the query - queryStatus.setLastUsedMillis(System.currentTimeMillis()); - - queryStorageCache.updateQueryStatus(queryStatus); - } else { - // TODO: Exception, or error? - throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, - "Concurrent next count can't be decremented: " + queryStatus.getConcurrentNextCount()); - } - - // get and increment the page number - queryStatus.setLastPageNumber(queryStatus.getLastPageNumber() + 1); - pageNumber = queryStatus.getLastPageNumber(); - - // update the number of results returned - queryStatus.setNumResultsReturned(queryStatus.getNumResultsReturned() + numResults); - - // update the last used datetime for the query - queryStatus.setLastUsedMillis(System.currentTimeMillis()); - - queryStorageCache.updateQueryStatus(queryStatus); - } else { - throw new QueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH, "Unable to find query status in cache."); - } - } finally { - statusLock.unlock(); - } - } else { - throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to acquire lock on query."); - } - return pageNumber; - } - /** * Releases the resources associated with this query. Any currently running calls to 'next' on the query will be stopped. Calls to 'next' after a 'cancel' * will start over at page 1. * * @param queryId * @param currentUser - * * @throws QueryException */ public void cancel(String queryId, ProxiedUserDetails currentUser) throws QueryException { @@ -451,7 +345,6 @@ public void cancel(String queryId, ProxiedUserDetails currentUser) throws QueryE } /** - * * Once the cancel request is validated, this method is called to actually cancel the query. * * @param queryId @@ -466,7 +359,6 @@ public void cancel(String queryId, boolean publishEvent) throws InterruptedExcep } /** - * * Once the cancel request is validated, this method is called to actually cancel the query. * * @param queryId @@ -485,40 +377,24 @@ public void cancel(String queryId, String poolName, boolean publishEvent) throws List nextCalls = nextCallMap.get(queryId); if (nextCalls != null) { nextCalls.forEach(NextCall::cancel); - - // TODO: lock the cache entry and change state to canceled - // try to be nice and acquire the lock before updating the status - QueryStorageLock statusLock = queryStorageCache.getQueryStatusLock(queryUUID); - if (statusLock.tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { - try { - // update query state to CANCELED - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryUUID); - queryStatus.setQueryState(CANCELED); - queryStorageCache.updateQueryStatus(queryStatus); - - if (poolName == null) { - poolName = queryStatus.getQueryKey().getQueryPool().getName(); - } - } finally { - statusLock.unlock(); - } - } else { - // TODO: Instead of throwing an exception, do we want to be mean here and update the state to canceled without acquiring a lock? Probably yes... - throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to acquire lock on query"); - } } if (publishEvent) { + // only the initial event publisher should update the status + QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(queryUUID, status -> { + // update query state to CANCELED + status.setQueryState(CANCELED); + }); + + if (poolName == null) { + poolName = queryStatus.getQueryKey().getQueryPool().getName(); + } + QueryRequest cancelRequest = QueryRequest.cancel(queryId); // publish a cancel event to all of the query services publishSelfEvent(cancelRequest); - // get the pool name if we don't have it already - if (poolName == null) { - poolName = queryStorageCache.getQueryStatus(queryUUID).getQueryKey().getQueryPool().getName(); - } - // publish a cancel event to the executor pool publishExecutorEvent(cancelRequest, poolName); } @@ -530,7 +406,6 @@ public void cancel(String queryId, String poolName, boolean publishEvent) throws * * @param queryId * @param currentUser - * * @throws QueryException */ public void close(String queryId, ProxiedUserDetails currentUser) throws QueryException { @@ -539,7 +414,7 @@ public void close(String queryId, ProxiedUserDetails currentUser) throws QueryEx QueryStatus queryStatus = validateRequest(queryId, currentUser); // close the query - close(queryId, queryStatus.getQueryKey().getQueryPool().getName(), true); + close(queryId, queryStatus.getQueryKey().getQueryPool().getName()); } catch (QueryException e) { throw e; } catch (Exception e) { @@ -550,63 +425,41 @@ public void close(String queryId, ProxiedUserDetails currentUser) throws QueryEx } /** - * * Once the close request is validated, this method is called to actually close the query. * * @param queryId * The id of the query to close - * @param publishEvent - * If true, the close request will be published on the bus * @throws InterruptedException * @throws QueryException */ - public void close(String queryId, boolean publishEvent) throws InterruptedException, QueryException { - close(queryId, null, publishEvent); + public void close(String queryId) throws InterruptedException, QueryException { + close(queryId, (String) null); } /** - * * Once the close request is validated, this method is called to actually close the query. * * @param queryId * The id of the query to close * @param poolName * The pool name for this query - * @param publishEvent - * If true, the close request will be published on the bus * @throws InterruptedException * @throws QueryException */ - public void close(String queryId, String poolName, boolean publishEvent) throws InterruptedException, QueryException { + public void close(String queryId, String poolName) throws InterruptedException, QueryException { UUID queryUUID = UUID.fromString(queryId); - QueryStorageLock statusLock = queryStorageCache.getQueryStatusLock(queryUUID); - if (statusLock.tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { - try { - // update query state to CLOSED - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryUUID); - queryStatus.setQueryState(CLOSED); - queryStorageCache.updateQueryStatus(queryStatus); - - if (poolName == null) { - poolName = queryStatus.getQueryKey().getQueryPool().getName(); - } - } finally { - statusLock.unlock(); - } - } else { - // TODO: Instead of throwing an exception, do we want to be mean here and update the state to canceled without acquiring a lock? Probably yes... - throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to acquire lock on query"); - } + QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(queryUUID, status -> { + // update query state to CLOSED + status.setQueryState(CLOSED); + }); - if (publishEvent) { - if (poolName == null) { - poolName = queryStorageCache.getQueryStatus(queryUUID).getQueryKey().getQueryPool().getName(); - } - - // publish a close event to the executor pool - publishExecutorEvent(QueryRequest.close(queryId), poolName); + if (poolName == null) { + poolName = queryStatus.getQueryKey().getQueryPool().getName(); } + + // publish a close event to the executor pool + publishExecutorEvent(QueryRequest.close(queryId), poolName); } private QueryStatus validateRequest(String queryId, ProxiedUserDetails currentUser) throws QueryException { @@ -635,12 +488,8 @@ public void handleRemoteRequest(QueryRequest queryRequest) { log.trace("Received remote cancel request."); cancel(queryRequest.getQueryId(), false); break; - case CLOSE: - log.trace("Received remote close request."); - close(queryRequest.getQueryId(), false); - break; default: - log.debug("Unknown remote query request method: {}", queryRequest.getMethod()); + log.debug("No handling specified for remote query request method: {}", queryRequest.getMethod()); } } catch (Exception e) { log.error("Remote request failed:" + queryRequest); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 0af575fe..9a578fa7 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -57,7 +57,7 @@ public class NextCall implements Callable> { private final BaseQueryMetric metric; - private NextCall(Builder builder) throws QueryException { + private NextCall(Builder builder) { this.nextCallProperties = builder.nextCallProperties; this.expirationProperties = builder.expirationProperties; this.queryQueueManager = builder.queryQueueManager; @@ -200,9 +200,7 @@ private boolean isFinished(String queryId) { log.info("Query [{}]: max call time reached, returning existing results: {} of {} results in {}ms", queryId, results.size(), maxResultsPerPage, callTimeMillis); - if (!results.isEmpty()) { - status = ResultsPage.Status.PARTIAL; - } + status = ResultsPage.Status.PARTIAL; // TODO: Figure out query metrics metric.setLifecycle(BaseQueryMetric.Lifecycle.NEXTTIMEOUT); @@ -257,10 +255,6 @@ public void cancel() { this.canceled = true; } - public boolean isCanceled() { - return canceled; - } - public Future> getFuture() { return future; } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java new file mode 100644 index 00000000..a6215fc1 --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java @@ -0,0 +1,75 @@ +package datawave.microservice.query.status; + +import datawave.microservice.common.storage.QueryStatus; +import datawave.microservice.common.storage.QueryStorageCache; +import datawave.microservice.common.storage.QueryStorageLock; +import datawave.microservice.query.config.QueryProperties; +import datawave.webservice.query.exception.DatawaveErrorCode; +import datawave.webservice.query.exception.QueryException; + +import java.util.UUID; + +import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CREATED; + +public class QueryStatusUpdateHelper { + + private final QueryProperties queryProperties; + private final QueryStorageCache queryStorageCache; + + public QueryStatusUpdateHelper(QueryProperties queryProperties, QueryStorageCache queryStorageCache) { + this.queryProperties = queryProperties; + this.queryStorageCache = queryStorageCache; + } + + public void claimConcurrentNext(QueryStatus queryStatus) throws QueryException { + // we can only call next on a created query + if (queryStatus.getQueryState() == CREATED) { + // increment the concurrent next count + if (queryStatus.getConcurrentNextCount() < queryProperties.getNextCall().getConcurrency()) { + queryStatus.setConcurrentNextCount(queryStatus.getConcurrentNextCount() + 1); + + // update the last used datetime for the query + queryStatus.setLastUsedMillis(System.currentTimeMillis()); + } else { + throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, + "Concurrent next call limit reached: " + queryProperties.getNextCall().getConcurrency()); + } + } else { + throw new QueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH, "Unable to find query status in cache with CREATED state."); + } + } + + public void releaseConcurrentNext(QueryStatus queryStatus) throws QueryException { + // decrement the concurrent next count + if (queryStatus.getConcurrentNextCount() > 0) { + queryStatus.setConcurrentNextCount(queryStatus.getConcurrentNextCount() - 1); + + // update the last used datetime for the query + queryStatus.setLastUsedMillis(System.currentTimeMillis()); + } else { + throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, + "Concurrent next count can't be decremented: " + queryStatus.getConcurrentNextCount()); + } + } + + public QueryStatus lockedUpdate(UUID queryUUID, StatusUpdater updater) throws QueryException, InterruptedException { + QueryStatus queryStatus = null; + QueryStorageLock statusLock = queryStorageCache.getQueryStatusLock(queryUUID); + if (statusLock.tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { + try { + queryStatus = queryStorageCache.getQueryStatus(queryUUID); + if (queryStatus != null) { + updater.apply(queryStatus); + queryStorageCache.updateQueryStatus(queryStatus); + } else { + throw new QueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH, "Unable to find query status in cache."); + } + } finally { + statusLock.unlock(); + } + } else { + updater.onLockFailed(); + } + return queryStatus; + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/StatusUpdater.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/StatusUpdater.java new file mode 100644 index 00000000..27c16b89 --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/StatusUpdater.java @@ -0,0 +1,13 @@ +package datawave.microservice.query.status; + +import datawave.microservice.common.storage.QueryStatus; +import datawave.webservice.query.exception.DatawaveErrorCode; +import datawave.webservice.query.exception.QueryException; + +public interface StatusUpdater { + void apply(QueryStatus queryStatus) throws QueryException; + + default void onLockFailed() throws QueryException { + throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to acquire lock on query"); + } +} From 27a600031ade4775e9d6587f8a371a1481e816d5 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 4 Jun 2021 13:16:26 -0400 Subject: [PATCH 060/218] Updated query monitoring to use the user's page timeout, and added page timeouts to the configuration. Performed some code cleanup as well. --- .../config/QueryExpirationProperties.java | 48 ++++++++++ .../query/config/QueryProperties.java | 2 +- .../microservice/query/QueryController.java | 4 - .../query/QueryManagementService.java | 96 +++++-------------- .../query/monitor/MonitorTask.java | 4 +- .../query/monitor/config/MonitorConfig.java | 2 +- .../microservice/query/runner/NextCall.java | 29 ++++-- 7 files changed, 97 insertions(+), 88 deletions(-) diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java index 8ed9bf8d..bb45574b 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java @@ -17,6 +17,14 @@ public class QueryExpirationProperties { @NotNull private TimeUnit callTimeUnit = TimeUnit.MINUTES; @Positive + private long pageMinTimeout = 1; + @NotNull + private TimeUnit pageMinTimeUnit = TimeUnit.MINUTES; + @Positive + private long pageMaxTimeout = 60; + @NotNull + private TimeUnit pageMaxTimeUnit = TimeUnit.MINUTES; + @Positive private long shortCircuitCheckTime = callTimeout / 2; @NotNull private TimeUnit shortCircuitCheckTimeUnit = TimeUnit.MINUTES; @@ -65,6 +73,46 @@ public void setCallTimeUnit(TimeUnit callTimeUnit) { this.callTimeUnit = callTimeUnit; } + public long getPageMinTimeout() { + return pageMinTimeout; + } + + public long getPageMinTimeoutMillis() { + return pageMinTimeUnit.toMillis(pageMinTimeout); + } + + public void setPageMinTimeout(long pageMinTimeout) { + this.pageMinTimeout = pageMinTimeout; + } + + public TimeUnit getPageMinTimeUnit() { + return pageMinTimeUnit; + } + + public void setPageMinTimeUnit(TimeUnit pageMinTimeUnit) { + this.pageMinTimeUnit = pageMinTimeUnit; + } + + public long getPageMaxTimeout() { + return pageMaxTimeout; + } + + public long getPageMaxTimeoutMillis() { + return pageMaxTimeUnit.toMillis(pageMaxTimeout); + } + + public void setPageMaxTimeout(long pageMaxTimeout) { + this.pageMaxTimeout = pageMaxTimeout; + } + + public TimeUnit getPageMaxTimeUnit() { + return pageMaxTimeUnit; + } + + public void setPageMaxTimeUnit(TimeUnit pageMaxTimeUnit) { + this.pageMaxTimeUnit = pageMaxTimeUnit; + } + public long getShortCircuitCheckTime() { return shortCircuitCheckTime; } diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java index 5aa67a8a..c1a72c59 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java @@ -25,7 +25,7 @@ public class QueryProperties { @NotEmpty private String executorServiceName = "executor"; - private QueryExpirationProperties expiration; + private QueryExpirationProperties expiration = new QueryExpirationProperties(); private NextCallProperties nextCall = new NextCallProperties(); private DefaultParameterProperties defaultParams = new DefaultParameterProperties(); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 50acf9a3..b42d2c92 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -7,8 +7,6 @@ import datawave.webservice.result.BaseQueryResponse; import datawave.webservice.result.GenericResponse; import datawave.webservice.result.VoidResponse; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.util.MultiValueMap; @@ -21,8 +19,6 @@ @RestController @RequestMapping(path = "/v1", produces = MediaType.APPLICATION_JSON_VALUE) public class QueryController { - private final Logger log = LoggerFactory.getLogger(this.getClass()); - private final QueryManagementService queryManagementService; public QueryController(QueryManagementService queryManagementService) { diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 5022583d..6af0773f 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -55,6 +55,7 @@ import java.util.List; import java.util.UUID; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CANCELED; import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CLOSED; @@ -87,10 +88,6 @@ public class QueryManagementService implements QueryRequestHandler { private final QueryStatusUpdateHelper queryStatusUpdateHelper; private final MultiValueMap nextCallMap = new LinkedMultiValueMap<>(); - // TODO: JWO: Pull these from configuration instead - private final int PAGE_TIMEOUT_MIN = 1; - private final int PAGE_TIMEOUT_MAX = 60; - public QueryManagementService(QueryProperties queryProperties, ApplicationContext appCtx, BusProperties busProperties, QueryParameters queryParameters, SecurityMarking securityMarking, QueryLogicFactory queryLogicFactory, QueryMetricFactory queryMetricFactory, ResponseObjectFactory responseObjectFactory, QueryStorageCache queryStorageCache, QueryQueueManager queryQueueManager, @@ -146,7 +143,7 @@ public TaskKey define(String queryLogicName, MultiValueMap parame // persist the query w/ query id in the query storage cache // TODO: JWO: storeQuery assumes that this is a 'create' call, but this is a 'define' call. // @formatter:off - TaskKey taskKey = queryStorageCache.defineQuery( + return queryStorageCache.defineQuery( new QueryPool(getPoolName()), createQuery(queryLogicName, parameters, userDn, currentUser.getDNs()), AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), currentUser), @@ -156,8 +153,6 @@ public TaskKey define(String queryLogicName, MultiValueMap parame // TODO: JWO: Figure out how to make query tracing work with our new architecture. Datawave issue #1155 // TODO: JWO: Figure out how to make query metrics work with our new architecture. Datawave issue #1156 - - return taskKey; } catch (Exception e) { log.error("Unknown error storing query", e); throw new BadRequestQueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR, e); @@ -278,13 +273,12 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th status.setLastPageNumber(status.getLastPageNumber() + 1); status.setNumResultsReturned(status.getNumResultsReturned() + resultsPage.getResults().size()); }); + success = true; response.setHasResults(true); response.setPageNumber(queryStatus.getLastPageNumber()); response.setLogicName(queryLogicName); response.setQueryId(queryId); - - success = true; return response; } else { if (nextCall.getMetric().getLifecycle() == BaseQueryMetric.Lifecycle.NEXTTIMEOUT) { @@ -331,10 +325,10 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th public void cancel(String queryId, ProxiedUserDetails currentUser) throws QueryException { try { // make sure the query is valid, and the user can act on it - QueryStatus queryStatus = validateRequest(queryId, currentUser); + validateRequest(queryId, currentUser); // cancel the query - cancel(queryId, queryStatus.getQueryKey().getQueryPool().getName(), true); + cancel(queryId, true); } catch (QueryException e) { throw e; } catch (Exception e) { @@ -355,24 +349,6 @@ public void cancel(String queryId, ProxiedUserDetails currentUser) throws QueryE * @throws QueryException */ public void cancel(String queryId, boolean publishEvent) throws InterruptedException, QueryException { - cancel(queryId, null, publishEvent); - } - - /** - * Once the cancel request is validated, this method is called to actually cancel the query. - * - * @param queryId - * The id of the query to cancel - * @param poolName - * The pool name for this query - * @param publishEvent - * If true, the cancel request will be published on the bus - * @throws InterruptedException - * @throws QueryException - */ - public void cancel(String queryId, String poolName, boolean publishEvent) throws InterruptedException, QueryException { - UUID queryUUID = UUID.fromString(queryId); - // if we have an active next call for this query locally, cancel it List nextCalls = nextCallMap.get(queryId); if (nextCalls != null) { @@ -381,22 +357,18 @@ public void cancel(String queryId, String poolName, boolean publishEvent) throws if (publishEvent) { // only the initial event publisher should update the status - QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(queryUUID, status -> { + QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(UUID.fromString(queryId), status -> { // update query state to CANCELED status.setQueryState(CANCELED); }); - if (poolName == null) { - poolName = queryStatus.getQueryKey().getQueryPool().getName(); - } - QueryRequest cancelRequest = QueryRequest.cancel(queryId); // publish a cancel event to all of the query services publishSelfEvent(cancelRequest); // publish a cancel event to the executor pool - publishExecutorEvent(cancelRequest, poolName); + publishExecutorEvent(cancelRequest, queryStatus.getQueryKey().getQueryPool().getName()); } } @@ -411,10 +383,10 @@ public void cancel(String queryId, String poolName, boolean publishEvent) throws public void close(String queryId, ProxiedUserDetails currentUser) throws QueryException { try { // make sure the query is valid, and the user can act on it - QueryStatus queryStatus = validateRequest(queryId, currentUser); + validateRequest(queryId, currentUser); // close the query - close(queryId, queryStatus.getQueryKey().getQueryPool().getName()); + close(queryId); } catch (QueryException e) { throw e; } catch (Exception e) { @@ -433,36 +405,16 @@ public void close(String queryId, ProxiedUserDetails currentUser) throws QueryEx * @throws QueryException */ public void close(String queryId) throws InterruptedException, QueryException { - close(queryId, (String) null); - } - - /** - * Once the close request is validated, this method is called to actually close the query. - * - * @param queryId - * The id of the query to close - * @param poolName - * The pool name for this query - * @throws InterruptedException - * @throws QueryException - */ - public void close(String queryId, String poolName) throws InterruptedException, QueryException { - UUID queryUUID = UUID.fromString(queryId); - - QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(queryUUID, status -> { + QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(UUID.fromString(queryId), status -> { // update query state to CLOSED status.setQueryState(CLOSED); }); - if (poolName == null) { - poolName = queryStatus.getQueryKey().getQueryPool().getName(); - } - // publish a close event to the executor pool - publishExecutorEvent(QueryRequest.close(queryId), poolName); + publishExecutorEvent(QueryRequest.close(queryId), queryStatus.getQueryKey().getQueryPool().getName()); } - private QueryStatus validateRequest(String queryId, ProxiedUserDetails currentUser) throws QueryException { + private void validateRequest(String queryId, ProxiedUserDetails currentUser) throws QueryException { // does the query exist? QueryStatus queryStatus = queryStorageCache.getQueryStatus(UUID.fromString(queryId)); if (queryStatus == null) { @@ -476,20 +428,16 @@ private QueryStatus validateRequest(String queryId, ProxiedUserDetails currentUs if (!query.getOwner().equals(userId)) { throw new UnauthorizedQueryException(DatawaveErrorCode.QUERY_OWNER_MISMATCH, MessageFormat.format("{0} != {1}", userId, query.getOwner())); } - - return queryStatus; } @Override public void handleRemoteRequest(QueryRequest queryRequest) { try { - switch (queryRequest.getMethod()) { - case CANCEL: - log.trace("Received remote cancel request."); - cancel(queryRequest.getQueryId(), false); - break; - default: - log.debug("No handling specified for remote query request method: {}", queryRequest.getMethod()); + if (queryRequest.getMethod() == QueryRequest.Method.CANCEL) { + log.trace("Received remote cancel request."); + cancel(queryRequest.getQueryId(), false); + } else { + log.debug("No handling specified for remote query request method: {}", queryRequest.getMethod()); } } catch (Exception e) { log.error("Remote request failed:" + queryRequest); @@ -633,15 +581,17 @@ protected void validateParameters(String queryLogicName, MultiValueMap PAGE_TIMEOUT_MAX)) { + long pageMinTimeoutMillis = queryProperties.getExpiration().getPageMinTimeoutMillis(); + long pageMaxTimeoutMillis = queryProperties.getExpiration().getPageMaxTimeoutMillis(); + long pageTimeoutMillis = TimeUnit.MINUTES.toMillis(queryParameters.getPageTimeout()); + if (queryParameters.getPageTimeout() != -1 && (pageTimeoutMillis < pageMinTimeoutMillis || pageTimeoutMillis > pageMaxTimeoutMillis)) { log.error("Invalid page timeout: " + queryParameters.getPageTimeout()); throw new BadRequestQueryException(DatawaveErrorCode.INVALID_PAGE_TIMEOUT); } @@ -727,7 +677,7 @@ protected void setInternalAuditParameters(String queryLogicName, String userDn, } private String writeValueAsString(Object object) { - String stringValue = ""; + String stringValue; try { stringValue = mapper.writeValueAsString(object); } catch (JsonProcessingException e) { diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java index 4bfab83f..15f24f77 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java @@ -69,7 +69,7 @@ private void monitor(long currentTimeMillis) { cancelQuery(queryId); } else if (status.isProgressIdle(currentTimeMillis, expirationProperties.getIdleTimeoutMillis())) { // if progress hasn't been made for the query in a while, apply the shock paddles - defibrilateQuery(queryId, status.getQueryKey().getQueryPool().getName()); + defibrillateQuery(queryId, status.getQueryKey().getQueryPool().getName()); } } else { if (status.isInactive(currentTimeMillis, monitorProperties.getInactiveQueryTimeToLiveMillis())) { @@ -90,7 +90,7 @@ private void cancelQuery(String queryId) { } } - private void defibrilateQuery(String queryId, String queryPool) { + private void defibrillateQuery(String queryId, String queryPool) { // publish a next event to the executor pool queryManagementService.publishNextEvent(queryId, queryPool); } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java index 206d5a56..01dc9949 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java @@ -27,7 +27,7 @@ public class MonitorConfig { @Bean public MonitorStatusCache monitorStatusCache(CacheInspector cacheInspector, CacheManager cacheManager) { log.debug("Using " + cacheManager.getClass() + " for caching"); - LockableCacheInspector lockableCacheInspector = null; + LockableCacheInspector lockableCacheInspector; if (cacheManager instanceof HazelcastCacheManager) lockableCacheInspector = new LockableHazelcastCacheInspector(cacheManager); else diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 9a578fa7..238a0387 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -24,12 +24,12 @@ import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; public class NextCall implements Callable> { private final Logger log = LoggerFactory.getLogger(this.getClass()); private final NextCallProperties nextCallProperties; - private final QueryExpirationProperties expirationProperties; private final QueryQueueManager queryQueueManager; private final QueryStorageCache queryStorageCache; private final String queryId; @@ -38,6 +38,10 @@ public class NextCall implements Callable> { private volatile boolean canceled = false; private volatile Future> future = null; + private final long callTimeoutMillis; + private final long shortCircuitCheckTimeMillis; + private final long shortCircuitTimeoutMillis; + private final int userResultsPerPage; private final boolean maxResultsOverridden; private final long maxResultsOverride; @@ -59,13 +63,24 @@ public class NextCall implements Callable> { private NextCall(Builder builder) { this.nextCallProperties = builder.nextCallProperties; - this.expirationProperties = builder.expirationProperties; this.queryQueueManager = builder.queryQueueManager; this.queryStorageCache = builder.queryStorageCache; this.queryId = builder.queryId; this.identifier = builder.identifier; QueryStatus status = getQueryStatus(); + long pageTimeoutMillis = TimeUnit.MINUTES.toMillis(status.getQuery().getPageTimeout()); + if (pageTimeoutMillis >= builder.expirationProperties.getPageMinTimeoutMillis() + && pageTimeoutMillis <= builder.expirationProperties.getPageMaxTimeoutMillis()) { + callTimeoutMillis = pageTimeoutMillis; + shortCircuitCheckTimeMillis = callTimeoutMillis / 2; + shortCircuitTimeoutMillis = Math.round(0.97 * callTimeoutMillis); + } else { + callTimeoutMillis = builder.expirationProperties.getCallTimeoutMillis(); + shortCircuitCheckTimeMillis = builder.expirationProperties.getShortCircuitCheckTimeMillis(); + shortCircuitTimeoutMillis = builder.expirationProperties.getShortCircuitTimeoutMillis(); + } + this.userResultsPerPage = status.getQuery().getPagesize(); this.maxResultsOverridden = status.getQuery().isMaxResultsOverridden(); this.maxResultsOverride = status.getQuery().getMaxResultsOverride(); @@ -86,7 +101,7 @@ private NextCall(Builder builder) { } @Override - public ResultsPage call() throws Exception { + public ResultsPage call() { startTimeMillis = System.currentTimeMillis(); QueryQueueListener resultListener = queryQueueManager.createListener(identifier, queryId); @@ -217,8 +232,8 @@ private boolean shortCircuitTimeout(long callTimeMillis) { // only return prematurely if we have at least 1 result if (!results.isEmpty()) { // if after the page size short circuit check time - if (callTimeMillis >= expirationProperties.getShortCircuitCheckTimeMillis()) { - float percentTimeComplete = (float) callTimeMillis / (float) (expirationProperties.getCallTimeout()); + if (callTimeMillis >= shortCircuitCheckTimeMillis) { + float percentTimeComplete = (float) callTimeMillis / (float) (callTimeoutMillis); float percentResultsComplete = (float) results.size() / (float) maxResultsPerPage; // if the percent results complete is less than the percent time complete, then break out if (percentResultsComplete < percentTimeComplete) { @@ -227,7 +242,7 @@ private boolean shortCircuitTimeout(long callTimeMillis) { } // if after the page short circuit timeout, then break out - if (callTimeMillis >= expirationProperties.getShortCircuitTimeoutMillis()) { + if (callTimeMillis >= shortCircuitTimeoutMillis) { timeout = true; } } @@ -236,7 +251,7 @@ private boolean shortCircuitTimeout(long callTimeMillis) { } private boolean callExpiredTimeout(long callTimeMillis) { - return callTimeMillis >= expirationProperties.getCallTimeoutMillis(); + return callTimeMillis >= callTimeoutMillis; } private QueryStatus getQueryStatus() { From 0b8ba9c24cf4b655a6a746b07059af11340971ff Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 4 Jun 2021 14:06:45 -0400 Subject: [PATCH 061/218] added createAndNext call to the query service controller. --- query-microservices/query-service/README.md | 2 +- .../microservice/query/QueryController.java | 9 +++ .../query/QueryManagementService.java | 76 +++++++++++++++---- 3 files changed, 70 insertions(+), 17 deletions(-) diff --git a/query-microservices/query-service/README.md b/query-microservices/query-service/README.md index 93f95764..447918c2 100644 --- a/query-microservices/query-service/README.md +++ b/query-microservices/query-service/README.md @@ -20,7 +20,7 @@ query capabilities. | | `POST` | /{queryLogic}/predict | Generate a query prediction using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | | | `POST` | /{queryLogic}/async/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | | | `PUT` | /{id}/reset | Resets the specified query | [QueryId] | N/A | [VoidResponse] | -| | `POST` | /{queryLogic}/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | +| ✓ | `POST` | /{queryLogic}/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | | | `POST` | /{queryLogic}/async/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | | | `GET` | /lookupContentUUID/{uuidType}/{uuid} | Returns content associated with the given UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] or [StreamingOutput] | | | `POST` | /lookupContentUUID | Returns content associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] or [StreamingOutput] | diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index b42d2c92..90c653cd 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -52,6 +52,15 @@ public GenericResponse create(@PathVariable(name = "queryLogic") String return resp; } + @Timed(name = "dw.query.createAndNext", absolute = true) + @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE_AND_NEXT) + @RequestMapping(path = "{queryLogic}/createAndNext", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public BaseQueryResponse createQueryAndNext(@PathVariable(name = "queryLogic") String queryLogic, @RequestParam MultiValueMap parameters, + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + return queryManagementService.createAndNext(queryLogic, parameters, currentUser); + } + @Timed(name = "dw.query.next", absolute = true) @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.NEXT) @RequestMapping(path = "{queryId}/next", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 6af0773f..f44ad075 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -52,6 +52,7 @@ import java.net.UnknownHostException; import java.text.MessageFormat; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.UUID; import java.util.concurrent.ExecutionException; @@ -217,6 +218,34 @@ public TaskKey create(String queryLogicName, MultiValueMap parame } } + /** + * Creates a datawave query and calls next. + *

+ * Validates the query parameters using the base validation, query logic-specific validation, and markings validation. If the parameters are valid, the + * query will be stored in the query storage cache where it can be acted upon. + * + * Once created, gets the next page of results from the query object. The response object type is dynamic, see the listQueryLogic operation to determine + * what the response type object will be. + * + * @param queryLogicName + * @param parameters + * @param currentUser + * @return + * @throws QueryException + */ + public BaseQueryResponse createAndNext(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) + throws QueryException { + try { + TaskKey taskKey = create(queryLogicName, parameters, currentUser); + return next(taskKey.getQueryId().toString(), currentUser.getPrimaryUser().getRoles()); + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error creating and nexting query", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error creating and nexting query query."); + } + } + /** * Gets the next page of results from the query object. If the object is no longer alive, meaning that the current session has expired, then this fail. The * response object type is dynamic, see the listQueryLogic operation to determine what the response type object will be. @@ -227,20 +256,41 @@ public TaskKey create(String queryLogicName, MultiValueMap parame * @throws QueryException */ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) throws QueryException { - UUID queryUUID = UUID.fromString(queryId); try { // make sure the query is valid, and the user can act on it validateRequest(queryId, currentUser); - // before we spin up a separate thread, make sure we are allowed to call next - QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(queryUUID, queryStatusUpdateHelper::claimConcurrentNext); - + return next(queryId, currentUser.getPrimaryUser().getRoles()); + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error getting next page for query " + queryId, e); + throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Unknown error getting next page for query " + queryId); + } + } + + /** + * Gets the next page of results from the query object. If the object is no longer alive, meaning that the current session has expired, then this fail. The + * response object type is dynamic, see the listQueryLogic operation to determine what the response type object will be. + * + * @param queryId + * @param userRoles + * @return + * @throws QueryException + */ + private BaseQueryResponse next(String queryId, Collection userRoles) throws Exception { + UUID queryUUID = UUID.fromString(queryId); + + // before we spin up a separate thread, make sure we are allowed to call next + boolean success = false; + QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(queryUUID, queryStatusUpdateHelper::claimConcurrentNext); + try { // publish a next event to the executor pool publishNextEvent(queryId, queryStatus.getQueryKey().getQueryPool().getName()); // get the query logic String queryLogicName = queryStatus.getQuery().getQueryLogicName(); - QueryLogic queryLogic = queryLogicFactory.getQueryLogic(queryStatus.getQuery().getQueryLogicName(), currentUser.getPrimaryUser().getRoles()); + QueryLogic queryLogic = queryLogicFactory.getQueryLogic(queryStatus.getQuery().getQueryLogicName(), userRoles); // @formatter:off final NextCall nextCall = new NextCall.Builder() @@ -254,7 +304,6 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th .build(); // @formatter:on - boolean success = false; nextCallMap.add(queryId, nextCall); try { // submit the next call to the executor @@ -300,17 +349,12 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th } finally { // remove this next call from the map, and decrement the next count for this query nextCallMap.get(queryId).remove(nextCall); - - // update query status if we failed - if (!success) { - queryStatusUpdateHelper.lockedUpdate(queryUUID, queryStatusUpdateHelper::releaseConcurrentNext); - } } - } catch (QueryException e) { - throw e; - } catch (Exception e) { - log.error("Unknown error getting next page for query " + queryId, e); - throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Unknown error getting next page for query " + queryId); + } finally { + // update query status if we failed + if (!success) { + queryStatusUpdateHelper.lockedUpdate(queryUUID, queryStatusUpdateHelper::releaseConcurrentNext); + } } } From 6ae8b6bae02df4eebcca47644b4e8c6540be99a7 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 4 Jun 2021 16:22:30 -0400 Subject: [PATCH 062/218] Added listQueryLogic, adminClose, and adminCancel to the query service controller. --- query-microservices/query-service/README.md | 6 +- .../microservice/query/QueryController.java | 45 ++-- .../query/QueryManagementService.java | 215 ++++++++++++++++-- .../query/monitor/MonitorTask.java | 32 +-- 4 files changed, 245 insertions(+), 53 deletions(-) diff --git a/query-microservices/query-service/README.md b/query-microservices/query-service/README.md index 447918c2..91b5b092 100644 --- a/query-microservices/query-service/README.md +++ b/query-microservices/query-service/README.md @@ -13,7 +13,7 @@ query capabilities. | Done? | Method | Operation | Description | Path Param | Request Body | Response Body | |:--------|:--------------|:-------------------------------------|:----------------------------------------------------------------------------------|:-------------------|:---------------------|:-----------------------------------------| -| | `GET` | /listQueryLogic | List QueryLogic types that are currently available | N/A | N/A | [QueryLogicResponse] | +| ✓ | `GET` | /listQueryLogic | List QueryLogic types that are currently available | N/A | N/A | [QueryLogicResponse] | | ✓ | `POST` | /{queryLogic}/define | Define a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | | ✓ | `POST` | /{queryLogic}/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | | | `POST` | /{queryLogic}/plan | Generate a query plan using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | @@ -31,9 +31,9 @@ query capabilities. | | `GET` | /{id}/async/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | | ✓ | `GET` | /{id}/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | | ✓ | `PUT` `POST` | /{id}/close | Closes the specified query | [QueryId] | N/A | [VoidResponse] | -| | `PUT` `POST` | /{id}/adminClose | Closes the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | `PUT` `POST` | /{id}/adminClose | Closes the specified query | [QueryId] | N/A | [VoidResponse] | | ✓ | `PUT` `POST` | /{id}/cancel | Cancels the specified query | [QueryId] | N/A | [VoidResponse] | -| | `PUT` `POST` | /{id}/adminCancel | Cancels the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | `PUT` `POST` | /{id}/adminCancel | Cancels the specified query | [QueryId] | N/A | [VoidResponse] | | | `GET` | /listAll | Returns a list of queries associated with the current user | N/A | N/A | [QueryImplListResponse] | | | `GET` | /{id} | Returns query info for the specified query | [QueryId] | N/A | [QueryImplListResponse] | | | `GET` | /list | Returns a list of queries associated with the current user with given name | N/A | [Name] | [QueryImplListResponse] | diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 90c653cd..336c7650 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -2,10 +2,10 @@ import com.codahale.metrics.annotation.Timed; import datawave.microservice.authorization.user.ProxiedUserDetails; -import datawave.microservice.common.storage.TaskKey; import datawave.microservice.query.web.annotation.EnrichQueryMetrics; import datawave.webservice.result.BaseQueryResponse; import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.QueryLogicResponse; import datawave.webservice.result.VoidResponse; import org.springframework.http.MediaType; import org.springframework.security.core.annotation.AuthenticationPrincipal; @@ -16,6 +16,8 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import javax.annotation.security.RolesAllowed; + @RestController @RequestMapping(path = "/v1", produces = MediaType.APPLICATION_JSON_VALUE) public class QueryController { @@ -25,17 +27,20 @@ public QueryController(QueryManagementService queryManagementService) { this.queryManagementService = queryManagementService; } + @Timed(name = "dw.query.listQueryLogic", absolute = true) + @RequestMapping(path = "listQueryLogic", method = {RequestMethod.GET}, + produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "text/html"}) + public QueryLogicResponse listQueryLogic() { + return queryManagementService.listQueryLogic(); + } + @Timed(name = "dw.query.defineQuery", absolute = true) @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE) @RequestMapping(path = "{queryLogic}/define", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public GenericResponse define(@PathVariable(name = "queryLogic") String queryLogic, @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { - TaskKey taskKey = queryManagementService.define(queryLogic, parameters, currentUser); - - GenericResponse resp = new GenericResponse<>(); - resp.setResult(taskKey.getQueryId().toString()); - return resp; + return queryManagementService.define(queryLogic, parameters, currentUser); } @Timed(name = "dw.query.createQuery", absolute = true) @@ -44,12 +49,7 @@ public GenericResponse define(@PathVariable(name = "queryLogic") String "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public GenericResponse create(@PathVariable(name = "queryLogic") String queryLogic, @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { - TaskKey taskKey = queryManagementService.create(queryLogic, parameters, currentUser); - - GenericResponse resp = new GenericResponse<>(); - resp.setHasResults(true); - resp.setResult(taskKey.getQueryId().toString()); - return resp; + return queryManagementService.create(queryLogic, parameters, currentUser); } @Timed(name = "dw.query.createAndNext", absolute = true) @@ -73,16 +73,29 @@ public BaseQueryResponse next(@PathVariable(name = "queryId") String queryId, @A @RequestMapping(path = "{queryId}/cancel", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public VoidResponse cancel(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { - queryManagementService.cancel(queryId, currentUser); - return null; + return queryManagementService.cancel(queryId, currentUser); + } + + @Timed(name = "dw.query.adminCancel", absolute = true) + @RolesAllowed({"Administrator", "JBossAdministrator"}) + @RequestMapping(path = "{queryId}/adminCancel", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", + "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse adminCancel(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + return queryManagementService.adminCancel(queryId, currentUser); } @Timed(name = "dw.query.close", absolute = true) @RequestMapping(path = "{queryId}/close", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public VoidResponse close(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { - queryManagementService.close(queryId, currentUser); - return null; + return queryManagementService.close(queryId, currentUser); } + @Timed(name = "dw.query.adminClose", absolute = true) + @RolesAllowed({"Administrator", "JBossAdministrator"}) + @RequestMapping(path = "{queryId}/adminClose", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", + "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse adminClose(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + return queryManagementService.adminClose(queryId, currentUser); + } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index f44ad075..71f514b1 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -24,6 +24,7 @@ import datawave.webservice.common.audit.AuditParameters; import datawave.webservice.common.audit.Auditor; import datawave.webservice.query.Query; +import datawave.webservice.query.QueryImpl; import datawave.webservice.query.cache.QueryMetricFactory; import datawave.webservice.query.cache.ResultsPage; import datawave.webservice.query.exception.BadRequestQueryException; @@ -35,8 +36,12 @@ import datawave.webservice.query.exception.UnauthorizedQueryException; import datawave.webservice.query.metric.BaseQueryMetric; import datawave.webservice.query.result.event.ResponseObjectFactory; +import datawave.webservice.query.result.logic.QueryLogicDescription; import datawave.webservice.query.util.QueryUncaughtExceptionHandler; import datawave.webservice.result.BaseQueryResponse; +import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.QueryLogicResponse; +import datawave.webservice.result.VoidResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.cloud.bus.BusProperties; @@ -48,12 +53,18 @@ import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; +import java.lang.reflect.Method; import java.net.InetAddress; import java.net.UnknownHostException; import java.text.MessageFormat; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Comparator; +import java.util.Date; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -117,6 +128,83 @@ public QueryManagementService(QueryProperties queryProperties, ApplicationContex this.queryStatusUpdateHelper = new QueryStatusUpdateHelper(this.queryProperties, this.queryStorageCache); } + /** + * List QueryLogic types that are currently available + * + * @return + */ + public QueryLogicResponse listQueryLogic() { + QueryLogicResponse response = new QueryLogicResponse(); + List> queryLogicList = queryLogicFactory.getQueryLogicList(); + List logicConfigurationList = new ArrayList<>(); + + // reference query necessary to avoid NPEs in getting the Transformer and BaseResponse + Query q = new QueryImpl(); + Date now = new Date(); + q.setExpirationDate(now); + q.setQuery("test"); + q.setQueryAuthorizations("ALL"); + + for (QueryLogic queryLogic : queryLogicList) { + try { + QueryLogicDescription logicDesc = new QueryLogicDescription(queryLogic.getLogicName()); + logicDesc.setAuditType(queryLogic.getAuditType(null).toString()); + logicDesc.setLogicDescription(queryLogic.getLogicDescription()); + + Set optionalQueryParameters = queryLogic.getOptionalQueryParameters(); + if (optionalQueryParameters != null) { + logicDesc.setSupportedParams(new ArrayList<>(optionalQueryParameters)); + } + Set requiredQueryParameters = queryLogic.getRequiredQueryParameters(); + if (requiredQueryParameters != null) { + logicDesc.setRequiredParams(new ArrayList<>(requiredQueryParameters)); + } + Set exampleQueries = queryLogic.getExampleQueries(); + if (exampleQueries != null) { + logicDesc.setExampleQueries(new ArrayList<>(exampleQueries)); + } + Set requiredRoles = queryLogic.getRequiredRoles(); + if (requiredRoles != null) { + List requiredRolesList = new ArrayList<>(queryLogic.getRequiredRoles()); + logicDesc.setRequiredRoles(requiredRolesList); + } + + try { + logicDesc.setResponseClass(queryLogic.getResponseClass(q)); + } catch (QueryException e) { + log.error("Unable to get response class for query logic: " + queryLogic.getLogicName(), e); + response.addException(e); + logicDesc.setResponseClass("unknown"); + } + + List querySyntax = new ArrayList<>(); + try { + Method m = queryLogic.getClass().getMethod("getQuerySyntaxParsers"); + Object result = m.invoke(queryLogic); + if (result instanceof Map) { + Map map = (Map) result; + for (Object o : map.keySet()) + querySyntax.add(o.toString()); + } + } catch (Exception e) { + log.warn("Unable to get query syntax for query logic: " + queryLogic.getClass().getCanonicalName()); + } + if (querySyntax.isEmpty()) { + querySyntax.add("CUSTOM"); + } + logicDesc.setQuerySyntax(querySyntax); + + logicConfigurationList.add(logicDesc); + } catch (Exception e) { + log.error("Error setting query logic description", e); + } + } + logicConfigurationList.sort(Comparator.comparing(QueryLogicDescription::getName)); + response.setQueryLogicList(logicConfigurationList); + + return response; + } + /** * Defines a datawave query. *

@@ -129,7 +217,8 @@ public QueryManagementService(QueryProperties queryProperties, ApplicationContex * @return * @throws QueryException */ - public TaskKey define(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + public GenericResponse define(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) + throws QueryException { // validate query and get a query logic QueryLogic queryLogic = validateQuery(queryLogicName, parameters, currentUser); @@ -144,7 +233,7 @@ public TaskKey define(String queryLogicName, MultiValueMap parame // persist the query w/ query id in the query storage cache // TODO: JWO: storeQuery assumes that this is a 'create' call, but this is a 'define' call. // @formatter:off - return queryStorageCache.defineQuery( + TaskKey taskKey = queryStorageCache.defineQuery( new QueryPool(getPoolName()), createQuery(queryLogicName, parameters, userDn, currentUser.getDNs()), AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), currentUser), @@ -154,6 +243,10 @@ public TaskKey define(String queryLogicName, MultiValueMap parame // TODO: JWO: Figure out how to make query tracing work with our new architecture. Datawave issue #1155 // TODO: JWO: Figure out how to make query metrics work with our new architecture. Datawave issue #1156 + + GenericResponse response = new GenericResponse<>(); + response.setResult(taskKey.getQueryId().toString()); + return response; } catch (Exception e) { log.error("Unknown error storing query", e); throw new BadRequestQueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR, e); @@ -172,7 +265,8 @@ public TaskKey define(String queryLogicName, MultiValueMap parame * @return * @throws QueryException */ - public TaskKey create(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + public GenericResponse create(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) + throws QueryException { try { // validate query and get a query logic QueryLogic queryLogic = validateQuery(queryLogicName, parameters, currentUser); @@ -205,7 +299,10 @@ public TaskKey create(String queryLogicName, MultiValueMap parame // TODO: JWO: Figure out how to make query metrics work with our new architecture. Datawave issue #1156 - return taskKey; + GenericResponse response = new GenericResponse<>(); + response.setHasResults(true); + response.setResult(taskKey.getQueryId().toString()); + return response; } catch (Exception e) { log.error("Unknown error storing query", e); throw new BadRequestQueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR, e); @@ -236,8 +333,8 @@ public TaskKey create(String queryLogicName, MultiValueMap parame public BaseQueryResponse createAndNext(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { try { - TaskKey taskKey = create(queryLogicName, parameters, currentUser); - return next(taskKey.getQueryId().toString(), currentUser.getPrimaryUser().getRoles()); + String queryId = create(queryLogicName, parameters, currentUser).getResult(); + return next(queryId, currentUser.getPrimaryUser().getRoles()); } catch (QueryException e) { throw e; } catch (Exception e) { @@ -364,15 +461,49 @@ private BaseQueryResponse next(String queryId, Collection userRoles) thr * * @param queryId * @param currentUser + * @return + * @throws QueryException + */ + public VoidResponse cancel(String queryId, ProxiedUserDetails currentUser) throws QueryException { + return cancel(queryId, currentUser, false); + } + + /** + * Releases the resources associated with this query. Any currently running calls to 'next' on the query will be stopped. Calls to 'next' after a 'cancel' + * will start over at page 1. + * + * @param queryId + * @param currentUser + * @return * @throws QueryException */ - public void cancel(String queryId, ProxiedUserDetails currentUser) throws QueryException { + public VoidResponse adminCancel(String queryId, ProxiedUserDetails currentUser) throws QueryException { + return cancel(queryId, currentUser, true); + } + + private VoidResponse cancel(String queryId, ProxiedUserDetails currentUser, boolean adminOverride) throws QueryException { try { // make sure the query is valid, and the user can act on it - validateRequest(queryId, currentUser); + QueryStatus queryStatus = validateRequest(queryId, currentUser, adminOverride); - // cancel the query - cancel(queryId, true); + VoidResponse response = new VoidResponse(); + switch (queryStatus.getQueryState()) { + case DEFINED: + case CREATED: + // close the query + cancel(queryId, true); + response.addMessage(queryId + " canceled."); + break; + // TODO: Should we throw an exception for these cases? + case CLOSED: + case CANCELED: + case FAILED: + response.addMessage(queryId + " was not canceled because it is not running."); + break; + default: + throw new IllegalStateException("Unexpected query state: " + queryStatus.getQueryState()); + } + return response; } catch (QueryException e) { throw e; } catch (Exception e) { @@ -422,15 +553,49 @@ public void cancel(String queryId, boolean publishEvent) throws InterruptedExcep * * @param queryId * @param currentUser + * @return * @throws QueryException */ - public void close(String queryId, ProxiedUserDetails currentUser) throws QueryException { + public VoidResponse close(String queryId, ProxiedUserDetails currentUser) throws QueryException { + return close(queryId, currentUser, false); + } + + /** + * Releases the resources associated with this query. Any currently running calls to 'next' on the query will continue until they finish. Calls to 'next' + * after a 'close' will start over at page 1. + * + * @param queryId + * @param currentUser + * @return + * @throws QueryException + */ + public VoidResponse adminClose(String queryId, ProxiedUserDetails currentUser) throws QueryException { + return close(queryId, currentUser, true); + } + + private VoidResponse close(String queryId, ProxiedUserDetails currentUser, boolean adminOverride) throws QueryException { try { // make sure the query is valid, and the user can act on it - validateRequest(queryId, currentUser); + QueryStatus queryStatus = validateRequest(queryId, currentUser, adminOverride); - // close the query - close(queryId); + VoidResponse response = new VoidResponse(); + switch (queryStatus.getQueryState()) { + case DEFINED: + case CREATED: + // close the query + close(queryId); + response.addMessage(queryId + " closed."); + break; + // TODO: Should we throw an exception for these cases? + case CLOSED: + case CANCELED: + case FAILED: + response.addMessage(queryId + " was not closed because it is not running."); + break; + default: + throw new IllegalStateException("Unexpected query state: " + queryStatus.getQueryState()); + } + return response; } catch (QueryException e) { throw e; } catch (Exception e) { @@ -458,20 +623,28 @@ public void close(String queryId) throws InterruptedException, QueryException { publishExecutorEvent(QueryRequest.close(queryId), queryStatus.getQueryKey().getQueryPool().getName()); } - private void validateRequest(String queryId, ProxiedUserDetails currentUser) throws QueryException { + private QueryStatus validateRequest(String queryId, ProxiedUserDetails currentUser) throws QueryException { + return validateRequest(queryId, currentUser, false); + } + + private QueryStatus validateRequest(String queryId, ProxiedUserDetails currentUser, boolean adminOverride) throws QueryException { // does the query exist? QueryStatus queryStatus = queryStorageCache.getQueryStatus(UUID.fromString(queryId)); if (queryStatus == null) { throw new NotFoundQueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH, MessageFormat.format("{0}", queryId)); } - // TODO: Check to see if this is an admin user - // does the current user own this query? - String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); - Query query = queryStatus.getQuery(); - if (!query.getOwner().equals(userId)) { - throw new UnauthorizedQueryException(DatawaveErrorCode.QUERY_OWNER_MISMATCH, MessageFormat.format("{0} != {1}", userId, query.getOwner())); + // admins requests can operate on any query, regardless of ownership + if (!adminOverride) { + // does the current user own this query? + String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + Query query = queryStatus.getQuery(); + if (!query.getOwner().equals(userId)) { + throw new UnauthorizedQueryException(DatawaveErrorCode.QUERY_OWNER_MISMATCH, MessageFormat.format("{0} != {1}", userId, query.getOwner())); + } } + + return queryStatus; } @Override diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java index 15f24f77..01e3cd33 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java @@ -63,19 +63,25 @@ public Void call() throws Exception { private void monitor(long currentTimeMillis) { for (QueryStatus status : queryStorageCache.getQueryStatus()) { String queryId = status.getQueryKey().getQueryId().toString(); - if (status.getQueryState() == QueryStatus.QUERY_STATE.DEFINED || status.getQueryState() == QueryStatus.QUERY_STATE.CREATED) { - if (status.isUserIdle(currentTimeMillis, expirationProperties.getIdleTimeoutMillis())) { - // if the user hasn't interacted with the query in a while, cancel it - cancelQuery(queryId); - } else if (status.isProgressIdle(currentTimeMillis, expirationProperties.getIdleTimeoutMillis())) { - // if progress hasn't been made for the query in a while, apply the shock paddles - defibrillateQuery(queryId, status.getQueryKey().getQueryPool().getName()); - } - } else { - if (status.isInactive(currentTimeMillis, monitorProperties.getInactiveQueryTimeToLiveMillis())) { - // if the query has been inactive for too long, evict it - deleteQuery(UUID.fromString(queryId)); - } + switch (status.getQueryState()) { + case DEFINED: + case CREATED: + if (status.isUserIdle(currentTimeMillis, expirationProperties.getIdleTimeoutMillis())) { + // if the user hasn't interacted with the query in a while, cancel it + cancelQuery(queryId); + } else if (status.isProgressIdle(currentTimeMillis, expirationProperties.getIdleTimeoutMillis())) { + // if progress hasn't been made for the query in a while, apply the shock paddles + defibrillateQuery(queryId, status.getQueryKey().getQueryPool().getName()); + } + break; + case CLOSED: + case CANCELED: + case FAILED: + if (status.isInactive(currentTimeMillis, monitorProperties.getInactiveQueryTimeToLiveMillis())) { + // if the query has been inactive (closed/canceled/failed) for too long, evict it + deleteQuery(UUID.fromString(queryId)); + } + break; } } } From 8f97ca16987aff7536011183193ae95261504e6b Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Mon, 7 Jun 2021 17:40:32 -0400 Subject: [PATCH 063/218] Added reset and remove calls to the query service controller. Added progress timeout to determine when a stalled query should be jumpstarted. Refactored some query management service logic to eliminate duplicate code. --- query-microservices/query-service/README.md | 4 +- .../config/QueryExpirationProperties.java | 24 +++ .../microservice/query/QueryController.java | 42 +++- .../query/QueryManagementService.java | 199 ++++++++++++++---- .../query/monitor/MonitorTask.java | 15 +- 5 files changed, 226 insertions(+), 58 deletions(-) diff --git a/query-microservices/query-service/README.md b/query-microservices/query-service/README.md index 91b5b092..56ac784c 100644 --- a/query-microservices/query-service/README.md +++ b/query-microservices/query-service/README.md @@ -19,7 +19,7 @@ query capabilities. | | `POST` | /{queryLogic}/plan | Generate a query plan using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | | | `POST` | /{queryLogic}/predict | Generate a query prediction using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | | | `POST` | /{queryLogic}/async/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | -| | `PUT` | /{id}/reset | Resets the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | `PUT` `POST` | /{id}/reset | Resets the specified query | [QueryId] | N/A | [VoidResponse] | | ✓ | `POST` | /{queryLogic}/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | | | `POST` | /{queryLogic}/async/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | | | `GET` | /lookupContentUUID/{uuidType}/{uuid} | Returns content associated with the given UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] or [StreamingOutput] | @@ -37,7 +37,7 @@ query capabilities. | | `GET` | /listAll | Returns a list of queries associated with the current user | N/A | N/A | [QueryImplListResponse] | | | `GET` | /{id} | Returns query info for the specified query | [QueryId] | N/A | [QueryImplListResponse] | | | `GET` | /list | Returns a list of queries associated with the current user with given name | N/A | [Name] | [QueryImplListResponse] | -| | `DELETE` | /{id}/remove | Remove (delete) the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | `DELETE` | /{id}/remove | Remove (delete) the specified query | [QueryId] | N/A | [VoidResponse] | | | `POST` | /{id}/duplicate | Duplicates the specified query | [QueryId] | [QueryParameters] | [GenericResponse] | | | `PUT` `POST` | /{id}/update | Updates the specified query | [QueryId] | [QueryParameters] | [GenericResponse] | | | `GET` | /{id}/listAll | Returns a list of queries associated with the specified user | [UserId] | N/A | [QueryImplListResponse] | diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java index bb45574b..753765a1 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java @@ -13,6 +13,10 @@ public class QueryExpirationProperties { @NotNull private TimeUnit idleTimeUnit = TimeUnit.MINUTES; @Positive + private long progressTimeout = 5; + @NotNull + private TimeUnit progressTimeUnit = TimeUnit.MINUTES; + @Positive private long callTimeout = 60; @NotNull private TimeUnit callTimeUnit = TimeUnit.MINUTES; @@ -53,6 +57,26 @@ public void setIdleTimeUnit(TimeUnit idleTimeUnit) { this.idleTimeUnit = idleTimeUnit; } + public long getProgressTimeout() { + return progressTimeout; + } + + public long getProgressTimeoutMillis() { + return progressTimeUnit.toMillis(progressTimeout); + } + + public void setProgressTimeout(long progressTimeout) { + this.progressTimeout = progressTimeout; + } + + public TimeUnit getProgressTimeUnit() { + return progressTimeUnit; + } + + public void setProgressTimeUnit(TimeUnit progressTimeUnit) { + this.progressTimeUnit = progressTimeUnit; + } + public long getCallTimeout() { return callTimeout; } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 336c7650..50364af8 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -3,6 +3,7 @@ import com.codahale.metrics.annotation.Timed; import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.query.web.annotation.EnrichQueryMetrics; +import datawave.webservice.query.exception.QueryException; import datawave.webservice.result.BaseQueryResponse; import datawave.webservice.result.GenericResponse; import datawave.webservice.result.QueryLogicResponse; @@ -39,7 +40,7 @@ public QueryLogicResponse listQueryLogic() { @RequestMapping(path = "{queryLogic}/define", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public GenericResponse define(@PathVariable(name = "queryLogic") String queryLogic, @RequestParam MultiValueMap parameters, - @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.define(queryLogic, parameters, currentUser); } @@ -48,7 +49,7 @@ public GenericResponse define(@PathVariable(name = "queryLogic") String @RequestMapping(path = "{queryLogic}/create", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public GenericResponse create(@PathVariable(name = "queryLogic") String queryLogic, @RequestParam MultiValueMap parameters, - @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.create(queryLogic, parameters, currentUser); } @@ -57,7 +58,7 @@ public GenericResponse create(@PathVariable(name = "queryLogic") String @RequestMapping(path = "{queryLogic}/createAndNext", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public BaseQueryResponse createQueryAndNext(@PathVariable(name = "queryLogic") String queryLogic, @RequestParam MultiValueMap parameters, - @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.createAndNext(queryLogic, parameters, currentUser); } @@ -65,14 +66,15 @@ public BaseQueryResponse createQueryAndNext(@PathVariable(name = "queryLogic") S @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.NEXT) @RequestMapping(path = "{queryId}/next", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public BaseQueryResponse next(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + public BaseQueryResponse next(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) + throws QueryException { return queryManagementService.next(queryId, currentUser); } @Timed(name = "dw.query.cancel", absolute = true) @RequestMapping(path = "{queryId}/cancel", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public VoidResponse cancel(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + public VoidResponse cancel(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.cancel(queryId, currentUser); } @@ -80,14 +82,15 @@ public VoidResponse cancel(@PathVariable(name = "queryId") String queryId, @Auth @RolesAllowed({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "{queryId}/adminCancel", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public VoidResponse adminCancel(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + public VoidResponse adminCancel(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) + throws QueryException { return queryManagementService.adminCancel(queryId, currentUser); } @Timed(name = "dw.query.close", absolute = true) @RequestMapping(path = "{queryId}/close", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public VoidResponse close(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + public VoidResponse close(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.close(queryId, currentUser); } @@ -95,7 +98,30 @@ public VoidResponse close(@PathVariable(name = "queryId") String queryId, @Authe @RolesAllowed({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "{queryId}/adminClose", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public VoidResponse adminClose(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws Exception { + public VoidResponse adminClose(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) + throws QueryException { return queryManagementService.adminClose(queryId, currentUser); } + + @Timed(name = "dw.query.reset", absolute = true) + @RequestMapping(path = "{queryId}/reset", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse reset(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return queryManagementService.reset(queryId, currentUser); + } + + @Timed(name = "dw.query.remove", absolute = true) + @RequestMapping(path = "{queryId}/remove", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse remove(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return queryManagementService.remove(queryId, currentUser); + } + + @Timed(name = "dw.query.update", absolute = true) + @RequestMapping(path = "{queryId}/update", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse update(@PathVariable(name = "queryId") String queryId, @RequestParam MultiValueMap parameters, + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return queryManagementService.update(queryId, parameters, currentUser); + } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 71f514b1..a4083f5c 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -71,6 +71,9 @@ import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CANCELED; import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CLOSED; +import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CREATED; +import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.DEFINED; +import static io.undertow.util.StatusCodes.BAD_REQUEST; @Service public class QueryManagementService implements QueryRequestHandler { @@ -219,38 +222,7 @@ public QueryLogicResponse listQueryLogic() { */ public GenericResponse define(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { - // validate query and get a query logic - QueryLogic queryLogic = validateQuery(queryLogicName, parameters, currentUser); - - String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); - log.trace(userId + " has authorizations " + currentUser.getPrimaryUser().getAuths()); - - // set some audit parameters which are used internally - String userDn = currentUser.getPrimaryUser().getDn().subjectDN(); - setInternalAuditParameters(queryLogicName, userDn, parameters); - - try { - // persist the query w/ query id in the query storage cache - // TODO: JWO: storeQuery assumes that this is a 'create' call, but this is a 'define' call. - // @formatter:off - TaskKey taskKey = queryStorageCache.defineQuery( - new QueryPool(getPoolName()), - createQuery(queryLogicName, parameters, userDn, currentUser.getDNs()), - AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), currentUser), - getMaxConcurrentTasks(queryLogic)); - // @formatter:on - - // TODO: JWO: Figure out how to make query tracing work with our new architecture. Datawave issue #1155 - - // TODO: JWO: Figure out how to make query metrics work with our new architecture. Datawave issue #1156 - - GenericResponse response = new GenericResponse<>(); - response.setResult(taskKey.getQueryId().toString()); - return response; - } catch (Exception e) { - log.error("Unknown error storing query", e); - throw new BadRequestQueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR, e); - } + return storeQuery(queryLogicName, parameters, currentUser, false); } /** @@ -267,6 +239,16 @@ public GenericResponse define(String queryLogicName, MultiValueMap create(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + return storeQuery(queryLogicName, parameters, currentUser, true); + } + + private GenericResponse storeQuery(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser, + boolean isCreateRequest) throws QueryException { + return storeQuery(queryLogicName, parameters, currentUser, isCreateRequest, null); + } + + private GenericResponse storeQuery(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser, + boolean isCreateRequest, String queryId) throws QueryException { try { // validate query and get a query logic QueryLogic queryLogic = validateQuery(queryLogicName, parameters, currentUser); @@ -278,19 +260,33 @@ public GenericResponse create(String queryLogicName, MultiValueMap create(String queryLogicName, MultiValueMap response = new GenericResponse<>(); - response.setHasResults(true); response.setResult(taskKey.getQueryId().toString()); + + if (isCreateRequest) { + response.setHasResults(true); + } + return response; } catch (Exception e) { log.error("Unknown error storing query", e); @@ -355,9 +355,14 @@ public BaseQueryResponse createAndNext(String queryLogicName, MultiValueMap parameters = new LinkedMultiValueMap<>(); + parameters.addAll(queryStatus.getQuery().getOptionalQueryParameters()); + queryStatus.getQuery().getParameters().forEach(x -> { + parameters.add(x.getParameterName(), x.getParameterValue()); + }); + + // create the query + storeQuery(queryStatus.getQuery().getQueryLogicName(), parameters, currentUser, true, queryId); + + VoidResponse response = new VoidResponse(); + response.addMessage(queryId + " reset."); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error resetting query " + queryId, e); + throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Unknown error resetting query " + queryId); + } + } + + /** + * Remove (delete) the query + * + * @param queryId + * @param currentUser + * @return + */ + public VoidResponse remove(String queryId, ProxiedUserDetails currentUser) throws QueryException { + try { + // make sure the query is valid, and the user can act on it + QueryStatus queryStatus = validateRequest(queryId, currentUser); + + // remove the query from the cache if it is not running + if (queryStatus.getQueryState() != CREATED) { + queryStorageCache.deleteQuery(queryStatus.getQueryKey().getQueryId()); + } + + VoidResponse response = new VoidResponse(); + response.addMessage(queryId + " removed."); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error removing query " + queryId, e); + throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Unknown error removing query " + queryId); + } + } + + // TODO: Flesh this out + public GenericResponse update(String queryId, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + try { + // make sure the query is valid, and the user can act on it + QueryStatus queryStatus = validateRequest(queryId, currentUser); + + switch (queryStatus.getQueryState()) { + case DEFINED: + case CLOSED: + case CANCELED: + case FAILED: + // unsafe updates that can only be made to a query that is not running + // fall through + case CREATED: + // safe updates that can be made to a running query + break; + + } + + return null; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error updating query " + queryId, e); + throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Unknown error updating query " + queryId); + } + } + private QueryStatus validateRequest(String queryId, ProxiedUserDetails currentUser) throws QueryException { return validateRequest(queryId, currentUser, false); } @@ -753,11 +857,18 @@ else if (queryLogic.getMaxConcurrentTasks() > 0) { } protected Query createQuery(String queryLogicName, MultiValueMap parameters, String userDn, List dnList) { + return createQuery(queryLogicName, parameters, userDn, dnList, null); + } + + protected Query createQuery(String queryLogicName, MultiValueMap parameters, String userDn, List dnList, String queryId) { Query q = responseObjectFactory.getQueryImpl(); q.initialize(userDn, dnList, queryLogicName, queryParameters, queryParameters.getUnknownParameters(parameters)); q.setColumnVisibility(securityMarking.toColumnVisibilityString()); q.setUncaughtExceptionHandler(new QueryUncaughtExceptionHandler()); Thread.currentThread().setUncaughtExceptionHandler(q.getUncaughtExceptionHandler()); + if (queryId != null) { + q.setId(UUID.fromString(queryId)); + } return q; } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java index 01e3cd33..fb43fcb4 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java @@ -64,17 +64,24 @@ private void monitor(long currentTimeMillis) { for (QueryStatus status : queryStorageCache.getQueryStatus()) { String queryId = status.getQueryKey().getQueryId().toString(); switch (status.getQueryState()) { - case DEFINED: case CREATED: + if (status.isProgressIdle(currentTimeMillis, expirationProperties.getProgressTimeoutMillis())) { + // if progress hasn't been made for the query in a while, apply the shock paddles + defibrillateQuery(queryId, status.getQueryKey().getQueryPool().getName()); + } + // fall through + case DEFINED: if (status.isUserIdle(currentTimeMillis, expirationProperties.getIdleTimeoutMillis())) { // if the user hasn't interacted with the query in a while, cancel it cancelQuery(queryId); - } else if (status.isProgressIdle(currentTimeMillis, expirationProperties.getIdleTimeoutMillis())) { - // if progress hasn't been made for the query in a while, apply the shock paddles - defibrillateQuery(queryId, status.getQueryKey().getQueryPool().getName()); } break; case CLOSED: + if (status.getConcurrentNextCount() > 0 && status.isProgressIdle(currentTimeMillis, expirationProperties.getProgressTimeoutMillis())) { + // if there are next calls waiting for results, and progress hasn't been made for the query in a while, apply the shock paddles + defibrillateQuery(queryId, status.getQueryKey().getQueryPool().getName()); + } + // fall through case CANCELED: case FAILED: if (status.isInactive(currentTimeMillis, monitorProperties.getInactiveQueryTimeToLiveMillis())) { From c237586f4d699e939a572e0a00d56e6b56bee337 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 9 Jun 2021 15:18:36 -0400 Subject: [PATCH 064/218] added an update query method to the query service controller and removed the reset method. --- .../query/config/QueryProperties.java | 17 + .../microservice/query/QueryController.java | 7 - .../query/QueryManagementService.java | 301 +++++++++++------- 3 files changed, 196 insertions(+), 129 deletions(-) diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java index c1a72c59..e8179f98 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java @@ -6,8 +6,15 @@ import javax.validation.constraints.NotNull; import javax.validation.constraints.Positive; import javax.validation.constraints.PositiveOrZero; +import java.util.Arrays; +import java.util.List; import java.util.concurrent.TimeUnit; +import static datawave.microservice.query.QueryParameters.QUERY_EXPIRATION; +import static datawave.microservice.query.QueryParameters.QUERY_MAX_RESULTS_OVERRIDE; +import static datawave.microservice.query.QueryParameters.QUERY_PAGESIZE; +import static datawave.microservice.query.QueryParameters.QUERY_PAGETIMEOUT; + @Validated public class QueryProperties { @NotEmpty @@ -24,6 +31,8 @@ public class QueryProperties { private TimeUnit lockLeaseTimeUnit = TimeUnit.MILLISECONDS; @NotEmpty private String executorServiceName = "executor"; + // These are the only parameters that can be updated for a running query + private List updatableParams = Arrays.asList(QUERY_EXPIRATION, QUERY_PAGESIZE, QUERY_PAGETIMEOUT, QUERY_MAX_RESULTS_OVERRIDE); private QueryExpirationProperties expiration = new QueryExpirationProperties(); private NextCallProperties nextCall = new NextCallProperties(); @@ -87,6 +96,14 @@ public void setExecutorServiceName(String executorServiceName) { this.executorServiceName = executorServiceName; } + public List getUpdatableParams() { + return updatableParams; + } + + public void setUpdatableParams(List updatableParams) { + this.updatableParams = updatableParams; + } + public QueryExpirationProperties getExpiration() { return expiration; } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 50364af8..71820456 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -103,13 +103,6 @@ public VoidResponse adminClose(@PathVariable(name = "queryId") String queryId, @ return queryManagementService.adminClose(queryId, currentUser); } - @Timed(name = "dw.query.reset", absolute = true) - @RequestMapping(path = "{queryId}/reset", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", - "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public VoidResponse reset(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { - return queryManagementService.reset(queryId, currentUser); - } - @Timed(name = "dw.query.remove", absolute = true) @RequestMapping(path = "{queryId}/remove", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index a4083f5c..6da293a8 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -53,6 +53,7 @@ import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; +import java.io.IOException; import java.lang.reflect.Method; import java.net.InetAddress; import java.net.UnknownHostException; @@ -73,7 +74,9 @@ import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CLOSED; import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CREATED; import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.DEFINED; +import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; import static io.undertow.util.StatusCodes.BAD_REQUEST; +import static io.undertow.util.StatusCodes.INTERNAL_SERVER_ERROR; @Service public class QueryManagementService implements QueryRequestHandler { @@ -222,7 +225,17 @@ public QueryLogicResponse listQueryLogic() { */ public GenericResponse define(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { - return storeQuery(queryLogicName, parameters, currentUser, false); + try { + TaskKey taskKey = storeQuery(queryLogicName, parameters, currentUser, false); + GenericResponse response = new GenericResponse<>(); + response.setResult(taskKey.getQueryId().toString()); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error defining query", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error defining query."); + } } /** @@ -239,79 +252,76 @@ public GenericResponse define(String queryLogicName, MultiValueMap create(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { - return storeQuery(queryLogicName, parameters, currentUser, true); + try { + TaskKey taskKey = storeQuery(queryLogicName, parameters, currentUser, true); + GenericResponse response = new GenericResponse<>(); + response.setResult(taskKey.getQueryId().toString()); + response.setHasResults(true); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error creating query", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error creating query."); + } } - private GenericResponse storeQuery(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser, - boolean isCreateRequest) throws QueryException { + private TaskKey storeQuery(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser, boolean isCreateRequest) + throws QueryException { return storeQuery(queryLogicName, parameters, currentUser, isCreateRequest, null); } - private GenericResponse storeQuery(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser, - boolean isCreateRequest, String queryId) throws QueryException { + private TaskKey storeQuery(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser, boolean isCreateRequest, + String queryId) throws QueryException { + // validate query and get a query logic + QueryLogic queryLogic = validateQuery(queryLogicName, parameters, currentUser); + + String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + log.trace(userId + " has authorizations " + currentUser.getPrimaryUser().getAuths()); + + // set some audit parameters which are used internally + String userDn = currentUser.getPrimaryUser().getDn().subjectDN(); + setInternalAuditParameters(queryLogicName, userDn, parameters); + + Query query = createQuery(queryLogicName, parameters, userDn, currentUser.getDNs(), queryId); + + // if this is a create request, send an audit record to the auditor + if (isCreateRequest) { + audit(query, queryLogic, parameters, currentUser); + } + try { - // validate query and get a query logic - QueryLogic queryLogic = validateQuery(queryLogicName, parameters, currentUser); - - String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); - log.trace(userId + " has authorizations " + currentUser.getPrimaryUser().getAuths()); - - // set some audit parameters which are used internally - String userDn = currentUser.getPrimaryUser().getDn().subjectDN(); - setInternalAuditParameters(queryLogicName, userDn, parameters); - - Query query = createQuery(queryLogicName, parameters, userDn, currentUser.getDNs()); - - // if this is a create request, send an audit record to the auditor + // persist the query w/ query id in the query storage cache + TaskKey taskKey; if (isCreateRequest) { - audit(query, queryLogic, parameters, currentUser); - } - - try { - // persist the query w/ query id in the query storage cache - TaskKey taskKey; - if (isCreateRequest) { - // @formatter:off - taskKey = queryStorageCache.createQuery( - new QueryPool(getPoolName()), - query, - AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), currentUser), - getMaxConcurrentTasks(queryLogic)); - // @formatter:on - } else { - // @formatter:off - taskKey = queryStorageCache.defineQuery( - new QueryPool(getPoolName()), - query, - AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), currentUser), - getMaxConcurrentTasks(queryLogic)); - // @formatter:on - } + // @formatter:off + taskKey = queryStorageCache.createQuery( + new QueryPool(getPoolName()), + query, + AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), currentUser), + getMaxConcurrentTasks(queryLogic)); + // @formatter:on // publish a create event to the executor pool publishExecutorEvent(QueryRequest.create(taskKey.getQueryId().toString()), getPoolName()); - - // TODO: JWO: Figure out how to make query tracing work with our new architecture. Datawave issue #1155 - - // TODO: JWO: Figure out how to make query metrics work with our new architecture. Datawave issue #1156 - - GenericResponse response = new GenericResponse<>(); - response.setResult(taskKey.getQueryId().toString()); - - if (isCreateRequest) { - response.setHasResults(true); - } - - return response; - } catch (Exception e) { - log.error("Unknown error storing query", e); - throw new BadRequestQueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR, e); + } else { + // @formatter:off + taskKey = queryStorageCache.defineQuery( + new QueryPool(getPoolName()), + query, + AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), currentUser), + getMaxConcurrentTasks(queryLogic)); + // @formatter:on } - } catch (QueryException e) { - throw e; + + // TODO: JWO: Figure out how to make query tracing work with our new architecture. Datawave issue #1155 + + // TODO: JWO: Figure out how to make query metrics work with our new architecture. Datawave issue #1156 + + return taskKey; } catch (Exception e) { - log.error("Unknown error creating query", e); - throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error creating query."); + log.error("Unknown error storing query", e); + throw new BadRequestQueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR, e); } } @@ -628,49 +638,6 @@ public void close(String queryId) throws InterruptedException, QueryException { publishExecutorEvent(QueryRequest.close(queryId), queryStatus.getQueryKey().getQueryPool().getName()); } - /** - * Resets the query named by {@code queryId}. If the query is not alive, meaning that the current session has expired (due to either timeout, or server - * failure), then this will reload the query and start it over. If the query is alive, it closes it and starts the query over. - * - * @param queryId - * @return - * @throws QueryException - */ - public VoidResponse reset(String queryId, ProxiedUserDetails currentUser) throws QueryException { - try { - // make sure the query is valid, and the user can act on it - QueryStatus queryStatus = validateRequest(queryId, currentUser); - - // cancel the query if it is defined or running - if (queryStatus.getQueryState() == DEFINED || queryStatus.getQueryState() == CREATED) { - cancel(queryStatus.getQueryKey().getQueryId().toString(), true); - } - - // remove the existing cache entry - queryStorageCache.deleteQuery(queryStatus.getQueryKey().getQueryId()); - - // TODO: Are we losing anything by recreating the parameters this way? - // recreate the query parameters - MultiValueMap parameters = new LinkedMultiValueMap<>(); - parameters.addAll(queryStatus.getQuery().getOptionalQueryParameters()); - queryStatus.getQuery().getParameters().forEach(x -> { - parameters.add(x.getParameterName(), x.getParameterValue()); - }); - - // create the query - storeQuery(queryStatus.getQuery().getQueryLogicName(), parameters, currentUser, true, queryId); - - VoidResponse response = new VoidResponse(); - response.addMessage(queryId + " reset."); - return response; - } catch (QueryException e) { - throw e; - } catch (Exception e) { - log.error("Unknown error resetting query " + queryId, e); - throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Unknown error resetting query " + queryId); - } - } - /** * Remove (delete) the query * @@ -683,9 +650,8 @@ public VoidResponse remove(String queryId, ProxiedUserDetails currentUser) throw // make sure the query is valid, and the user can act on it QueryStatus queryStatus = validateRequest(queryId, currentUser); - // remove the query from the cache if it is not running - if (queryStatus.getQueryState() != CREATED) { - queryStorageCache.deleteQuery(queryStatus.getQueryKey().getQueryId()); + if (!remove(queryStatus)) { + throw new QueryException("Failed to remove " + queryId, INTERNAL_SERVER_ERROR + "-1"); } VoidResponse response = new VoidResponse(); @@ -698,27 +664,82 @@ public VoidResponse remove(String queryId, ProxiedUserDetails currentUser) throw throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Unknown error removing query " + queryId); } } - - // TODO: Flesh this out + + private boolean remove(QueryStatus queryStatus) throws IOException { + boolean success = false; + // remove the query from the cache if it is not running + if (queryStatus.getQueryState() != CREATED) { + success = queryStorageCache.deleteQuery(queryStatus.getQueryKey().getQueryId()); + } + return success; + } + public GenericResponse update(String queryId, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { try { // make sure the query is valid, and the user can act on it QueryStatus queryStatus = validateRequest(queryId, currentUser); - switch (queryStatus.getQueryState()) { - case DEFINED: - case CLOSED: - case CANCELED: - case FAILED: - // unsafe updates that can only be made to a query that is not running - // fall through - case CREATED: - // safe updates that can be made to a running query - break; + GenericResponse response = new GenericResponse<>(); + if (!parameters.isEmpty()) { + // TODO: Are we losing anything by recreating the parameters this way? + // recreate the query parameters + MultiValueMap currentParams = new LinkedMultiValueMap<>(); + currentParams.addAll(queryStatus.getQuery().getOptionalQueryParameters()); + queryStatus.getQuery().getParameters().forEach(x -> { + currentParams.add(x.getParameterName(), x.getParameterValue()); + }); + boolean updated = false; + if (queryStatus.getQueryState() == DEFINED) { + // update all parameters if the state is defined + updated = updateParameters(parameters, currentParams); + + // redefine the query + if (updated) { + storeQuery(queryStatus.getQuery().getQueryLogicName(), currentParams, currentUser, false, queryId); + } + } else if (queryStatus.getQueryState() == CREATED) { + // if the query is created/running, update safe parameters only + List ignoredParams = new ArrayList<>(parameters.keySet()); + List safeParams = new ArrayList<>(queryProperties.getUpdatableParams()); + safeParams.retainAll(parameters.keySet()); + ignoredParams.removeAll(safeParams); + + // only update the safe parameters if the query is running + updated = updateParameters(safeParams, parameters, currentParams); + + if (updated) { + // validate the update + String queryLogicName = queryStatus.getQuery().getQueryLogicName(); + validateQuery(queryLogicName, parameters, currentUser); + + // create a new query object + String userDn = currentUser.getPrimaryUser().getDn().subjectDN(); + Query query = createQuery(queryLogicName, parameters, userDn, currentUser.getDNs(), queryId); + + // save the new query object in the cache + queryStatusUpdateHelper.lockedUpdate(UUID.fromString(queryId), status -> { + status.setQuery(query); + }); + } + + if (!ignoredParams.isEmpty()) { + response.addMessage("The following parameters cannot be updated for a running query: " + String.join(",", ignoredParams)); + } + } else { + throw new QueryException("Cannot update a query unless it is defined or running.", BAD_REQUEST + "-1"); + } + + if (updated) { + response.addMessage(queryId + " updated."); + } else { + response.addMessage(queryId + " unchanged."); + } + } else { + throw new QueryException("No parameters specified for update.", BAD_REQUEST + "-1"); } - return null; + return response; } catch (QueryException e) { throw e; } catch (Exception e) { @@ -727,6 +748,42 @@ public GenericResponse update(String queryId, MultiValueMap newParameters, MultiValueMap currentParams) throws QueryException { + return updateParameters(newParameters.keySet(), newParameters, currentParams); + } + + /** + * Updates the current params with the new params for the given parameter names. + * + * @param parameterNames + * @param newParameters + * @param currentParams + * @return true if current params was modified + */ + private boolean updateParameters(Collection parameterNames, MultiValueMap newParameters, MultiValueMap currentParams) + throws QueryException { + boolean paramsUpdated = false; + for (String paramName : parameterNames) { + if (newParameters.get(paramName) != null && !newParameters.get(paramName).isEmpty()) { + if (!newParameters.get(paramName).get(0).equals(currentParams.getFirst(paramName))) { + // if the new value differs from the old value, update the old value + currentParams.put(paramName, newParameters.remove(paramName)); + paramsUpdated = true; + } + } else { + throw new QueryException("Cannot update a query parameter without a value: " + paramName, BAD_REQUEST + "-1"); + } + } + return paramsUpdated; + } + private QueryStatus validateRequest(String queryId, ProxiedUserDetails currentUser) throws QueryException { return validateRequest(queryId, currentUser, false); } @@ -892,7 +949,7 @@ protected QueryLogic validateQuery(String queryLogicName, MultiValueMap parameters) throws QueryException { // add query logic name to parameters - parameters.add(QueryParameters.QUERY_LOGIC_NAME, queryLogicName); + parameters.add(QUERY_LOGIC_NAME, queryLogicName); log.debug(writeValueAsString(parameters)); From b5d910e53fffedbda8b4bb705eadc99397771cd4 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 9 Jun 2021 17:33:33 -0400 Subject: [PATCH 065/218] Revised the reset logic to cancel the existing query and create a new, duplicate query with a new query id. Implemented the duplicate call for the query service controller. --- query-microservices/query-service/README.md | 72 +++++++-------- .../microservice/query/QueryController.java | 17 +++- .../query/QueryManagementService.java | 87 ++++++++++++++++--- 3 files changed, 127 insertions(+), 49 deletions(-) diff --git a/query-microservices/query-service/README.md b/query-microservices/query-service/README.md index 56ac784c..98bbc151 100644 --- a/query-microservices/query-service/README.md +++ b/query-microservices/query-service/README.md @@ -11,42 +11,42 @@ query capabilities. ### User API -| Done? | Method | Operation | Description | Path Param | Request Body | Response Body | -|:--------|:--------------|:-------------------------------------|:----------------------------------------------------------------------------------|:-------------------|:---------------------|:-----------------------------------------| -| ✓ | `GET` | /listQueryLogic | List QueryLogic types that are currently available | N/A | N/A | [QueryLogicResponse] | -| ✓ | `POST` | /{queryLogic}/define | Define a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | -| ✓ | `POST` | /{queryLogic}/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | -| | `POST` | /{queryLogic}/plan | Generate a query plan using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | -| | `POST` | /{queryLogic}/predict | Generate a query prediction using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | -| | `POST` | /{queryLogic}/async/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | -| ✓ | `PUT` `POST` | /{id}/reset | Resets the specified query | [QueryId] | N/A | [VoidResponse] | -| ✓ | `POST` | /{queryLogic}/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | -| | `POST` | /{queryLogic}/async/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | -| | `GET` | /lookupContentUUID/{uuidType}/{uuid} | Returns content associated with the given UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] or [StreamingOutput] | -| | `POST` | /lookupContentUUID | Returns content associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] or [StreamingOutput] | -| | `GET` | /lookupUUID/{uuidType}/{uuid} | Returns event associated with the given batch of UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] or [StreamingOutput] | -| | `POST` | /lookupUUID | Returns event(s) associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] or [StreamingOutput] | -| | `GET` | /{id}/plan | Returns the plan for the specified query | [QueryId] | N/A | [GenericResponse] | -| | `GET` | /{id}/predictions | Returns the predictions for the specified query | [QueryId] | N/A | [GenericResponse] | -| | `GET` | /{id}/async/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | -| ✓ | `GET` | /{id}/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | -| ✓ | `PUT` `POST` | /{id}/close | Closes the specified query | [QueryId] | N/A | [VoidResponse] | -| ✓ | `PUT` `POST` | /{id}/adminClose | Closes the specified query | [QueryId] | N/A | [VoidResponse] | -| ✓ | `PUT` `POST` | /{id}/cancel | Cancels the specified query | [QueryId] | N/A | [VoidResponse] | -| ✓ | `PUT` `POST` | /{id}/adminCancel | Cancels the specified query | [QueryId] | N/A | [VoidResponse] | -| | `GET` | /listAll | Returns a list of queries associated with the current user | N/A | N/A | [QueryImplListResponse] | -| | `GET` | /{id} | Returns query info for the specified query | [QueryId] | N/A | [QueryImplListResponse] | -| | `GET` | /list | Returns a list of queries associated with the current user with given name | N/A | [Name] | [QueryImplListResponse] | -| ✓ | `DELETE` | /{id}/remove | Remove (delete) the specified query | [QueryId] | N/A | [VoidResponse] | -| | `POST` | /{id}/duplicate | Duplicates the specified query | [QueryId] | [QueryParameters] | [GenericResponse] | -| | `PUT` `POST` | /{id}/update | Updates the specified query | [QueryId] | [QueryParameters] | [GenericResponse] | -| | `GET` | /{id}/listAll | Returns a list of queries associated with the specified user | [UserId] | N/A | [QueryImplListResponse] | -| | `POST` | /purgeQueryCache | Purges the cache of query objects | N/A | N/A | [VoidResponse] | -| | `GET` | /enableTracing | Enables tracing for queries which match the given criteria | N/A | [QueryRegex], [User] | [VoidResponse] | -| | `GET` | /disableTracing | Disables tracing for queries which match the given criteria | N/A | [QueryRegex], [User] | [VoidResponse] | -| | `GET` | /disableAllTracing | Disables tracing for all queries | N/A | N/A | [VoidResponse] | -| | `POST` | /{logicName}/execute | Create a query using the specified query logic and params, and stream the results | [QueryLogicName] | [QueryParameters] | [StreamingOutput] | -| | `POST` | /{logicName}/async/execute | Create a query using the specified query logic and params, and stream the results | [QueryLogicName] | [QueryParameters] | [StreamingOutput] | +| Done? | Method | Operation | Description | Path Param | Request Body | Response Body | +|:--------|:--------------|:-------------------------------------|:----------------------------------------------------------------------------------|:-------------------|:---------------------|:-------------------------------------------| +| ✓ | `GET` | /listQueryLogic | List QueryLogic types that are currently available | N/A | N/A | [QueryLogicResponse] | +| ✓ | `POST` | /{queryLogic}/define | Define a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| ✓ | `POST` | /{queryLogic}/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| | `POST` | /{queryLogic}/plan | Generate a query plan using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| | `POST` | /{queryLogic}/predict | Generate a query prediction using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| | `POST` | /{queryLogic}/async/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| ✓ | `PUT` `POST` | /{id}/reset | Resets the specified query | [QueryId] | N/A | [VoidResponse]
[GenericResponse] | +| ✓ | `POST` | /{queryLogic}/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | +| | `POST` | /{queryLogic}/async/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | +| | `GET` | /lookupContentUUID/{uuidType}/{uuid} | Returns content associated with the given UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] or [StreamingOutput] | +| | `POST` | /lookupContentUUID | Returns content associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] or [StreamingOutput] | +| | `GET` | /lookupUUID/{uuidType}/{uuid} | Returns event associated with the given batch of UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] or [StreamingOutput] | +| | `POST` | /lookupUUID | Returns event(s) associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] or [StreamingOutput] | +| | `GET` | /{id}/plan | Returns the plan for the specified query | [QueryId] | N/A | [GenericResponse] | +| | `GET` | /{id}/predictions | Returns the predictions for the specified query | [QueryId] | N/A | [GenericResponse] | +| | `GET` | /{id}/async/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | +| ✓ | `GET` | /{id}/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | +| ✓ | `PUT` `POST` | /{id}/close | Closes the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | `PUT` `POST` | /{id}/adminClose | Closes the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | `PUT` `POST` | /{id}/cancel | Cancels the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | `PUT` `POST` | /{id}/adminCancel | Cancels the specified query | [QueryId] | N/A | [VoidResponse] | +| | `GET` | /listAll | Returns a list of queries associated with the current user | N/A | N/A | [QueryImplListResponse] | +| | `GET` | /{id} | Returns query info for the specified query | [QueryId] | N/A | [QueryImplListResponse] | +| | `GET` | /list | Returns a list of queries associated with the current user with given name | N/A | [Name] | [QueryImplListResponse] | +| ✓ | `DELETE` | /{id}/remove | Remove (delete) the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | `POST` | /{id}/duplicate | Duplicates the specified query | [QueryId] | [QueryParameters] | [GenericResponse] | +| ✓ | `PUT` `POST` | /{id}/update | Updates the specified query | [QueryId] | [QueryParameters] | [GenericResponse] | +| | `GET` | /{id}/listAll | Returns a list of queries associated with the specified user | [UserId] | N/A | [QueryImplListResponse] | +| | `POST` | /purgeQueryCache | Purges the cache of query objects | N/A | N/A | [VoidResponse] | +| | `GET` | /enableTracing | Enables tracing for queries which match the given criteria | N/A | [QueryRegex], [User] | [VoidResponse] | +| | `GET` | /disableTracing | Disables tracing for queries which match the given criteria | N/A | [QueryRegex], [User] | [VoidResponse] | +| | `GET` | /disableAllTracing | Disables tracing for all queries | N/A | N/A | [VoidResponse] | +| | `POST` | /{logicName}/execute | Create a query using the specified query logic and params, and stream the results | [QueryLogicName] | [QueryParameters] | [StreamingOutput] | +| | `POST` | /{logicName}/async/execute | Create a query using the specified query logic and params, and stream the results | [QueryLogicName] | [QueryParameters] | [StreamingOutput] | --- diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 71820456..e064fa72 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -102,7 +102,14 @@ public VoidResponse adminClose(@PathVariable(name = "queryId") String queryId, @ throws QueryException { return queryManagementService.adminClose(queryId, currentUser); } - + + @Timed(name = "dw.query.reset", absolute = true) + @RequestMapping(path = "{queryId}/reset", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse reset(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return queryManagementService.reset(queryId, currentUser); + } + @Timed(name = "dw.query.remove", absolute = true) @RequestMapping(path = "{queryId}/remove", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) @@ -117,4 +124,12 @@ public GenericResponse update(@PathVariable(name = "queryId") String que @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.update(queryId, parameters, currentUser); } + + @Timed(name = "dw.query.duplicate", absolute = true) + @RequestMapping(path = "{queryId}/duplicate", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", + "application/x-protostuff"}) + public GenericResponse duplicate(@PathVariable(name = "queryId") String queryId, @RequestParam MultiValueMap parameters, + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return queryManagementService.duplicate(queryId, parameters, currentUser); + } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 6da293a8..6997297f 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -637,7 +637,41 @@ public void close(String queryId) throws InterruptedException, QueryException { // publish a close event to the executor pool publishExecutorEvent(QueryRequest.close(queryId), queryStatus.getQueryKey().getQueryPool().getName()); } - + + /** + * Resets the query named by {@code queryId}. If the query is not alive, meaning that the current session has expired (due to either timeout, or server + * failure), then this will reload the query and start it over. If the query is alive, it closes it and starts the query over. + * + * @param queryId + * @return + * @throws QueryException + */ + public GenericResponse reset(String queryId, ProxiedUserDetails currentUser) throws QueryException { + try { + // make sure the query is valid, and the user can act on it + QueryStatus queryStatus = validateRequest(queryId, currentUser); + + // cancel the query if it is running + if (queryStatus.getQueryState() == CREATED) { + cancel(queryStatus.getQueryKey().getQueryId().toString(), true); + } + + // create a new query which is an exact copy of the specified query + TaskKey taskKey = duplicate(queryStatus, new LinkedMultiValueMap<>(), currentUser); + + GenericResponse response = new GenericResponse<>(); + response.addMessage(queryId + " reset."); + response.setResult(taskKey.getQueryId().toString()); + response.setHasResults(true); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error resetting query " + queryId, e); + throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Unknown error resetting query " + queryId); + } + } + /** * Remove (delete) the query * @@ -685,9 +719,7 @@ public GenericResponse update(String queryId, MultiValueMap currentParams = new LinkedMultiValueMap<>(); currentParams.addAll(queryStatus.getQuery().getOptionalQueryParameters()); - queryStatus.getQuery().getParameters().forEach(x -> { - currentParams.add(x.getParameterName(), x.getParameterValue()); - }); + queryStatus.getQuery().getParameters().forEach(x -> currentParams.add(x.getParameterName(), x.getParameterValue())); boolean updated = false; if (queryStatus.getQueryState() == DEFINED) { @@ -718,9 +750,7 @@ public GenericResponse update(String queryId, MultiValueMap { - status.setQuery(query); - }); + queryStatusUpdateHelper.lockedUpdate(UUID.fromString(queryId), status -> status.setQuery(query)); } if (!ignoredParams.isEmpty()) { @@ -747,7 +777,44 @@ public GenericResponse update(String queryId, MultiValueMap duplicate(String queryId, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + try { + // make sure the query is valid, and the user can act on it + QueryStatus queryStatus = validateRequest(queryId, currentUser); + + // define a duplicate query from the existing query + TaskKey taskKey = duplicate(queryStatus, parameters, currentUser); + + GenericResponse response = new GenericResponse<>(); + response.addMessage(queryId + " duplicated."); + response.setResult(taskKey.getQueryId().toString()); + response.setHasResults(true); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error resetting query " + queryId, e); + throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Unknown error resetting query " + queryId); + } + } + + private TaskKey duplicate(QueryStatus queryStatus, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + // TODO: Are we losing anything by recreating the parameters this way? + // recreate the query parameters + MultiValueMap currentParams = new LinkedMultiValueMap<>(); + currentParams.addAll(queryStatus.getQuery().getOptionalQueryParameters()); + queryStatus.getQuery().getParameters().forEach(x -> { + currentParams.add(x.getParameterName(), x.getParameterValue()); + }); + + // updated all of the passed in parameters + updateParameters(parameters, currentParams); + + // define a duplicate query + return storeQuery(queryStatus.getQuery().getQueryLogicName(), currentParams, currentUser, true); + } + /** * Updates the current params with the new params. * @@ -913,10 +980,6 @@ else if (queryLogic.getMaxConcurrentTasks() > 0) { } } - protected Query createQuery(String queryLogicName, MultiValueMap parameters, String userDn, List dnList) { - return createQuery(queryLogicName, parameters, userDn, dnList, null); - } - protected Query createQuery(String queryLogicName, MultiValueMap parameters, String userDn, List dnList, String queryId) { Query q = responseObjectFactory.getQueryImpl(); q.initialize(userDn, dnList, queryLogicName, queryParameters, queryParameters.getUnknownParameters(parameters)); From df8319038c92612f2c519958743c54dfb2c9451c Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 9 Jun 2021 17:36:53 -0400 Subject: [PATCH 066/218] formatting --- .../microservice/query/QueryController.java | 15 ++++++------ .../query/QueryManagementService.java | 24 +++++++++---------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index e064fa72..93e960e2 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -102,14 +102,15 @@ public VoidResponse adminClose(@PathVariable(name = "queryId") String queryId, @ throws QueryException { return queryManagementService.adminClose(queryId, currentUser); } - + @Timed(name = "dw.query.reset", absolute = true) @RequestMapping(path = "{queryId}/reset", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public GenericResponse reset(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public GenericResponse reset(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) + throws QueryException { return queryManagementService.reset(queryId, currentUser); } - + @Timed(name = "dw.query.remove", absolute = true) @RequestMapping(path = "{queryId}/remove", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) @@ -124,12 +125,12 @@ public GenericResponse update(@PathVariable(name = "queryId") String que @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.update(queryId, parameters, currentUser); } - + @Timed(name = "dw.query.duplicate", absolute = true) - @RequestMapping(path = "{queryId}/duplicate", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", - "application/x-protostuff"}) + @RequestMapping(path = "{queryId}/duplicate", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public GenericResponse duplicate(@PathVariable(name = "queryId") String queryId, @RequestParam MultiValueMap parameters, - @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.duplicate(queryId, parameters, currentUser); } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 6997297f..3a449264 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -637,7 +637,7 @@ public void close(String queryId) throws InterruptedException, QueryException { // publish a close event to the executor pool publishExecutorEvent(QueryRequest.close(queryId), queryStatus.getQueryKey().getQueryPool().getName()); } - + /** * Resets the query named by {@code queryId}. If the query is not alive, meaning that the current session has expired (due to either timeout, or server * failure), then this will reload the query and start it over. If the query is alive, it closes it and starts the query over. @@ -650,15 +650,15 @@ public GenericResponse reset(String queryId, ProxiedUserDetails currentU try { // make sure the query is valid, and the user can act on it QueryStatus queryStatus = validateRequest(queryId, currentUser); - + // cancel the query if it is running if (queryStatus.getQueryState() == CREATED) { cancel(queryStatus.getQueryKey().getQueryId().toString(), true); } - + // create a new query which is an exact copy of the specified query TaskKey taskKey = duplicate(queryStatus, new LinkedMultiValueMap<>(), currentUser); - + GenericResponse response = new GenericResponse<>(); response.addMessage(queryId + " reset."); response.setResult(taskKey.getQueryId().toString()); @@ -671,7 +671,7 @@ public GenericResponse reset(String queryId, ProxiedUserDetails currentU throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Unknown error resetting query " + queryId); } } - + /** * Remove (delete) the query * @@ -777,15 +777,15 @@ public GenericResponse update(String queryId, MultiValueMap duplicate(String queryId, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { try { // make sure the query is valid, and the user can act on it QueryStatus queryStatus = validateRequest(queryId, currentUser); - + // define a duplicate query from the existing query TaskKey taskKey = duplicate(queryStatus, parameters, currentUser); - + GenericResponse response = new GenericResponse<>(); response.addMessage(queryId + " duplicated."); response.setResult(taskKey.getQueryId().toString()); @@ -798,7 +798,7 @@ public GenericResponse duplicate(String queryId, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { // TODO: Are we losing anything by recreating the parameters this way? // recreate the query parameters @@ -807,14 +807,14 @@ private TaskKey duplicate(QueryStatus queryStatus, MultiValueMap queryStatus.getQuery().getParameters().forEach(x -> { currentParams.add(x.getParameterName(), x.getParameterValue()); }); - + // updated all of the passed in parameters updateParameters(parameters, currentParams); - + // define a duplicate query return storeQuery(queryStatus.getQuery().getQueryLogicName(), currentParams, currentUser, true); } - + /** * Updates the current params with the new params. * From c522f76bc0b49092bc754f5433a0c4ebc9ef5e24 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 10 Jun 2021 15:08:26 -0400 Subject: [PATCH 067/218] Added an implementation for multiple query service controller methods: adminRemove, adminRemoveAll, adminCancelAll, adminCloseAll, list, adminList, & get query. --- query-microservices/query-service/README.md | 77 +++++---- .../microservice/query/QueryController.java | 88 ++++++++-- .../query/QueryManagementService.java | 154 ++++++++++++++++-- .../query/config/QueryServiceConfig.java | 4 +- .../query/monitor/MonitorTask.java | 42 ++--- .../monitor/config/MonitorProperties.java | 4 +- .../microservice/query/QueryServiceTest.java | 4 - 7 files changed, 271 insertions(+), 102 deletions(-) diff --git a/query-microservices/query-service/README.md b/query-microservices/query-service/README.md index 98bbc151..da8598d4 100644 --- a/query-microservices/query-service/README.md +++ b/query-microservices/query-service/README.md @@ -11,42 +11,47 @@ query capabilities. ### User API -| Done? | Method | Operation | Description | Path Param | Request Body | Response Body | -|:--------|:--------------|:-------------------------------------|:----------------------------------------------------------------------------------|:-------------------|:---------------------|:-------------------------------------------| -| ✓ | `GET` | /listQueryLogic | List QueryLogic types that are currently available | N/A | N/A | [QueryLogicResponse] | -| ✓ | `POST` | /{queryLogic}/define | Define a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | -| ✓ | `POST` | /{queryLogic}/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | -| | `POST` | /{queryLogic}/plan | Generate a query plan using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | -| | `POST` | /{queryLogic}/predict | Generate a query prediction using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | -| | `POST` | /{queryLogic}/async/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | -| ✓ | `PUT` `POST` | /{id}/reset | Resets the specified query | [QueryId] | N/A | [VoidResponse]
[GenericResponse] | -| ✓ | `POST` | /{queryLogic}/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | -| | `POST` | /{queryLogic}/async/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | -| | `GET` | /lookupContentUUID/{uuidType}/{uuid} | Returns content associated with the given UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] or [StreamingOutput] | -| | `POST` | /lookupContentUUID | Returns content associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] or [StreamingOutput] | -| | `GET` | /lookupUUID/{uuidType}/{uuid} | Returns event associated with the given batch of UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] or [StreamingOutput] | -| | `POST` | /lookupUUID | Returns event(s) associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] or [StreamingOutput] | -| | `GET` | /{id}/plan | Returns the plan for the specified query | [QueryId] | N/A | [GenericResponse] | -| | `GET` | /{id}/predictions | Returns the predictions for the specified query | [QueryId] | N/A | [GenericResponse] | -| | `GET` | /{id}/async/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | -| ✓ | `GET` | /{id}/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | -| ✓ | `PUT` `POST` | /{id}/close | Closes the specified query | [QueryId] | N/A | [VoidResponse] | -| ✓ | `PUT` `POST` | /{id}/adminClose | Closes the specified query | [QueryId] | N/A | [VoidResponse] | -| ✓ | `PUT` `POST` | /{id}/cancel | Cancels the specified query | [QueryId] | N/A | [VoidResponse] | -| ✓ | `PUT` `POST` | /{id}/adminCancel | Cancels the specified query | [QueryId] | N/A | [VoidResponse] | -| | `GET` | /listAll | Returns a list of queries associated with the current user | N/A | N/A | [QueryImplListResponse] | -| | `GET` | /{id} | Returns query info for the specified query | [QueryId] | N/A | [QueryImplListResponse] | -| | `GET` | /list | Returns a list of queries associated with the current user with given name | N/A | [Name] | [QueryImplListResponse] | -| ✓ | `DELETE` | /{id}/remove | Remove (delete) the specified query | [QueryId] | N/A | [VoidResponse] | -| ✓ | `POST` | /{id}/duplicate | Duplicates the specified query | [QueryId] | [QueryParameters] | [GenericResponse] | -| ✓ | `PUT` `POST` | /{id}/update | Updates the specified query | [QueryId] | [QueryParameters] | [GenericResponse] | -| | `GET` | /{id}/listAll | Returns a list of queries associated with the specified user | [UserId] | N/A | [QueryImplListResponse] | -| | `POST` | /purgeQueryCache | Purges the cache of query objects | N/A | N/A | [VoidResponse] | -| | `GET` | /enableTracing | Enables tracing for queries which match the given criteria | N/A | [QueryRegex], [User] | [VoidResponse] | -| | `GET` | /disableTracing | Disables tracing for queries which match the given criteria | N/A | [QueryRegex], [User] | [VoidResponse] | -| | `GET` | /disableAllTracing | Disables tracing for all queries | N/A | N/A | [VoidResponse] | -| | `POST` | /{logicName}/execute | Create a query using the specified query logic and params, and stream the results | [QueryLogicName] | [QueryParameters] | [StreamingOutput] | -| | `POST` | /{logicName}/async/execute | Create a query using the specified query logic and params, and stream the results | [QueryLogicName] | [QueryParameters] | [StreamingOutput] | +| Done? | New? |Admin? | Method | Operation | Description | Path Param | Request Body | Response Body | +|:--------|:--------|:--------|:--------------|:-----------------------------------------|:-----------------------------------------------------------------------------------------------------------|:------------------------|:-------------------------------|:-------------------------------------------| +| ✓ | | | `GET` | /listQueryLogic | List QueryLogic types that are currently available | N/A | N/A | [QueryLogicResponse] | +| ✓ | | | `POST` | /{queryLogic}/define | Define a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| ✓ | | | `POST` | /{queryLogic}/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| | | | `POST` | /{queryLogic}/plan | Generate a query plan using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| | | | `POST` | /{queryLogic}/predict | Generate a query prediction using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| ✓ | | | `POST` | /{queryLogic}/async/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| ✓ | | | `PUT` `POST` | /{id}/reset | Resets the specified query | [QueryId] | N/A | [VoidResponse]
[GenericResponse] | +| ✓ | | | `POST` | /{queryLogic}/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | +| ✓ | | | `POST` | /{queryLogic}/async/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | +| | | | `GET` | /lookupContentUUID/{uuidType}/{uuid} | Returns content associated with the given UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] or [StreamingOutput] | +| | | | `POST` | /lookupContentUUID | Returns content associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] or [StreamingOutput] | +| | | | `GET` | /lookupUUID/{uuidType}/{uuid} | Returns event associated with the given batch of UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] or [StreamingOutput] | +| | | | `POST` | /lookupUUID | Returns event(s) associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] or [StreamingOutput] | +| | | | `GET` | /{id}/plan | Returns the plan for the specified query | [QueryId] | N/A | [GenericResponse] | +| | | | `GET` | /{id}/predictions | Returns the predictions for the specified query | [QueryId] | N/A | [GenericResponse] | +| ✓ | | | `GET` | /{id}/async/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | +| ✓ | | | `GET` | /{id}/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | +| ✓ | | | `PUT` `POST` | /{id}/close | Closes the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | | ✓ | `PUT` `POST` | /{id}/adminClose | Closes the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | ✓ | ✓ | `PUT` `POST` | /adminCloseAll | Closes all running queries | N/A | N/A | [VoidResponse] | +| ✓ | | | `PUT` `POST` | /{id}/cancel | Cancels the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | | ✓ | `PUT` `POST` | /{id}/adminCancel | Cancels the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | ✓ | ✓ | `PUT` `POST` | /adminCancelAll | Cancels all running queries | N/A | N/A | [VoidResponse] | +| ✓ | | | `GET` | /listAll | Returns a list of queries associated with the current user | N/A | N/A | [QueryImplListResponse] | +| ✓ | | | `GET` | /{id} | Returns query info for the specified query | [QueryId] | N/A | [QueryImplListResponse] | +| ✓ | | | `GET` | /list | Returns a list of queries for this caller, filtering by the (optional) query id, and (optional) query name | N/A | [QueryId], [QueryName] | [QueryImplListResponse] | +| ✓ | ✓ | ✓ | `GET` | /adminList | Returns a list of queries, filtered by the (optional) user, (optional) query id, and (optional) query name | N/A | [User], [QueryId], [QueryName] | [QueryImplListResponse] | +| ✓ | | | `DELETE` | /{id}/remove | Remove (delete) the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | ✓ | ✓ | `DELETE` | /{id}/adminRemove | Remove (delete) the specified query | [QueryId] | N/A | [VoidResponse] | +| ✓ | ✓ | ✓ | `DELETE` | /{id}/adminRemoveAll | Removes all queries which aren't running | N/A | N/A | [VoidResponse] | +| ✓ | | | `POST` | /{id}/duplicate | Duplicates the specified query | [QueryId] | [QueryParameters] | [GenericResponse] | +| ✓ | | | `PUT` `POST` | /{id}/update | Updates the specified query | [QueryId] | [QueryParameters] | [GenericResponse] | +| ✓ | | | `GET` | /{id}/listAll | Returns a list of queries associated with the specified user | [UserId] | N/A | [QueryImplListResponse] | +| ✓ | | | `POST` | /purgeQueryCache | Purges the cache of query objects | N/A | N/A | [VoidResponse] | +| ✓ | | | `GET` | /enableTracing | Enables tracing for queries which match the given criteria | N/A | [QueryRegex], [User] | [VoidResponse] | +| ✓ | | | `GET` | /disableTracing | Disables tracing for queries which match the given criteria | N/A | [QueryRegex], [User] | [VoidResponse] | +| ✓ | | | `GET` | /disableAllTracing | Disables tracing for all queries | N/A | N/A | [VoidResponse] | +| | | | `POST` | /{logicName}/execute | Create a query using the specified query logic and params, and stream the results | [QueryLogicName] | [QueryParameters] | [StreamingOutput] | +| | | | `POST` | /{logicName}/async/execute | Create a query using the specified query logic and params, and stream the results | [QueryLogicName] | [QueryParameters] | [StreamingOutput] | --- diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 93e960e2..208602e1 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -6,6 +6,7 @@ import datawave.webservice.query.exception.QueryException; import datawave.webservice.result.BaseQueryResponse; import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.QueryImplListResponse; import datawave.webservice.result.QueryLogicResponse; import datawave.webservice.result.VoidResponse; import org.springframework.http.MediaType; @@ -39,7 +40,7 @@ public QueryLogicResponse listQueryLogic() { @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE) @RequestMapping(path = "{queryLogic}/define", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public GenericResponse define(@PathVariable(name = "queryLogic") String queryLogic, @RequestParam MultiValueMap parameters, + public GenericResponse define(@PathVariable String queryLogic, @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.define(queryLogic, parameters, currentUser); } @@ -48,7 +49,7 @@ public GenericResponse define(@PathVariable(name = "queryLogic") String @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE) @RequestMapping(path = "{queryLogic}/create", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public GenericResponse create(@PathVariable(name = "queryLogic") String queryLogic, @RequestParam MultiValueMap parameters, + public GenericResponse create(@PathVariable String queryLogic, @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.create(queryLogic, parameters, currentUser); } @@ -57,7 +58,7 @@ public GenericResponse create(@PathVariable(name = "queryLogic") String @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE_AND_NEXT) @RequestMapping(path = "{queryLogic}/createAndNext", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public BaseQueryResponse createQueryAndNext(@PathVariable(name = "queryLogic") String queryLogic, @RequestParam MultiValueMap parameters, + public BaseQueryResponse createQueryAndNext(@PathVariable String queryLogic, @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.createAndNext(queryLogic, parameters, currentUser); } @@ -66,15 +67,14 @@ public BaseQueryResponse createQueryAndNext(@PathVariable(name = "queryLogic") S @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.NEXT) @RequestMapping(path = "{queryId}/next", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public BaseQueryResponse next(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) - throws QueryException { + public BaseQueryResponse next(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.next(queryId, currentUser); } @Timed(name = "dw.query.cancel", absolute = true) @RequestMapping(path = "{queryId}/cancel", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public VoidResponse cancel(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public VoidResponse cancel(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.cancel(queryId, currentUser); } @@ -82,15 +82,14 @@ public VoidResponse cancel(@PathVariable(name = "queryId") String queryId, @Auth @RolesAllowed({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "{queryId}/adminCancel", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public VoidResponse adminCancel(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) - throws QueryException { + public VoidResponse adminCancel(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.adminCancel(queryId, currentUser); } @Timed(name = "dw.query.close", absolute = true) @RequestMapping(path = "{queryId}/close", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public VoidResponse close(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public VoidResponse close(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.close(queryId, currentUser); } @@ -98,30 +97,36 @@ public VoidResponse close(@PathVariable(name = "queryId") String queryId, @Authe @RolesAllowed({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "{queryId}/adminClose", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public VoidResponse adminClose(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) - throws QueryException { + public VoidResponse adminClose(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.adminClose(queryId, currentUser); } @Timed(name = "dw.query.reset", absolute = true) @RequestMapping(path = "{queryId}/reset", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public GenericResponse reset(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) - throws QueryException { + public GenericResponse reset(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.reset(queryId, currentUser); } @Timed(name = "dw.query.remove", absolute = true) @RequestMapping(path = "{queryId}/remove", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public VoidResponse remove(@PathVariable(name = "queryId") String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public VoidResponse remove(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.remove(queryId, currentUser); } + @Timed(name = "dw.query.adminRemove", absolute = true) + @RolesAllowed({"Administrator", "JBossAdministrator"}) + @RequestMapping(path = "{queryId}/adminRemove", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse adminRemove(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return queryManagementService.adminRemove(queryId, currentUser); + } + @Timed(name = "dw.query.update", absolute = true) @RequestMapping(path = "{queryId}/update", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public GenericResponse update(@PathVariable(name = "queryId") String queryId, @RequestParam MultiValueMap parameters, + public GenericResponse update(@PathVariable String queryId, @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.update(queryId, parameters, currentUser); } @@ -129,8 +134,59 @@ public GenericResponse update(@PathVariable(name = "queryId") String que @Timed(name = "dw.query.duplicate", absolute = true) @RequestMapping(path = "{queryId}/duplicate", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public GenericResponse duplicate(@PathVariable(name = "queryId") String queryId, @RequestParam MultiValueMap parameters, + public GenericResponse duplicate(@PathVariable String queryId, @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.duplicate(queryId, parameters, currentUser); } + + @Timed(name = "dw.query.list", absolute = true) + @RequestMapping(path = "list", method = {RequestMethod.GET}, produces = {"text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", + "application/x-protobuf", "application/x-protostuff"}) + public QueryImplListResponse list(@RequestParam(required = false) String id, @RequestParam(required = false) String queryName, + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return queryManagementService.list(id, queryName, currentUser); + } + + @Timed(name = "dw.query.adminList", absolute = true) + @RolesAllowed({"Administrator", "JBossAdministrator"}) + @RequestMapping(path = "adminList", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", + "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public QueryImplListResponse adminList(@RequestParam(required = false) String id, @RequestParam(required = false) String user, + @RequestParam(required = false) String queryName, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return queryManagementService.adminList(id, queryName, user, currentUser); + } + + @Timed(name = "dw.query.get", absolute = true) + @RequestMapping(path = "{queryId}", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", + "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public QueryImplListResponse get(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return queryManagementService.list(queryId, null, currentUser); + } + + // TODO: Update this to cancel all queries on a per-pool basis + @Timed(name = "dw.query.adminCancelAll", absolute = true) + @RolesAllowed({"Administrator", "JBossAdministrator"}) + @RequestMapping(path = "adminCancelAll", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse adminCancelAll(@AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return queryManagementService.adminCancelAll(currentUser); + } + + // TODO: Update this to close all queries on a per-pool basis + @Timed(name = "dw.query.adminCloseAll", absolute = true) + @RolesAllowed({"Administrator", "JBossAdministrator"}) + @RequestMapping(path = "adminCloseAll", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse adminCloseAll(@AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return queryManagementService.adminCancelAll(currentUser); + } + + // TODO: Update this to remove all queries on a per-pool basis + @Timed(name = "dw.query.adminRemoveAll", absolute = true) + @RolesAllowed({"Administrator", "JBossAdministrator"}) + @RequestMapping(path = "adminRemoveAll", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public VoidResponse adminRemoveAll(@AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return queryManagementService.adminRemoveAll(currentUser); + } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 3a449264..fbfdc789 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -40,6 +40,7 @@ import datawave.webservice.query.util.QueryUncaughtExceptionHandler; import datawave.webservice.result.BaseQueryResponse; import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.QueryImplListResponse; import datawave.webservice.result.QueryLogicResponse; import datawave.webservice.result.VoidResponse; import org.slf4j.Logger; @@ -69,6 +70,7 @@ import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CANCELED; import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CLOSED; @@ -496,6 +498,26 @@ public VoidResponse adminCancel(String queryId, ProxiedUserDetails currentUser) return cancel(queryId, currentUser, true); } + public VoidResponse adminCancelAll(ProxiedUserDetails currentUser) throws QueryException { + try { + List queryStatuses = queryStorageCache.getQueryStatus(); + queryStatuses.removeIf(s -> s.getQueryState() != CREATED); + + VoidResponse response = new VoidResponse(); + for (QueryStatus queryStatus : queryStatuses) { + cancel(queryStatus.getQueryKey().getQueryId().toString(), true); + response.addMessage(queryStatus.getQueryKey().getQueryId().toString() + " canceled."); + } + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + QueryException queryException = new QueryException(DatawaveErrorCode.CANCELLATION_ERROR, e, "Error encountered while canceling all queries."); + log.error("Error encountered while canceling all queries", queryException); + throw queryException; + } + } + private VoidResponse cancel(String queryId, ProxiedUserDetails currentUser, boolean adminOverride) throws QueryException { try { // make sure the query is valid, and the user can act on it @@ -503,13 +525,13 @@ private VoidResponse cancel(String queryId, ProxiedUserDetails currentUser, bool VoidResponse response = new VoidResponse(); switch (queryStatus.getQueryState()) { - case DEFINED: case CREATED: - // close the query + // cancel the query cancel(queryId, true); response.addMessage(queryId + " canceled."); break; // TODO: Should we throw an exception for these cases? + case DEFINED: case CLOSED: case CANCELED: case FAILED: @@ -588,6 +610,26 @@ public VoidResponse adminClose(String queryId, ProxiedUserDetails currentUser) t return close(queryId, currentUser, true); } + public VoidResponse adminCloseAll(ProxiedUserDetails currentUser) throws QueryException { + try { + List queryStatuses = queryStorageCache.getQueryStatus(); + queryStatuses.removeIf(s -> s.getQueryState() != CREATED); + + VoidResponse response = new VoidResponse(); + for (QueryStatus queryStatus : queryStatuses) { + close(queryStatus.getQueryKey().getQueryId().toString()); + response.addMessage(queryStatus.getQueryKey().getQueryId().toString() + " closed."); + } + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + QueryException queryException = new QueryException(DatawaveErrorCode.CANCELLATION_ERROR, e, "Error encountered while closing all queries."); + log.error("Error encountered while closing all queries", queryException); + throw queryException; + } + } + private VoidResponse close(String queryId, ProxiedUserDetails currentUser, boolean adminOverride) throws QueryException { try { // make sure the query is valid, and the user can act on it @@ -595,13 +637,13 @@ private VoidResponse close(String queryId, ProxiedUserDetails currentUser, boole VoidResponse response = new VoidResponse(); switch (queryStatus.getQueryState()) { - case DEFINED: case CREATED: // close the query close(queryId); response.addMessage(queryId + " closed."); break; // TODO: Should we throw an exception for these cases? + case DEFINED: case CLOSED: case CANCELED: case FAILED: @@ -680,12 +722,49 @@ public GenericResponse reset(String queryId, ProxiedUserDetails currentU * @return */ public VoidResponse remove(String queryId, ProxiedUserDetails currentUser) throws QueryException { + return remove(queryId, currentUser, false); + } + + /** + * Remove (delete) the query + * + * @param queryId + * @param currentUser + * @return + */ + public VoidResponse adminRemove(String queryId, ProxiedUserDetails currentUser) throws QueryException { + return remove(queryId, currentUser, true); + } + + public VoidResponse adminRemoveAll(ProxiedUserDetails currentUser) throws QueryException { + try { + List queryStatuses = queryStorageCache.getQueryStatus(); + queryStatuses.removeIf(s -> s.getQueryState() == CREATED); + + VoidResponse response = new VoidResponse(); + for (QueryStatus queryStatus : queryStatuses) { + response.addMessage(queryStatus.getQueryKey().getQueryId().toString() + " removed."); + } + return response; + } catch (Exception e) { + QueryException queryException = new QueryException(DatawaveErrorCode.CANCELLATION_ERROR, e, "Error encountered while canceling all queries."); + log.error("Error encountered while canceling all queries", queryException); + throw queryException; + } + } + + private VoidResponse remove(String queryId, ProxiedUserDetails currentUser, boolean adminOverride) throws QueryException { try { // make sure the query is valid, and the user can act on it - QueryStatus queryStatus = validateRequest(queryId, currentUser); + QueryStatus queryStatus = validateRequest(queryId, currentUser, adminOverride); - if (!remove(queryStatus)) { - throw new QueryException("Failed to remove " + queryId, INTERNAL_SERVER_ERROR + "-1"); + // remove the query if it is not running + if (queryStatus.getQueryState() != CREATED) { + if (!remove(queryStatus)) { + throw new QueryException("Failed to remove " + queryId, INTERNAL_SERVER_ERROR + "-1"); + } + } else { + throw new QueryException("Cannot remove a running query.", BAD_REQUEST + "-1"); } VoidResponse response = new VoidResponse(); @@ -700,12 +779,7 @@ public VoidResponse remove(String queryId, ProxiedUserDetails currentUser) throw } private boolean remove(QueryStatus queryStatus) throws IOException { - boolean success = false; - // remove the query from the cache if it is not running - if (queryStatus.getQueryState() != CREATED) { - success = queryStorageCache.deleteQuery(queryStatus.getQueryKey().getQueryId()); - } - return success; + return queryStorageCache.deleteQuery(queryStatus.getQueryKey().getQueryId()); } public GenericResponse update(String queryId, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { @@ -851,6 +925,57 @@ private boolean updateParameters(Collection parameterNames, MultiValueMa return paramsUpdated; } + // list all queries for the user matching the query name, if specified + public QueryImplListResponse list(String queryId, String queryName, ProxiedUserDetails currentUser) throws QueryException { + try { + // looking up queries for this user + String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + + // get all of the query statuses for this user + List queries = list(queryId, queryName, userId); + + QueryImplListResponse response = new QueryImplListResponse(); + response.setQuery(queries); + return response; + } catch (Exception e) { + String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + log.error("Unknown error listing queries for " + userId, e); + throw new QueryException(DatawaveErrorCode.QUERY_LISTING_ERROR, e, "Unknown error listing queries for " + userId); + } + } + + // list all queries + public QueryImplListResponse adminList(String queryId, String queryName, String userId, ProxiedUserDetails currentUser) throws QueryException { + try { + // get all of the query statuses for this user + List queries = list(queryId, queryName, userId); + + QueryImplListResponse response = new QueryImplListResponse(); + response.setQuery(queries); + return response; + } catch (Exception e) { + log.error("Unknown error listing queries for " + userId, e); + throw new QueryException(DatawaveErrorCode.QUERY_LISTING_ERROR, e, "Unknown error listing queries for " + userId); + } + } + + private List list(String queryId, String queryName, String userId) { + List queries; + if (queryId != null && !queryId.isEmpty()) { + // get the query for the given id + queries = new ArrayList<>(); + queries.add(queryStorageCache.getQueryState(UUID.fromString(queryId)).getQueryStatus().getQuery()); + } else { + // get all of the queries + queries = queryStorageCache.getQueryStatus().stream().map(QueryStatus::getQuery).collect(Collectors.toList()); + } + + // only keep queries with the given userId and query name + queries.removeIf(q -> (userId != null && !q.getOwner().equals(userId)) || (queryName != null && !q.getQueryName().equals(queryName))); + + return queries; + } + private QueryStatus validateRequest(String queryId, ProxiedUserDetails currentUser) throws QueryException { return validateRequest(queryId, currentUser, false); } @@ -1044,11 +1169,6 @@ protected void validateParameters(String queryLogicName, MultiValueMap= queryParameters.getExpirationDate().getTime()) { - log.error("Invalid expiration date: " + queryParameters.getExpirationDate()); - throw new BadRequestQueryException(DatawaveErrorCode.INVALID_EXPIRATION_DATE); - } - // Ensure begin date does not occur after the end date (if dates are not null) if ((queryParameters.getBeginDate() != null && queryParameters.getEndDate() != null) && queryParameters.getBeginDate().after(queryParameters.getEndDate())) { diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java index 777bb989..7502effd 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java @@ -20,7 +20,9 @@ public class QueryServiceConfig { @ConditionalOnMissingBean @RequestScope public QueryParameters queryParameters() { - return new DefaultQueryParameters(); + DefaultQueryParameters queryParameters = new DefaultQueryParameters(); + queryParameters.clear(); + return queryParameters; } @Bean diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java index fb43fcb4..c1d19d81 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java @@ -15,6 +15,9 @@ import java.util.UUID; import java.util.concurrent.Callable; +import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CLOSED; +import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CREATED; + public class MonitorTask implements Callable { private final Logger log = LoggerFactory.getLogger(this.getClass()); @@ -63,32 +66,19 @@ public Void call() throws Exception { private void monitor(long currentTimeMillis) { for (QueryStatus status : queryStorageCache.getQueryStatus()) { String queryId = status.getQueryKey().getQueryId().toString(); - switch (status.getQueryState()) { - case CREATED: - if (status.isProgressIdle(currentTimeMillis, expirationProperties.getProgressTimeoutMillis())) { - // if progress hasn't been made for the query in a while, apply the shock paddles - defibrillateQuery(queryId, status.getQueryKey().getQueryPool().getName()); - } - // fall through - case DEFINED: - if (status.isUserIdle(currentTimeMillis, expirationProperties.getIdleTimeoutMillis())) { - // if the user hasn't interacted with the query in a while, cancel it - cancelQuery(queryId); - } - break; - case CLOSED: - if (status.getConcurrentNextCount() > 0 && status.isProgressIdle(currentTimeMillis, expirationProperties.getProgressTimeoutMillis())) { - // if there are next calls waiting for results, and progress hasn't been made for the query in a while, apply the shock paddles - defibrillateQuery(queryId, status.getQueryKey().getQueryPool().getName()); - } - // fall through - case CANCELED: - case FAILED: - if (status.isInactive(currentTimeMillis, monitorProperties.getInactiveQueryTimeToLiveMillis())) { - // if the query has been inactive (closed/canceled/failed) for too long, evict it - deleteQuery(UUID.fromString(queryId)); - } - break; + + // if the query is not running, and has been inactive for too long, evict it + if (status.getQueryState() != CREATED && status.isInactive(currentTimeMillis, monitorProperties.getInactiveQueryTimeToLiveMillis())) { + deleteQuery(UUID.fromString(queryId)); + } + // if the query is running, or closed and in the process of finishing up it's last next call, but it isn't making progress, poke it + else if ((status.getQueryState() == CREATED || (status.getQueryState() == CLOSED && status.getConcurrentNextCount() > 0) + && status.isProgressIdle(currentTimeMillis, expirationProperties.getProgressTimeoutMillis()))) { + defibrillateQuery(queryId, status.getQueryKey().getQueryPool().getName()); + } + // if the query is running, but the user hasn't interacted with it in a while, cancel it + else if (status.getQueryState() == CREATED && status.isUserIdle(currentTimeMillis, expirationProperties.getIdleTimeoutMillis())) { + cancelQuery(queryId); } } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java index 9147a960..e887ad1a 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java @@ -28,9 +28,9 @@ public class MonitorProperties { private TimeUnit lockLeaseTimeUnit = TimeUnit.MILLISECONDS; // The amount of time that an inactive query should remain in the query cache @PositiveOrZero - private long inactiveQueryTimeToLive = 24; + private long inactiveQueryTimeToLive = 1; @NotNull - private TimeUnit inactiveQueryTimeUnit = TimeUnit.HOURS; + private TimeUnit inactiveQueryTimeUnit = TimeUnit.DAYS; public String getSchedulerCrontab() { return schedulerCrontab; diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java index 790b9d61..3b82df18 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java @@ -67,9 +67,7 @@ public void testDefineQuery() { MultiValueMap map = new LinkedMultiValueMap<>(); map.set(DefaultQueryParameters.QUERY_STRING, "FIELD:SOME_VALUE"); map.set(DefaultQueryParameters.QUERY_NAME, "The Greatest Query in the World - Tribute"); - map.set(DefaultQueryParameters.QUERY_PERSISTENCE, "PERSISTENT"); map.set(DefaultQueryParameters.QUERY_AUTHORIZATIONS, "ALL"); - map.set(DefaultQueryParameters.QUERY_EXPIRATION, "20500101 000000.000"); map.set(DefaultQueryParameters.QUERY_BEGIN, "20000101 000000.000"); map.set(DefaultQueryParameters.QUERY_END, "20500101 000000.000"); map.set(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, "ALL"); @@ -98,9 +96,7 @@ public void testCreateQuery() { MultiValueMap map = new LinkedMultiValueMap<>(); map.set(DefaultQueryParameters.QUERY_STRING, "FIELD:SOME_VALUE"); map.set(DefaultQueryParameters.QUERY_NAME, "The Greatest Query in the World - Tribute"); - map.set(DefaultQueryParameters.QUERY_PERSISTENCE, "PERSISTENT"); map.set(DefaultQueryParameters.QUERY_AUTHORIZATIONS, "ALL"); - map.set(DefaultQueryParameters.QUERY_EXPIRATION, "20500101 000000.000"); map.set(DefaultQueryParameters.QUERY_BEGIN, "20000101 000000.000"); map.set(DefaultQueryParameters.QUERY_END, "20500101 000000.000"); map.set(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, "ALL"); From fc360948efdec1bb39bd569d83852a9390bdf971 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 10 Jun 2021 17:01:32 -0400 Subject: [PATCH 068/218] Added logging for admin user actions in the query controller. --- .../microservice/query/QueryController.java | 8 +-- .../query/QueryManagementService.java | 59 ++++++++----------- 2 files changed, 29 insertions(+), 38 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 208602e1..efb6c213 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -163,7 +163,7 @@ public QueryImplListResponse get(@PathVariable String queryId, @AuthenticationPr return queryManagementService.list(queryId, null, currentUser); } - // TODO: Update this to cancel all queries on a per-pool basis + // TODO: Update this to cancel all queries on a per-pool basis? @Timed(name = "dw.query.adminCancelAll", absolute = true) @RolesAllowed({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "adminCancelAll", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", @@ -172,16 +172,16 @@ public VoidResponse adminCancelAll(@AuthenticationPrincipal ProxiedUserDetails c return queryManagementService.adminCancelAll(currentUser); } - // TODO: Update this to close all queries on a per-pool basis + // TODO: Update this to close all queries on a per-pool basis? @Timed(name = "dw.query.adminCloseAll", absolute = true) @RolesAllowed({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "adminCloseAll", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public VoidResponse adminCloseAll(@AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { - return queryManagementService.adminCancelAll(currentUser); + return queryManagementService.adminCloseAll(currentUser); } - // TODO: Update this to remove all queries on a per-pool basis + // TODO: Update this to remove all queries on a per-pool basis? @Timed(name = "dw.query.adminRemoveAll", absolute = true) @RolesAllowed({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "adminRemoveAll", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index fbfdc789..882fb1db 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -495,10 +495,12 @@ public VoidResponse cancel(String queryId, ProxiedUserDetails currentUser) throw * @throws QueryException */ public VoidResponse adminCancel(String queryId, ProxiedUserDetails currentUser) throws QueryException { + log.info("Cancel '" + queryId + "'called by admin: " + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); return cancel(queryId, currentUser, true); } public VoidResponse adminCancelAll(ProxiedUserDetails currentUser) throws QueryException { + log.info("Cancel All called by admin: " + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); try { List queryStatuses = queryStorageCache.getQueryStatus(); queryStatuses.removeIf(s -> s.getQueryState() != CREATED); @@ -607,10 +609,12 @@ public VoidResponse close(String queryId, ProxiedUserDetails currentUser) throws * @throws QueryException */ public VoidResponse adminClose(String queryId, ProxiedUserDetails currentUser) throws QueryException { + log.info("Close '" + queryId + "'called by admin: " + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); return close(queryId, currentUser, true); } public VoidResponse adminCloseAll(ProxiedUserDetails currentUser) throws QueryException { + log.info("Close All called by admin: " + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); try { List queryStatuses = queryStorageCache.getQueryStatus(); queryStatuses.removeIf(s -> s.getQueryState() != CREATED); @@ -733,10 +737,12 @@ public VoidResponse remove(String queryId, ProxiedUserDetails currentUser) throw * @return */ public VoidResponse adminRemove(String queryId, ProxiedUserDetails currentUser) throws QueryException { + log.info("Remove '" + queryId + "'called by admin: " + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); return remove(queryId, currentUser, true); } public VoidResponse adminRemoveAll(ProxiedUserDetails currentUser) throws QueryException { + log.info("Remove All called by admin: " + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); try { List queryStatuses = queryStorageCache.getQueryStatus(); queryStatuses.removeIf(s -> s.getQueryState() == CREATED); @@ -927,28 +933,30 @@ private boolean updateParameters(Collection parameterNames, MultiValueMa // list all queries for the user matching the query name, if specified public QueryImplListResponse list(String queryId, String queryName, ProxiedUserDetails currentUser) throws QueryException { - try { - // looking up queries for this user - String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); - - // get all of the query statuses for this user - List queries = list(queryId, queryName, userId); - - QueryImplListResponse response = new QueryImplListResponse(); - response.setQuery(queries); - return response; - } catch (Exception e) { - String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); - log.error("Unknown error listing queries for " + userId, e); - throw new QueryException(DatawaveErrorCode.QUERY_LISTING_ERROR, e, "Unknown error listing queries for " + userId); - } + return list(queryId, queryName, ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); } // list all queries public QueryImplListResponse adminList(String queryId, String queryName, String userId, ProxiedUserDetails currentUser) throws QueryException { + log.info("List '" + String.join(",", Arrays.asList(queryId, queryName, userId)) + "'called by admin: " + + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); + return list(queryId, queryName, userId); + } + + private QueryImplListResponse list(String queryId, String queryName, String userId) throws QueryException { try { - // get all of the query statuses for this user - List queries = list(queryId, queryName, userId); + List queries; + if (queryId != null && !queryId.isEmpty()) { + // get the query for the given id + queries = new ArrayList<>(); + queries.add(queryStorageCache.getQueryState(UUID.fromString(queryId)).getQueryStatus().getQuery()); + } else { + // get all of the queries + queries = queryStorageCache.getQueryStatus().stream().map(QueryStatus::getQuery).collect(Collectors.toList()); + } + + // only keep queries with the given userId and query name + queries.removeIf(q -> (userId != null && !q.getOwner().equals(userId)) || (queryName != null && !q.getQueryName().equals(queryName))); QueryImplListResponse response = new QueryImplListResponse(); response.setQuery(queries); @@ -959,23 +967,6 @@ public QueryImplListResponse adminList(String queryId, String queryName, String } } - private List list(String queryId, String queryName, String userId) { - List queries; - if (queryId != null && !queryId.isEmpty()) { - // get the query for the given id - queries = new ArrayList<>(); - queries.add(queryStorageCache.getQueryState(UUID.fromString(queryId)).getQueryStatus().getQuery()); - } else { - // get all of the queries - queries = queryStorageCache.getQueryStatus().stream().map(QueryStatus::getQuery).collect(Collectors.toList()); - } - - // only keep queries with the given userId and query name - queries.removeIf(q -> (userId != null && !q.getOwner().equals(userId)) || (queryName != null && !q.getQueryName().equals(queryName))); - - return queries; - } - private QueryStatus validateRequest(String queryId, ProxiedUserDetails currentUser) throws QueryException { return validateRequest(queryId, currentUser, false); } From ef3ae6b4f05009357241f6d5b1dd848845a9d9c0 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Fri, 11 Jun 2021 15:40:38 +0000 Subject: [PATCH 069/218] Refactored the query storage to be in the query starter instead --- .../query-service/service/pom.xml | 9 ++++++++- .../query/QueryManagementService.java | 18 +++++++++--------- .../query/monitor/MonitorTask.java | 8 ++++---- .../query/monitor/QueryMonitor.java | 2 +- .../microservice/query/runner/NextCall.java | 10 +++++----- .../query/status/QueryStatusUpdateHelper.java | 8 ++++---- .../query/status/StatusUpdater.java | 2 +- .../QueryMetricsEnrichmentFilterAdvice.java | 4 ++-- 8 files changed, 34 insertions(+), 27 deletions(-) diff --git a/query-microservices/query-service/service/pom.xml b/query-microservices/query-service/service/pom.xml index 4d640aa3..a317a852 100644 --- a/query-microservices/query-service/service/pom.xml +++ b/query-microservices/query-service/service/pom.xml @@ -40,6 +40,13 @@ pom import + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-query + ${version.microservice.starter-query} + test-jar + test + @@ -65,7 +72,7 @@ gov.nsa.datawave.microservice - query-storage + spring-boot-starter-datawave-query test-jar test diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 882fb1db..07617369 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -7,11 +7,11 @@ import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.authorization.util.AuthorizationsUtil; import datawave.microservice.common.audit.PrivateAuditConstants; -import datawave.microservice.common.storage.QueryPool; -import datawave.microservice.common.storage.QueryQueueManager; -import datawave.microservice.common.storage.QueryStatus; -import datawave.microservice.common.storage.QueryStorageCache; -import datawave.microservice.common.storage.TaskKey; +import datawave.microservice.query.logic.QueryPool; +import datawave.microservice.query.storage.QueryQueueManager; +import datawave.microservice.query.storage.QueryStatus; +import datawave.microservice.query.storage.QueryStorageCache; +import datawave.microservice.query.storage.TaskKey; import datawave.microservice.query.config.QueryProperties; import datawave.microservice.query.logic.QueryLogic; import datawave.microservice.query.logic.QueryLogicFactory; @@ -72,10 +72,10 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CANCELED; -import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CLOSED; -import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CREATED; -import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.DEFINED; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CANCELED; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CLOSED; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CREATED; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.DEFINED; import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; import static io.undertow.util.StatusCodes.BAD_REQUEST; import static io.undertow.util.StatusCodes.INTERNAL_SERVER_ERROR; diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java index c1d19d81..7eed832e 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java @@ -1,7 +1,7 @@ package datawave.microservice.query.monitor; -import datawave.microservice.common.storage.QueryStatus; -import datawave.microservice.common.storage.QueryStorageCache; +import datawave.microservice.query.storage.QueryStatus; +import datawave.microservice.query.storage.QueryStorageCache; import datawave.microservice.query.QueryManagementService; import datawave.microservice.query.config.QueryExpirationProperties; import datawave.microservice.query.monitor.cache.MonitorStatus; @@ -15,8 +15,8 @@ import java.util.UUID; import java.util.concurrent.Callable; -import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CLOSED; -import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CREATED; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CLOSED; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CREATED; public class MonitorTask implements Callable { private final Logger log = LoggerFactory.getLogger(this.getClass()); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java index 8894eeaa..fed1bbee 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java @@ -1,6 +1,6 @@ package datawave.microservice.query.monitor; -import datawave.microservice.common.storage.QueryStorageCache; +import datawave.microservice.query.storage.QueryStorageCache; import datawave.microservice.query.QueryManagementService; import datawave.microservice.query.config.QueryExpirationProperties; import datawave.microservice.query.config.QueryProperties; diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 238a0387..126cfdd5 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -1,10 +1,10 @@ package datawave.microservice.query.runner; -import datawave.microservice.common.storage.QueryQueueListener; -import datawave.microservice.common.storage.QueryQueueManager; -import datawave.microservice.common.storage.QueryStatus; -import datawave.microservice.common.storage.QueryStorageCache; -import datawave.microservice.common.storage.Result; +import datawave.microservice.query.storage.QueryQueueListener; +import datawave.microservice.query.storage.QueryQueueManager; +import datawave.microservice.query.storage.QueryStatus; +import datawave.microservice.query.storage.QueryStorageCache; +import datawave.microservice.query.storage.Result; import datawave.microservice.query.config.NextCallProperties; import datawave.microservice.query.config.QueryExpirationProperties; import datawave.microservice.query.config.QueryProperties; diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java index a6215fc1..c4440f9d 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java @@ -1,15 +1,15 @@ package datawave.microservice.query.status; -import datawave.microservice.common.storage.QueryStatus; -import datawave.microservice.common.storage.QueryStorageCache; -import datawave.microservice.common.storage.QueryStorageLock; +import datawave.microservice.query.storage.QueryStatus; +import datawave.microservice.query.storage.QueryStorageCache; +import datawave.microservice.query.storage.QueryStorageLock; import datawave.microservice.query.config.QueryProperties; import datawave.webservice.query.exception.DatawaveErrorCode; import datawave.webservice.query.exception.QueryException; import java.util.UUID; -import static datawave.microservice.common.storage.QueryStatus.QUERY_STATE.CREATED; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CREATED; public class QueryStatusUpdateHelper { diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/StatusUpdater.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/StatusUpdater.java index 27c16b89..29c3a30e 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/StatusUpdater.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/StatusUpdater.java @@ -1,6 +1,6 @@ package datawave.microservice.query.status; -import datawave.microservice.common.storage.QueryStatus; +import datawave.microservice.query.storage.QueryStatus; import datawave.webservice.query.exception.DatawaveErrorCode; import datawave.webservice.query.exception.QueryException; diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java index a83b7c22..f18b0d64 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java @@ -1,7 +1,7 @@ package datawave.microservice.query.web.filter; -import datawave.microservice.common.storage.QueryState; -import datawave.microservice.common.storage.QueryStorageCache; +import datawave.microservice.query.storage.QueryState; +import datawave.microservice.query.storage.QueryStorageCache; import datawave.microservice.query.logic.QueryLogicFactory; import datawave.microservice.query.web.QueryMetrics; import datawave.microservice.query.web.annotation.EnrichQueryMetrics; From 7438dbac57627a96c9be472adaabc820ca3043a5 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Fri, 11 Jun 2021 20:13:00 +0000 Subject: [PATCH 070/218] Removed the TaskNotification in lieu of the QueryRequest. Changed queryid and query pool to be String objects for avoid excessive object creations and conversions. --- .../query/QueryManagementService.java | 29 +++++++++---------- .../query/monitor/MonitorTask.java | 11 ++++--- .../microservice/query/runner/NextCall.java | 3 +- .../query/status/QueryStatusUpdateHelper.java | 2 +- 4 files changed, 20 insertions(+), 25 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 07617369..c8f87d65 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -7,7 +7,6 @@ import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.authorization.util.AuthorizationsUtil; import datawave.microservice.common.audit.PrivateAuditConstants; -import datawave.microservice.query.logic.QueryPool; import datawave.microservice.query.storage.QueryQueueManager; import datawave.microservice.query.storage.QueryStatus; import datawave.microservice.query.storage.QueryStorageCache; @@ -298,7 +297,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p if (isCreateRequest) { // @formatter:off taskKey = queryStorageCache.createQuery( - new QueryPool(getPoolName()), + getPoolName(), query, AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), currentUser), getMaxConcurrentTasks(queryLogic)); @@ -309,7 +308,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p } else { // @formatter:off taskKey = queryStorageCache.defineQuery( - new QueryPool(getPoolName()), + getPoolName(), query, AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), currentUser), getMaxConcurrentTasks(queryLogic)); @@ -393,14 +392,12 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th * @throws QueryException */ private BaseQueryResponse next(String queryId, Collection userRoles) throws Exception { - UUID queryUUID = UUID.fromString(queryId); - // before we spin up a separate thread, make sure we are allowed to call next boolean success = false; - QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(queryUUID, queryStatusUpdateHelper::claimConcurrentNext); + QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(queryId, queryStatusUpdateHelper::claimConcurrentNext); try { // publish a next event to the executor pool - publishNextEvent(queryId, queryStatus.getQueryKey().getQueryPool().getName()); + publishNextEvent(queryId, queryStatus.getQueryKey().getQueryPool()); // get the query logic String queryLogicName = queryStatus.getQuery().getQueryLogicName(); @@ -431,7 +428,7 @@ private BaseQueryResponse next(String queryId, Collection userRoles) thr BaseQueryResponse response = queryLogic.getTransformer(queryStatus.getQuery()).createResponse(resultsPage); // after all of our work is done, perform our final query status update for this next call - queryStatus = queryStatusUpdateHelper.lockedUpdate(queryUUID, status -> { + queryStatus = queryStatusUpdateHelper.lockedUpdate(queryId, status -> { queryStatusUpdateHelper.releaseConcurrentNext(status); status.setLastPageNumber(status.getLastPageNumber() + 1); status.setNumResultsReturned(status.getNumResultsReturned() + resultsPage.getResults().size()); @@ -467,7 +464,7 @@ private BaseQueryResponse next(String queryId, Collection userRoles) thr } finally { // update query status if we failed if (!success) { - queryStatusUpdateHelper.lockedUpdate(queryUUID, queryStatusUpdateHelper::releaseConcurrentNext); + queryStatusUpdateHelper.lockedUpdate(queryId, queryStatusUpdateHelper::releaseConcurrentNext); } } } @@ -571,7 +568,7 @@ public void cancel(String queryId, boolean publishEvent) throws InterruptedExcep if (publishEvent) { // only the initial event publisher should update the status - QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(UUID.fromString(queryId), status -> { + QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(queryId, status -> { // update query state to CANCELED status.setQueryState(CANCELED); }); @@ -582,7 +579,7 @@ public void cancel(String queryId, boolean publishEvent) throws InterruptedExcep publishSelfEvent(cancelRequest); // publish a cancel event to the executor pool - publishExecutorEvent(cancelRequest, queryStatus.getQueryKey().getQueryPool().getName()); + publishExecutorEvent(cancelRequest, queryStatus.getQueryKey().getQueryPool()); } } @@ -675,13 +672,13 @@ private VoidResponse close(String queryId, ProxiedUserDetails currentUser, boole * @throws QueryException */ public void close(String queryId) throws InterruptedException, QueryException { - QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(UUID.fromString(queryId), status -> { + QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(queryId, status -> { // update query state to CLOSED status.setQueryState(CLOSED); }); // publish a close event to the executor pool - publishExecutorEvent(QueryRequest.close(queryId), queryStatus.getQueryKey().getQueryPool().getName()); + publishExecutorEvent(QueryRequest.close(queryId), queryStatus.getQueryKey().getQueryPool()); } /** @@ -830,7 +827,7 @@ public GenericResponse update(String queryId, MultiValueMap status.setQuery(query)); + queryStatusUpdateHelper.lockedUpdate(queryId, status -> status.setQuery(query)); } if (!ignoredParams.isEmpty()) { @@ -949,7 +946,7 @@ private QueryImplListResponse list(String queryId, String queryName, String user if (queryId != null && !queryId.isEmpty()) { // get the query for the given id queries = new ArrayList<>(); - queries.add(queryStorageCache.getQueryState(UUID.fromString(queryId)).getQueryStatus().getQuery()); + queries.add(queryStorageCache.getQueryState(queryId).getQueryStatus().getQuery()); } else { // get all of the queries queries = queryStorageCache.getQueryStatus().stream().map(QueryStatus::getQuery).collect(Collectors.toList()); @@ -973,7 +970,7 @@ private QueryStatus validateRequest(String queryId, ProxiedUserDetails currentUs private QueryStatus validateRequest(String queryId, ProxiedUserDetails currentUser, boolean adminOverride) throws QueryException { // does the query exist? - QueryStatus queryStatus = queryStorageCache.getQueryStatus(UUID.fromString(queryId)); + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); if (queryStatus == null) { throw new NotFoundQueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH, MessageFormat.format("{0}", queryId)); } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java index 7eed832e..ff2a1649 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java @@ -12,7 +12,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.UUID; import java.util.concurrent.Callable; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CLOSED; @@ -69,12 +68,12 @@ private void monitor(long currentTimeMillis) { // if the query is not running, and has been inactive for too long, evict it if (status.getQueryState() != CREATED && status.isInactive(currentTimeMillis, monitorProperties.getInactiveQueryTimeToLiveMillis())) { - deleteQuery(UUID.fromString(queryId)); + deleteQuery(queryId); } // if the query is running, or closed and in the process of finishing up it's last next call, but it isn't making progress, poke it else if ((status.getQueryState() == CREATED || (status.getQueryState() == CLOSED && status.getConcurrentNextCount() > 0) && status.isProgressIdle(currentTimeMillis, expirationProperties.getProgressTimeoutMillis()))) { - defibrillateQuery(queryId, status.getQueryKey().getQueryPool().getName()); + defibrillateQuery(queryId, status.getQueryKey().getQueryPool()); } // if the query is running, but the user hasn't interacted with it in a while, cancel it else if (status.getQueryState() == CREATED && status.isUserIdle(currentTimeMillis, expirationProperties.getIdleTimeoutMillis())) { @@ -98,11 +97,11 @@ private void defibrillateQuery(String queryId, String queryPool) { queryManagementService.publishNextEvent(queryId, queryPool); } - private void deleteQuery(UUID queryUUID) { + private void deleteQuery(String queryId) { try { - queryStorageCache.deleteQuery(queryUUID); + queryStorageCache.deleteQuery(queryId); } catch (IOException e) { - log.error("Encountered error while trying to evict inactive query: " + queryUUID.toString(), e); + log.error("Encountered error while trying to evict inactive query: " + queryId, e); } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 126cfdd5..083bdd84 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -21,7 +21,6 @@ import java.util.LinkedList; import java.util.List; -import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -257,7 +256,7 @@ private boolean callExpiredTimeout(long callTimeMillis) { private QueryStatus getQueryStatus() { if (queryStatus == null || isQueryStatusExpired()) { lastStatusUpdateTime = System.currentTimeMillis(); - queryStatus = queryStorageCache.getQueryStatus(UUID.fromString(queryId)); + queryStatus = queryStorageCache.getQueryStatus(queryId); } return queryStatus; } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java index c4440f9d..ca01187b 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java @@ -52,7 +52,7 @@ public void releaseConcurrentNext(QueryStatus queryStatus) throws QueryException } } - public QueryStatus lockedUpdate(UUID queryUUID, StatusUpdater updater) throws QueryException, InterruptedException { + public QueryStatus lockedUpdate(String queryUUID, StatusUpdater updater) throws QueryException, InterruptedException { QueryStatus queryStatus = null; QueryStorageLock statusLock = queryStorageCache.getQueryStatusLock(queryUUID); if (statusLock.tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { From 2b1754523a83cbdb9bee75d4845e3a886194d812 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Mon, 14 Jun 2021 14:15:42 -0400 Subject: [PATCH 071/218] Added javadoc for the query management service. --- .../query/QueryManagementService.java | 1032 ++++++++++++++--- .../query/config/QueryServiceConfig.java | 9 +- .../microservice/query/runner/NextCall.java | 7 +- .../query/status/QueryStatusUpdateHelper.java | 3 +- 4 files changed, 893 insertions(+), 158 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index c8f87d65..251bbf73 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -30,6 +30,7 @@ import datawave.webservice.query.exception.DatawaveErrorCode; import datawave.webservice.query.exception.NoResultsQueryException; import datawave.webservice.query.exception.NotFoundQueryException; +import datawave.webservice.query.exception.QueryCanceledQueryException; import datawave.webservice.query.exception.QueryException; import datawave.webservice.query.exception.TimeoutQueryException; import datawave.webservice.query.exception.UnauthorizedQueryException; @@ -76,8 +77,6 @@ import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CREATED; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.DEFINED; import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; -import static io.undertow.util.StatusCodes.BAD_REQUEST; -import static io.undertow.util.StatusCodes.INTERNAL_SERVER_ERROR; @Service public class QueryManagementService implements QueryRequestHandler { @@ -136,9 +135,11 @@ public QueryManagementService(QueryProperties queryProperties, ApplicationContex } /** - * List QueryLogic types that are currently available + * Gets a list of descriptions for the configured query logics, sorted by query logic name. + *

+ * The descriptions include things like the audit type, optional and required parameters, required roles, and response class. * - * @return + * @return the query logic descriptions */ public QueryLogicResponse listQueryLogic() { QueryLogicResponse response = new QueryLogicResponse(); @@ -213,23 +214,41 @@ public QueryLogicResponse listQueryLogic() { } /** - * Defines a datawave query. + * Defines a query using the given query logic and parameters. *

- * Validates the query parameters using the base validation, query logic-specific validation, and markings validation. If the parameters are valid, the - * query will be stored in the query storage cache where it can be acted upon in a subsequent call. + * Defined queries cannot be started and run.
+ * Auditing is not performed when defining a query.
+ * Updates can be made to any parameter using {@link #update}.
+ * Create a runnable query from a defined query using {@link #duplicate} or {@link #reset}.
+ * Delete a defined query using {@link #remove}.
+ * Aside from a limited set of admin actions, only the query owner can act on a defined query. * * @param queryLogicName + * the requested query logic, not null * @param parameters + * the query parameters, not null * @param currentUser - * @return + * the user who called this method, not null + * @return a generic response containing the query id + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws QueryException + * if query storage fails * @throws QueryException + * if there is an unknown error */ public GenericResponse define(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { try { TaskKey taskKey = storeQuery(queryLogicName, parameters, currentUser, false); GenericResponse response = new GenericResponse<>(); - response.setResult(taskKey.getQueryId().toString()); + response.setResult(taskKey.getQueryId()); return response; } catch (QueryException e) { throw e; @@ -240,23 +259,45 @@ public GenericResponse define(String queryLogicName, MultiValueMap - * Validates the query parameters using the base validation, query logic-specific validation, and markings validation. If the parameters are valid, the - * query will be stored in the query storage cache where it can be acted upon in a subsequent call. + * Created queries will start running immediately.
+ * Auditing is performed before the query is started.
+ * Query results can be retrieved using {@link #next}.
+ * Updates can be made to any parameter which doesn't affect the scope of the query using {@link #update}.
+ * Stop a running query gracefully using {@link #close} or forcefully using {@link #cancel}.
+ * Stop, and restart a running query using {@link #reset}.
+ * Create a copy of a running query using {@link #duplicate}.
+ * Aside from a limited set of admin actions, only the query owner can act on a running query. * * @param queryLogicName + * the requested query logic, not null * @param parameters + * the query parameters, not null * @param currentUser - * @return + * the user who called this method, not null + * @return a generic response containing the query id + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails * @throws QueryException + * if query storage fails + * @throws QueryException + * if there is an unknown error */ public GenericResponse create(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { try { TaskKey taskKey = storeQuery(queryLogicName, parameters, currentUser, true); GenericResponse response = new GenericResponse<>(); - response.setResult(taskKey.getQueryId().toString()); + response.setResult(taskKey.getQueryId()); response.setHasResults(true); return response; } catch (QueryException e) { @@ -272,6 +313,39 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p return storeQuery(queryLogicName, parameters, currentUser, isCreateRequest, null); } + /** + * Validates the query request, creates an entry in the query storage cache, and publishes a create event to the executor service. + *

+ * Validation is run against the requested logic, the parameters, and the security markings in {@link #validateQuery}.
+ * Auditing is performed when {@param isCreateRequest} is true using {@link #audit}.
+ * If {@param queryId} is null, a query id will be generated automatically. + * + * @param queryLogicName + * the requested query logic, not null + * @param parameters + * the query parameters, not null + * @param currentUser + * the user who called this method, not null + * @param isCreateRequest + * whether this is a create call true, or a define call false + * @param queryId + * the desired query id, may be null + * @return the task key returned from query storage + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws QueryException + * if there is an unknown error + */ private TaskKey storeQuery(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser, boolean isCreateRequest, String queryId) throws QueryException { // validate query and get a query logic @@ -304,7 +378,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p // @formatter:on // publish a create event to the executor pool - publishExecutorEvent(QueryRequest.create(taskKey.getQueryId().toString()), getPoolName()); + publishExecutorEvent(QueryRequest.create(taskKey.getQueryId()), getPoolName()); } else { // @formatter:off taskKey = queryStorageCache.defineQuery( @@ -322,24 +396,49 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p return taskKey; } catch (Exception e) { log.error("Unknown error storing query", e); - throw new BadRequestQueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR, e); + throw new QueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR, e); } } /** - * Creates a datawave query and calls next. + * Creates a query using the given query logic and parameters, and returns the first page of results. *

- * Validates the query parameters using the base validation, query logic-specific validation, and markings validation. If the parameters are valid, the - * query will be stored in the query storage cache where it can be acted upon. - * - * Once created, gets the next page of results from the query object. The response object type is dynamic, see the listQueryLogic operation to determine - * what the response type object will be. + * Created queries will start running immediately.
+ * Auditing is performed before the query is started.
+ * Subsequent query results can be retrieved using {@link #next}.
+ * Updates can be made to any parameter which doesn't affect the scope of the query using {@link #update}.
+ * Stop a running query gracefully using {@link #close} or forcefully using {@link #cancel}.
+ * Stop, and restart a running query using {@link #reset}.
+ * Create a copy of a running query using {@link #duplicate}.
+ * Aside from a limited set of admin actions, only the query owner can act on a running query. * * @param queryLogicName + * the requested query logic, not null * @param parameters + * the query parameters, not null * @param currentUser - * @return + * the user who called this method, not null + * @return a base query response containing the first page of results + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws TimeoutQueryException + * if the next call times out + * @throws NoResultsQueryException + * if no query results are found * @throws QueryException + * if this next task is rejected by the executor + * @throws QueryException + * if there is an unknown error */ public BaseQueryResponse createAndNext(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { @@ -349,19 +448,45 @@ public BaseQueryResponse createAndNext(String queryLogicName, MultiValueMap + * Next can only be called on a running query.
+ * If configuration allows, multiple next calls may be run concurrently for a query.
+ * Only the query owner can call next on the specified query. * * @param queryId + * the query id, not null * @param currentUser - * @return + * the user who called this method, not null + * @return a base query response containing the next page of results + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws BadRequestQueryException + * if the query is not running + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if the next call is interrupted + * @throws TimeoutQueryException + * if the query times out + * @throws NoResultsQueryException + * if no query results are found * @throws QueryException + * if this next task is rejected by the executor + * @throws QueryException + * if next call execution fails + * @throws QueryException + * if query logic creation fails + * @throws QueryException + * if there is an unknown error */ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) throws QueryException { try { @@ -372,7 +497,7 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th if (queryStatus.getQueryState() == CREATED) { return next(queryId, currentUser.getPrimaryUser().getRoles()); } else { - throw new QueryException("Cannot call next on a query with state: " + queryStatus.getQueryState().name(), BAD_REQUEST + "-1"); + throw new BadRequestQueryException("Cannot call next on a query that is not running"); } } catch (QueryException e) { throw e; @@ -383,15 +508,33 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th } /** - * Gets the next page of results from the query object. If the object is no longer alive, meaning that the current session has expired, then this fail. The - * response object type is dynamic, see the listQueryLogic operation to determine what the response type object will be. + * Gets the next page of results for the given query, and publishes a next event to the executor service. + *

+ * If configuration allows, multiple next calls may be run concurrently for a query. * * @param queryId + * the query id, not null * @param userRoles - * @return + * the roles of the user who called this method, not null + * @return a base query response containing the next page of results + * @throws NotFoundQueryException + * if the query cannot be found * @throws QueryException + * if query lock acquisition fails + * @throws InterruptedException + * if the next call is interrupted + * @throws TimeoutQueryException + * if the query times out + * @throws NoResultsQueryException + * if no query results are found + * @throws QueryException + * if this next task is rejected by the executor + * @throws QueryException + * if next call execution fails + * @throws QueryException + * if query logic creation fails */ - private BaseQueryResponse next(String queryId, Collection userRoles) throws Exception { + private BaseQueryResponse next(String queryId, Collection userRoles) throws InterruptedException, QueryException { // before we spin up a separate thread, make sure we are allowed to call next boolean success = false; QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(queryId, queryStatusUpdateHelper::claimConcurrentNext); @@ -441,8 +584,10 @@ private BaseQueryResponse next(String queryId, Collection userRoles) thr response.setQueryId(queryId); return response; } else { - if (nextCall.getMetric().getLifecycle() == BaseQueryMetric.Lifecycle.NEXTTIMEOUT) { - throw new TimeoutQueryException(DatawaveErrorCode.QUERY_TIMEOUT, MessageFormat.format("{0}", queryId)); + if (nextCall.isCanceled()) { + throw new QueryCanceledQueryException(DatawaveErrorCode.QUERY_CANCELED, MessageFormat.format("{0} canceled;", queryId)); + } else if (nextCall.getMetric().getLifecycle() == BaseQueryMetric.Lifecycle.NEXTTIMEOUT) { + throw new TimeoutQueryException(DatawaveErrorCode.QUERY_TIMEOUT, MessageFormat.format("{0} timed out.", queryId)); } else { throw new NoResultsQueryException(DatawaveErrorCode.NO_QUERY_RESULTS_FOUND, MessageFormat.format("{0}", queryId)); } @@ -450,17 +595,15 @@ private BaseQueryResponse next(String queryId, Collection userRoles) thr } catch (TaskRejectedException e) { throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Next task rejected by the executor for query " + queryId); } catch (ExecutionException e) { - // try to unwrap the execution exception - Throwable cause = e.getCause(); - if (cause instanceof Exception) { - throw (Exception) e.getCause(); - } else { - throw e; - } + // try to unwrap the execution exception and throw a query exception + throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e.getCause(), "Next call execution failed"); } finally { // remove this next call from the map, and decrement the next count for this query nextCallMap.get(queryId).remove(nextCall); } + } catch (CloneNotSupportedException e) { + throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, + "Unable to create instance of the requested query logic " + queryStatus.getQuery().getQueryLogicName()); } finally { // update query status if we failed if (!success) { @@ -470,32 +613,82 @@ private BaseQueryResponse next(String queryId, Collection userRoles) thr } /** - * Releases the resources associated with this query. Any currently running calls to 'next' on the query will be stopped. Calls to 'next' after a 'cancel' - * will start over at page 1. + * Cancels the specified query. + *

+ * Cancel can only be called on a running query, or a query that is in the process of closing.
+ * Outstanding next calls will be stopped immediately, but will return partial results if applicable.
+ * Aside from admins, only the query owner can cancel the specified query. * * @param queryId + * the query id, not null * @param currentUser - * @return + * the user who called this method, not null + * @return a void response indicating that the query was canceled + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws BadRequestQueryException + * if the query is not running * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if the cancel call is interrupted + * @throws QueryException + * if there is an unknown error */ public VoidResponse cancel(String queryId, ProxiedUserDetails currentUser) throws QueryException { return cancel(queryId, currentUser, false); } /** - * Releases the resources associated with this query. Any currently running calls to 'next' on the query will be stopped. Calls to 'next' after a 'cancel' - * will start over at page 1. + * Cancels the specified query using admin privileges. + *

+ * Cancel can only be called on a running query, or a query that is in the process of closing.
+ * Outstanding next calls will be stopped immediately, but will return partial results if applicable.
+ * Only admin users should be allowed to call this method. * * @param queryId + * the query id, not null * @param currentUser - * @return + * the user who called this method, not null + * @return a void response indicating that the query was canceled + * @throws NotFoundQueryException + * if the query cannot be found + * @throws BadRequestQueryException + * if the query is not running + * @throws QueryException + * if query lock acquisition fails * @throws QueryException + * if the cancel call is interrupted + * @throws QueryException + * if there is an unknown error */ public VoidResponse adminCancel(String queryId, ProxiedUserDetails currentUser) throws QueryException { log.info("Cancel '" + queryId + "'called by admin: " + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); return cancel(queryId, currentUser, true); } + /** + * Cancels all queries using admin privileges. + *

+ * Cancel can only be called on a running query, or a query that is in the process of closing.
+ * Queries that are not running will be ignored by this method.
+ * Outstanding next calls will be stopped immediately, but will return partial results if applicable.
+ * Only admin users should be allowed to call this method. + * + * @param currentUser + * the user who called this method, not null + * @return a void response specifying which queries were canceled + * @throws NotFoundQueryException + * if a query cannot be found + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if the cancel call is interrupted + * @throws QueryException + * if there is an unknown error + */ public VoidResponse adminCancelAll(ProxiedUserDetails currentUser) throws QueryException { log.info("Cancel All called by admin: " + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); try { @@ -504,8 +697,8 @@ public VoidResponse adminCancelAll(ProxiedUserDetails currentUser) throws QueryE VoidResponse response = new VoidResponse(); for (QueryStatus queryStatus : queryStatuses) { - cancel(queryStatus.getQueryKey().getQueryId().toString(), true); - response.addMessage(queryStatus.getQueryKey().getQueryId().toString() + " canceled."); + cancel(queryStatus.getQueryKey().getQueryId(), true); + response.addMessage(queryStatus.getQueryKey().getQueryId() + " canceled."); } return response; } catch (QueryException e) { @@ -517,47 +710,76 @@ public VoidResponse adminCancelAll(ProxiedUserDetails currentUser) throws QueryE } } + /** + * Cancels the specified query. + *

+ * Cancel can only be called on a running query, or a query that is in the process of closing.
+ * Outstanding next calls will be stopped immediately, but will return partial results if applicable.
+ * Publishes a cancel event to the query and executor services.
+ * Query ownership will only be validated when {@param adminOverride} is set to true. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @param adminOverride + * whether or not this is an admin action + * @return a void response indicating that the query was canceled + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws BadRequestQueryException + * if the query is not running + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if the cancel call is interrupted + * @throws QueryException + * if there is an unknown error + */ private VoidResponse cancel(String queryId, ProxiedUserDetails currentUser, boolean adminOverride) throws QueryException { try { // make sure the query is valid, and the user can act on it QueryStatus queryStatus = validateRequest(queryId, currentUser, adminOverride); - VoidResponse response = new VoidResponse(); - switch (queryStatus.getQueryState()) { - case CREATED: - // cancel the query - cancel(queryId, true); - response.addMessage(queryId + " canceled."); - break; - // TODO: Should we throw an exception for these cases? - case DEFINED: - case CLOSED: - case CANCELED: - case FAILED: - response.addMessage(queryId + " was not canceled because it is not running."); - break; - default: - throw new IllegalStateException("Unexpected query state: " + queryStatus.getQueryState()); + // if the query is running, or if the query is closing and finishing up a next call + if (queryStatus.getQueryState() == CREATED || (queryStatus.getQueryState() == CLOSED && queryStatus.getConcurrentNextCount() > 0)) { + cancel(queryId, true); + } else { + throw new BadRequestQueryException(queryId + " cannot be canceled because it is not running."); } + + VoidResponse response = new VoidResponse(); + response.addMessage(queryId + " canceled."); return response; } catch (QueryException e) { throw e; } catch (Exception e) { - QueryException queryException = new QueryException(DatawaveErrorCode.CANCELLATION_ERROR, e, "query_id: " + queryId); - log.error("Unable to cancel query with id " + queryId, queryException); + QueryException queryException = new QueryException(DatawaveErrorCode.CANCELLATION_ERROR, e, "Unknown error canceling query " + queryId); + log.error("Unknown error canceling query " + queryId, queryException); throw queryException; } } /** - * Once the cancel request is validated, this method is called to actually cancel the query. + * Cancels the specified query, and optionally publishes a cancel event to the query and executor services. + *

+ * Cancels any locally-running next calls.
+ * Called with {@param publishEvent} set to true when the user calls cancel.
+ * When {@param publishEvent} is true, changes the query state to {@link QueryStatus.QUERY_STATE#CANCELED}.
+ * Called with {@param publishEvent} set to false when handling a remote cancel event. * * @param queryId - * The id of the query to cancel + * the query id, not null * @param publishEvent - * If true, the cancel request will be published on the bus - * @throws InterruptedException + * whether or not to publish an event + * @throws NotFoundQueryException + * if the query cannot be found * @throws QueryException + * if query lock acquisition fails + * @throws InterruptedException + * if the cancel call is interrupted */ public void cancel(String queryId, boolean publishEvent) throws InterruptedException, QueryException { // if we have an active next call for this query locally, cancel it @@ -584,32 +806,82 @@ public void cancel(String queryId, boolean publishEvent) throws InterruptedExcep } /** - * Releases the resources associated with this query. Any currently running calls to 'next' on the query will continue until they finish. Calls to 'next' - * after a 'close' will start over at page 1. + * Closes the specified query. + *

+ * Close can only be called on a running query.
+ * Outstanding next calls will be allowed to run until they can return a full page, or they timeout.
+ * Aside from admins, only the query owner can close the specified query. * * @param queryId + * the query id, not null * @param currentUser - * @return + * the user who called this method, not null + * @return a void response indicating that the query was closed + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws BadRequestQueryException + * if the query is not running * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if the cancel call is interrupted + * @throws QueryException + * if there is an unknown error */ public VoidResponse close(String queryId, ProxiedUserDetails currentUser) throws QueryException { return close(queryId, currentUser, false); } /** - * Releases the resources associated with this query. Any currently running calls to 'next' on the query will continue until they finish. Calls to 'next' - * after a 'close' will start over at page 1. + * Closes the specified query using admin privileges. + *

+ * Close can only be called on a running query.
+ * Outstanding next calls will be allowed to run until they can return a full page, or they timeout.
+ * Only admin users should be allowed to call this method. * * @param queryId + * the query id, not null * @param currentUser - * @return + * the user who called this method, not null + * @return a void response indicating that the query was closed + * @throws NotFoundQueryException + * if the query cannot be found + * @throws BadRequestQueryException + * if the query is not running + * @throws QueryException + * if query lock acquisition fails * @throws QueryException + * if the cancel call is interrupted + * @throws QueryException + * if there is an unknown error */ public VoidResponse adminClose(String queryId, ProxiedUserDetails currentUser) throws QueryException { log.info("Close '" + queryId + "'called by admin: " + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); return close(queryId, currentUser, true); } + /** + * Closes all queries using admin privileges. + *

+ * Close can only be called on a running query.
+ * Queries that are not running will be ignored by this method.
+ * Outstanding next calls will be allowed to run until they can return a full page, or they timeout.
+ * Only admin users should be allowed to call this method. + * + * @param currentUser + * the user who called this method, not null + * @return a void response specifying which queries were closed + * @throws NotFoundQueryException + * if a query cannot be found + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if the cancel call is interrupted + * @throws QueryException + * if there is an unknown error + */ public VoidResponse adminCloseAll(ProxiedUserDetails currentUser) throws QueryException { log.info("Close All called by admin: " + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); try { @@ -618,58 +890,82 @@ public VoidResponse adminCloseAll(ProxiedUserDetails currentUser) throws QueryEx VoidResponse response = new VoidResponse(); for (QueryStatus queryStatus : queryStatuses) { - close(queryStatus.getQueryKey().getQueryId().toString()); - response.addMessage(queryStatus.getQueryKey().getQueryId().toString() + " closed."); + close(queryStatus.getQueryKey().getQueryId()); + response.addMessage(queryStatus.getQueryKey().getQueryId() + " closed."); } return response; } catch (QueryException e) { throw e; } catch (Exception e) { - QueryException queryException = new QueryException(DatawaveErrorCode.CANCELLATION_ERROR, e, "Error encountered while closing all queries."); + QueryException queryException = new QueryException(DatawaveErrorCode.QUERY_CLOSE_ERROR, e, "Error encountered while closing all queries."); log.error("Error encountered while closing all queries", queryException); throw queryException; } } + /** + * Closes the specified query. + *

+ * Close can only be called on a running query.
+ * Outstanding next calls will be allowed to run until they can return a full page, or they timeout.
+ * Query ownership will only be validated when {@param adminOverride} is set to true. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @param adminOverride + * whether or not this is an admin action + * @return a void response indicating that the query was closed + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws BadRequestQueryException + * if the query is not running + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if the cancel call is interrupted + * @throws QueryException + * if there is an unknown error + */ private VoidResponse close(String queryId, ProxiedUserDetails currentUser, boolean adminOverride) throws QueryException { try { // make sure the query is valid, and the user can act on it QueryStatus queryStatus = validateRequest(queryId, currentUser, adminOverride); - VoidResponse response = new VoidResponse(); - switch (queryStatus.getQueryState()) { - case CREATED: - // close the query - close(queryId); - response.addMessage(queryId + " closed."); - break; - // TODO: Should we throw an exception for these cases? - case DEFINED: - case CLOSED: - case CANCELED: - case FAILED: - response.addMessage(queryId + " was not closed because it is not running."); - break; - default: - throw new IllegalStateException("Unexpected query state: " + queryStatus.getQueryState()); + if (queryStatus.getQueryState() == CREATED) { + close(queryId); + } else { + throw new BadRequestQueryException(queryId + " cannot be closed because it is not running."); } + + VoidResponse response = new VoidResponse(); + response.addMessage(queryId + " closed."); return response; } catch (QueryException e) { throw e; } catch (Exception e) { - QueryException queryException = new QueryException(DatawaveErrorCode.CLOSE_ERROR, e, "query_id: " + queryId); - log.error("Unable to close query with id " + queryId, queryException); + QueryException queryException = new QueryException(DatawaveErrorCode.QUERY_CLOSE_ERROR, e, "Unknown error closing query " + queryId); + log.error("Unknown error closing query " + queryId, queryException); throw queryException; } } /** - * Once the close request is validated, this method is called to actually close the query. + * Closes the specified query, and publishes a close event to the executor services. + *

+ * Changes the query state to {@link QueryStatus.QUERY_STATE#CLOSED}. * * @param queryId - * The id of the query to close - * @throws InterruptedException + * the query id, not null + * @throws NotFoundQueryException + * if the query cannot be found * @throws QueryException + * if query lock acquisition fails + * @throws InterruptedException + * if the cancel call is interrupted */ public void close(String queryId) throws InterruptedException, QueryException { QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(queryId, status -> { @@ -682,12 +978,41 @@ public void close(String queryId) throws InterruptedException, QueryException { } /** - * Resets the query named by {@code queryId}. If the query is not alive, meaning that the current session has expired (due to either timeout, or server - * failure), then this will reload the query and start it over. If the query is alive, it closes it and starts the query over. + * Stops, and restarts the specified query. + *

+ * Reset can be called on any query, whether it's running or not.
+ * If the specified query is still running, it will be canceled. See {@link #cancel}.
+ * Reset creates a new, identical query, with a new query id.
+ * Reset queries will start running immediately.
+ * Auditing is performed before the new query is started. * * @param queryId - * @return + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @return a generic response containing the new query id + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if the cancel call is interrupted + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails * @throws QueryException + * if query storage fails + * @throws QueryException + * if there is an unknown error */ public GenericResponse reset(String queryId, ProxiedUserDetails currentUser) throws QueryException { try { @@ -696,7 +1021,7 @@ public GenericResponse reset(String queryId, ProxiedUserDetails currentU // cancel the query if it is running if (queryStatus.getQueryState() == CREATED) { - cancel(queryStatus.getQueryKey().getQueryId().toString(), true); + cancel(queryStatus.getQueryKey().getQueryId(), true); } // create a new query which is an exact copy of the specified query @@ -704,40 +1029,79 @@ public GenericResponse reset(String queryId, ProxiedUserDetails currentU GenericResponse response = new GenericResponse<>(); response.addMessage(queryId + " reset."); - response.setResult(taskKey.getQueryId().toString()); + response.setResult(taskKey.getQueryId()); response.setHasResults(true); return response; } catch (QueryException e) { throw e; } catch (Exception e) { log.error("Unknown error resetting query " + queryId, e); - throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Unknown error resetting query " + queryId); + throw new QueryException(DatawaveErrorCode.QUERY_RESET_ERROR, e, "Unknown error resetting query " + queryId); } } /** - * Remove (delete) the query + * Removes the specified query from query storage. + *

+ * Remove can only be called on a query that is not running.
+ * Aside from admins, only the query owner can remove the specified query. * * @param queryId + * the query id, not null * @param currentUser - * @return + * the user who called this method, not null + * @return a void response indicating that the query was removed + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws BadRequestQueryException + * if the query is running + * @throws QueryException + * if there is an unknown error */ public VoidResponse remove(String queryId, ProxiedUserDetails currentUser) throws QueryException { return remove(queryId, currentUser, false); } /** - * Remove (delete) the query + * Removes the specified query from query storage using admin privileges. + *

+ * Remove can only be called on a query that is not running.
+ * Only admin users should be allowed to call this method. * * @param queryId + * the query id, not null * @param currentUser - * @return + * the user who called this method, not null + * @return a void response indicating that the query was removed + * @throws NotFoundQueryException + * if the query cannot be found + * @throws BadRequestQueryException + * if the query is running + * @throws QueryException + * if there is an unknown error */ public VoidResponse adminRemove(String queryId, ProxiedUserDetails currentUser) throws QueryException { log.info("Remove '" + queryId + "'called by admin: " + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); return remove(queryId, currentUser, true); } + /** + * Removes all queries from query storage using admin privileges. + *

+ * Remove can only be called on a query that is not running.
+ * Queries that are running will be ignored by this method.
+ * Only admin users should be allowed to call this method. + * + * @param currentUser + * the user who called this method, not null + * @return a void response specifying which queries were removed + * @throws NotFoundQueryException + * if a query cannot be found + * @throws QueryException + * if there is an unknown error + */ public VoidResponse adminRemoveAll(ProxiedUserDetails currentUser) throws QueryException { log.info("Remove All called by admin: " + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); try { @@ -746,16 +1110,38 @@ public VoidResponse adminRemoveAll(ProxiedUserDetails currentUser) throws QueryE VoidResponse response = new VoidResponse(); for (QueryStatus queryStatus : queryStatuses) { - response.addMessage(queryStatus.getQueryKey().getQueryId().toString() + " removed."); + response.addMessage(queryStatus.getQueryKey().getQueryId() + " removed."); } return response; } catch (Exception e) { - QueryException queryException = new QueryException(DatawaveErrorCode.CANCELLATION_ERROR, e, "Error encountered while canceling all queries."); - log.error("Error encountered while canceling all queries", queryException); + QueryException queryException = new QueryException(DatawaveErrorCode.QUERY_REMOVAL_ERROR, e, "Error encountered while removing all queries."); + log.error("Error encountered while removing all queries", queryException); throw queryException; } } + /** + * Removes the specified query from query storage. + *

+ * Remove can only be called on a query that is not running.
+ * Query ownership will only be validated when {@param adminOverride} is set to true. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @param adminOverride + * whether or not this is an admin action + * @return a void response indicating that the query was removed + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws BadRequestQueryException + * if the query is running + * @throws QueryException + * if there is an unknown error + */ private VoidResponse remove(String queryId, ProxiedUserDetails currentUser, boolean adminOverride) throws QueryException { try { // make sure the query is valid, and the user can act on it @@ -764,10 +1150,10 @@ private VoidResponse remove(String queryId, ProxiedUserDetails currentUser, bool // remove the query if it is not running if (queryStatus.getQueryState() != CREATED) { if (!remove(queryStatus)) { - throw new QueryException("Failed to remove " + queryId, INTERNAL_SERVER_ERROR + "-1"); + throw new QueryException("Failed to remove " + queryId); } } else { - throw new QueryException("Cannot remove a running query.", BAD_REQUEST + "-1"); + throw new BadRequestQueryException("Cannot remove a running query."); } VoidResponse response = new VoidResponse(); @@ -777,7 +1163,7 @@ private VoidResponse remove(String queryId, ProxiedUserDetails currentUser, bool throw e; } catch (Exception e) { log.error("Unknown error removing query " + queryId, e); - throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Unknown error removing query " + queryId); + throw new QueryException(DatawaveErrorCode.QUERY_REMOVAL_ERROR, e, "Unknown error removing query " + queryId); } } @@ -785,6 +1171,51 @@ private boolean remove(QueryStatus queryStatus) throws IOException { return queryStorageCache.deleteQuery(queryStatus.getQueryKey().getQueryId()); } + /** + * Updates the specified query. + *

+ * Update can only be called on a defined, or running query.
+ * Auditing is not performed when updating a defined query.
+ * No auditable parameters should be updated when updating a running query.
+ * Any query parameter can be updated for a defined query.
+ * Query parameters which don't affect the scope of the query can be updated for a running query.
+ * The list of parameters that can be updated for a running query is configurable.
+ * Auditable parameters should never be added to the updatable parameters configuration.
+ * Query string, date range, query logic, and auths should never be updated for a running query.
+ * Only the query owner can call update on the specified query. + * + * @param queryId + * the query id, not null + * @param parameters + * the query parameter updates, not null + * @param currentUser + * the user who called this method, not null + * @return a generic response containing the query id + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if the query is not defined, or running + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if the update call is interrupted + * @throws BadRequestQueryException + * if no parameters are specified + * @throws QueryException + * if query storage fails + * @throws QueryException + * if there is an unknown error + */ public GenericResponse update(String queryId, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { try { // make sure the query is valid, and the user can act on it @@ -798,7 +1229,7 @@ public GenericResponse update(String queryId, MultiValueMap currentParams.add(x.getParameterName(), x.getParameterValue())); - boolean updated = false; + boolean updated; if (queryStatus.getQueryState() == DEFINED) { // update all parameters if the state is defined updated = updateParameters(parameters, currentParams); @@ -834,16 +1265,17 @@ public GenericResponse update(String queryId, MultiValueMap update(String queryId, MultiValueMap + * Duplicate can be called on any query, whether it's running or not.
+ * Duplicate creates a new, identical query, with a new query id.
+ * Provided parameter updates will be applied to the new query.
+ * Duplicated queries will start running immediately.
+ * Auditing is performed before the new query is started. + * + * @param queryId + * the query id, not null + * @param parameters + * the query parameter updates, not null + * @param currentUser + * the user who called this method, not null + * @return a generic response containing the new query id + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws QueryException + * if query lock acquisition fails + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws QueryException + * if there is an unknown error + */ public GenericResponse duplicate(String queryId, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { try { // make sure the query is valid, and the user can act on it @@ -865,25 +1334,58 @@ public GenericResponse duplicate(String queryId, MultiValueMap response = new GenericResponse<>(); response.addMessage(queryId + " duplicated."); - response.setResult(taskKey.getQueryId().toString()); + response.setResult(taskKey.getQueryId()); response.setHasResults(true); return response; } catch (QueryException e) { throw e; } catch (Exception e) { - log.error("Unknown error resetting query " + queryId, e); - throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Unknown error resetting query " + queryId); + log.error("Unknown error duplicating query " + queryId, e); + throw new QueryException(DatawaveErrorCode.QUERY_DUPLICATION_ERROR, e, "Unknown error duplicating query " + queryId); } } + /** + * Creates a copy of the specified query. + *

+ * Duplicate can be called on any query, whether it's running or not.
+ * Duplicate creates a new, identical query, with a new query id.
+ * Provided parameter updates will be applied to the new query.
+ * Duplicated queries will start running immediately.
+ * Auditing is performed before the new query is started. + * + * @param queryStatus + * the query status, not null + * @param parameters + * the query parameter updates, not null + * @param currentUser + * the user who called this method, not null + * @return a generic response containing the new query id + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws QueryException + * if query lock acquisition fails + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + */ private TaskKey duplicate(QueryStatus queryStatus, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { // TODO: Are we losing anything by recreating the parameters this way? // recreate the query parameters MultiValueMap currentParams = new LinkedMultiValueMap<>(); currentParams.addAll(queryStatus.getQuery().getOptionalQueryParameters()); - queryStatus.getQuery().getParameters().forEach(x -> { - currentParams.add(x.getParameterName(), x.getParameterValue()); - }); + queryStatus.getQuery().getParameters().forEach(x -> currentParams.add(x.getParameterName(), x.getParameterValue())); // updated all of the passed in parameters updateParameters(parameters, currentParams); @@ -892,27 +1394,25 @@ private TaskKey duplicate(QueryStatus queryStatus, MultiValueMap return storeQuery(queryStatus.getQuery().getQueryLogicName(), currentParams, currentUser, true); } - /** - * Updates the current params with the new params. - * - * @param newParameters - * @param currentParams - * @return true if current params was modified - */ - private boolean updateParameters(MultiValueMap newParameters, MultiValueMap currentParams) throws QueryException { + private boolean updateParameters(MultiValueMap newParameters, MultiValueMap currentParams) throws BadRequestQueryException { return updateParameters(newParameters.keySet(), newParameters, currentParams); } /** - * Updates the current params with the new params for the given parameter names. + * Updates the current parameters with the new parameters, limited to the specified parameter names. * * @param parameterNames + * the parameters names to override, not null * @param newParameters + * the new parameters, may be null * @param currentParams - * @return true if current params was modified + * the current parameters, not null + * @return true is the current parameters were changed + * @throws BadRequestQueryException + * if a parameter is passed without a value */ private boolean updateParameters(Collection parameterNames, MultiValueMap newParameters, MultiValueMap currentParams) - throws QueryException { + throws BadRequestQueryException { boolean paramsUpdated = false; for (String paramName : parameterNames) { if (newParameters.get(paramName) != null && !newParameters.get(paramName).isEmpty()) { @@ -922,24 +1422,68 @@ private boolean updateParameters(Collection parameterNames, MultiValueMa paramsUpdated = true; } } else { - throw new QueryException("Cannot update a query parameter without a value: " + paramName, BAD_REQUEST + "-1"); + throw new BadRequestQueryException("Cannot update a query parameter without a value: " + paramName); } } return paramsUpdated; } - // list all queries for the user matching the query name, if specified + /** + * Gets a list of queries for the calling user. + *

+ * Returns all matching queries owned by the calling user, filtering by query id and query name. + * + * @param queryId + * the query id, may be null + * @param queryName + * the query name, may be null + * @param currentUser + * the user who called this method, not null + * @return a list response containing the matching queries + * @throws QueryException + * if there is an unknown error + */ public QueryImplListResponse list(String queryId, String queryName, ProxiedUserDetails currentUser) throws QueryException { return list(queryId, queryName, ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); } - // list all queries + /** + * Gets a list of queries for the specified user using admin privileges. + *

+ * Returns all matching queries owned by any user, filtered by user ID, query ID, and query name.
+ * Only admin users should be allowed to call this method. + * + * @param queryId + * the query id, may be null + * @param queryName + * the query name, may be null + * @param userId + * the user whose queries we want to list, may be null + * @param currentUser + * the user who called this method, not null + * @return a list response containing the matching queries + * @throws QueryException + * if there is an unknown error + */ public QueryImplListResponse adminList(String queryId, String queryName, String userId, ProxiedUserDetails currentUser) throws QueryException { log.info("List '" + String.join(",", Arrays.asList(queryId, queryName, userId)) + "'called by admin: " + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); return list(queryId, queryName, userId); } + /** + * Gets a list of all matching queries, filtered by user ID, query ID, and query name. + * + * @param queryId + * the query id, may be null + * @param queryName + * the query name, may be null + * @param userId + * the user whose queries we want to list, may be null + * @return a list response containing the matching queries + * @throws QueryException + * if there is an unknown error + */ private QueryImplListResponse list(String queryId, String queryName, String userId) throws QueryException { try { List queries; @@ -968,7 +1512,23 @@ private QueryStatus validateRequest(String queryId, ProxiedUserDetails currentUs return validateRequest(queryId, currentUser, false); } - private QueryStatus validateRequest(String queryId, ProxiedUserDetails currentUser, boolean adminOverride) throws QueryException { + /** + * Validates the user request by ensuring that the query exists, and the user owns the query or is an admin. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @param adminOverride + * whether or not this is an admin request + * @return a query status object for the specified query + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + */ + private QueryStatus validateRequest(String queryId, ProxiedUserDetails currentUser, boolean adminOverride) + throws NotFoundQueryException, UnauthorizedQueryException { // does the query exist? QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); if (queryStatus == null) { @@ -988,6 +1548,15 @@ private QueryStatus validateRequest(String queryId, ProxiedUserDetails currentUs return queryStatus; } + /** + * Receives remote query requests and handles them. + *

+ * Cancels any running next calls when the request method is {@link QueryRequest.Method#CANCEL}.
+ * Takes no action for other remote query requests. + * + * @param queryRequest + * the remote query request, not null + */ @Override public void handleRemoteRequest(QueryRequest queryRequest) { try { @@ -998,11 +1567,30 @@ public void handleRemoteRequest(QueryRequest queryRequest) { log.debug("No handling specified for remote query request method: {}", queryRequest.getMethod()); } } catch (Exception e) { - log.error("Remote request failed:" + queryRequest); + log.error("Unknown error handling remote request:" + queryRequest); } } - protected void audit(Query query, QueryLogic queryLogic, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + /** + * Creates and submits an audit record to the audit service. + *

+ * The audit request submission will fail if the audit service is unable to validate the audit message. + * + * @param query + * the query to be audited, not null + * @param queryLogic + * the query logic, not null + * @param parameters + * the query parameters, not null + * @param currentUser + * the user who called this method, not null + * @throws BadRequestQueryException + * if the audit parameters fail validation + * @throws BadRequestQueryException + * if there is an error auditing the query + */ + protected void audit(Query query, QueryLogic queryLogic, MultiValueMap parameters, ProxiedUserDetails currentUser) + throws BadRequestQueryException { Auditor.AuditType auditType = queryLogic.getAuditType(query); parameters.add(PrivateAuditConstants.AUDIT_TYPE, auditType.name()); @@ -1044,14 +1632,36 @@ protected void audit(Query query, QueryLogic queryLogic, MultiValueMap + * If no pool is specified in the query parameters, the default pool will be used. + * + * @return the pool name for this query + */ protected String getPoolName() { return (queryParameters.getPool() != null) ? queryParameters.getPool() : queryProperties.getDefaultParams().getPool(); } + /** + * Gets the pool-specific executor service name. + * + * @param poolName + * the pool name, null returns *-null + * @return the pool-specific executor service name + */ protected String getPooledExecutorName(String poolName) { return String.join("-", Arrays.asList(queryProperties.getExecutorServiceName(), poolName)); } + /** + * Publishes a next event for the given query id and pool. + * + * @param queryId + * the query id, not null + * @param queryPool + * the pool to use, not null + */ public void publishNextEvent(String queryId, String queryPool) { publishExecutorEvent(QueryRequest.next(queryId), queryPool); } @@ -1078,6 +1688,16 @@ private void publishSelfEvent(QueryRequest queryRequest) { // @formatter:on } + /** + * Gets the maximum number of concurrent query tasks allowed for this logic. + *

+ * If the max concurrent tasks is overridden, that value will be used.
+ * Otherwise, the value is determined via the query logic, if defined, or via the configuration default. + * + * @param queryLogic + * the requested query logic, not null + * @return the maximum concurrent tasks limit + */ protected int getMaxConcurrentTasks(QueryLogic queryLogic) { // if there's an override, use it if (queryParameters.isMaxConcurrentTasksOverridden()) { @@ -1093,6 +1713,23 @@ else if (queryLogic.getMaxConcurrentTasks() > 0) { } } + /** + * Creates and initializes a query object using the provided query parameters. + *

+ * If a query id is not specified, then a random one will be generated. + * + * @param queryLogicName + * the requested query logic, not null + * @param parameters + * the query parameters, not null + * @param userDn + * the user dn, not null + * @param dnList + * the user dn list, not null + * @param queryId + * the desired query id, may be null + * @return an instantiated query object + */ protected Query createQuery(String queryLogicName, MultiValueMap parameters, String userDn, List dnList, String queryId) { Query q = responseObjectFactory.getQueryImpl(); q.initialize(userDn, dnList, queryLogicName, queryParameters, queryParameters.getUnknownParameters(parameters)); @@ -1106,10 +1743,28 @@ protected Query createQuery(String queryLogicName, MultiValueMap } /** - * This method will provide some initial query validation for the define and create query calls. + * Validates the query parameters, security markings, and instantiates the requested query logic. + *

+ * If the query is not valid, an exception will be thrown. + * + * @param queryLogicName + * the requested query logic, not null + * @param parameters + * the query parameters, not null + * @param currentUser + * the user who called this method, not null + * @return the query logic + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails */ protected QueryLogic validateQuery(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) - throws QueryException { + throws BadRequestQueryException, UnauthorizedQueryException { // validate the query parameters validateParameters(queryLogicName, parameters); @@ -1123,7 +1778,25 @@ protected QueryLogic validateQuery(String queryLogicName, MultiValueMap parameters) throws QueryException { + /** + * Performs query parameter validation. + *

+ * If the parameters are not valid, an exception will be thrown. + * + * @param queryLogicName + * the requested query logic, not null + * @param parameters + * the query parameters, not null + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if an invalid page size was requested + * @throws BadRequestQueryException + * if an invalid page timeout was requested + * @throws BadRequestQueryException + * if an invalid begin and end date was requested + */ + protected void validateParameters(String queryLogicName, MultiValueMap parameters) throws BadRequestQueryException { // add query logic name to parameters parameters.add(QUERY_LOGIC_NAME, queryLogicName); @@ -1165,7 +1838,22 @@ protected void validateParameters(String queryLogicName, MultiValueMap createQueryLogic(String queryLogicName, ProxiedUserDetails currentUser) throws QueryException { + /** + * Creates a query logic instance for the given user. + *

+ * The user's roles will be checked when instantiating the query logic. + * + * @param queryLogicName + * the requested query logic, not null + * @param currentUser + * the user who called this method, not null + * @return the requested query logic + * @throws BadRequestQueryException + * if the query logic does not exist + * @throws BadRequestQueryException + * if the user does not have the required roles for the query logic + */ + protected QueryLogic createQueryLogic(String queryLogicName, ProxiedUserDetails currentUser) throws BadRequestQueryException { // will throw IllegalArgumentException if not defined try { return queryLogicFactory.getQueryLogic(queryLogicName, currentUser.getPrimaryUser().getRoles()); @@ -1175,7 +1863,28 @@ protected QueryLogic createQueryLogic(String queryLogicName, ProxiedUserDetai } } - protected void validateQueryLogic(QueryLogic queryLogic, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + /** + * Performs query parameter validation using the query logic. + * + * @param queryLogic + * the requested query logic, not null + * @param parameters + * the query parameters, not null + * @param currentUser + * the user who called this method, not null + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws BadRequestQueryException + * if an invalid page size was requested + * @throws BadRequestQueryException + * if an invalid max results override was requested + * @throws BadRequestQueryException + * if an invalid max concurrent tasks override was requested + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + */ + protected void validateQueryLogic(QueryLogic queryLogic, MultiValueMap parameters, ProxiedUserDetails currentUser) + throws BadRequestQueryException, UnauthorizedQueryException { queryLogic.validate(parameters); // always check against the max @@ -1213,9 +1922,16 @@ protected void validateQueryLogic(QueryLogic queryLogic, MultiValueMap parameters) throws QueryException { + /** + * Performs security marking validation using the configured security markings. See {@link SecurityMarking#validate}. + * + * @param parameters + * the query parameters, not null + * @throws BadRequestQueryException + * if security marking validation fails + */ + protected void validateSecurityMarkings(MultiValueMap parameters) throws BadRequestQueryException { try { - securityMarking.clear(); securityMarking.validate(parameters); } catch (IllegalArgumentException e) { log.error("Failed security markings validation", e); @@ -1223,6 +1939,18 @@ protected void validateSecurityMarkings(MultiValueMap parameters) } } + /** + * Sets some audit parameters which are used internally to assist with auditing. + *

+ * If any of these parameters exist in the current parameter map, they will first be removed. + * + * @param queryLogicName + * the requested query logic, not null + * @param userDn + * the user dn, not null + * @param parameters + * the query parameters, not null + */ protected void setInternalAuditParameters(String queryLogicName, String userDn, MultiValueMap parameters) { // Set private audit-related parameters, stripping off any that the user might have passed in first. // These are parameters that aren't passed in by the user, but rather are computed from other sources. diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java index 7502effd..7aa13b47 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java @@ -1,6 +1,7 @@ package datawave.microservice.query.config; -import datawave.marking.MarkingFunctions; +import datawave.marking.ColumnVisibilitySecurityMarking; +import datawave.marking.SecurityMarking; import datawave.microservice.query.DefaultQueryParameters; import datawave.microservice.query.QueryParameters; import datawave.webservice.query.cache.QueryMetricFactory; @@ -28,8 +29,10 @@ public QueryParameters queryParameters() { @Bean @ConditionalOnMissingBean @RequestScope - public MarkingFunctions markingFunctions() { - return new MarkingFunctions.Default(); + public SecurityMarking securityMarking() { + SecurityMarking securityMarking = new ColumnVisibilitySecurityMarking(); + securityMarking.clear(); + return securityMarking; } @Bean diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 083bdd84..bd609d01 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -12,7 +12,6 @@ import datawave.webservice.query.cache.QueryMetricFactory; import datawave.webservice.query.cache.ResultsPage; import datawave.webservice.query.data.ObjectSizeOf; -import datawave.webservice.query.exception.QueryException; import datawave.webservice.query.metric.BaseQueryMetric; import datawave.webservice.query.metric.QueryMetric; import org.slf4j.Logger; @@ -265,6 +264,10 @@ private boolean isQueryStatusExpired() { return (System.currentTimeMillis() - lastStatusUpdateTime) > nextCallProperties.getStatusUpdateIntervalMillis(); } + public boolean isCanceled() { + return canceled; + } + public void cancel() { this.canceled = true; } @@ -337,7 +340,7 @@ public Builder setIdentifier(String identifier) { return this; } - public NextCall build() throws QueryException { + public NextCall build() { return new NextCall(this); } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java index ca01187b..c3bc7dc6 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java @@ -5,6 +5,7 @@ import datawave.microservice.query.storage.QueryStorageLock; import datawave.microservice.query.config.QueryProperties; import datawave.webservice.query.exception.DatawaveErrorCode; +import datawave.webservice.query.exception.NotFoundQueryException; import datawave.webservice.query.exception.QueryException; import java.util.UUID; @@ -62,7 +63,7 @@ public QueryStatus lockedUpdate(String queryUUID, StatusUpdater updater) throws updater.apply(queryStatus); queryStorageCache.updateQueryStatus(queryStatus); } else { - throw new QueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH, "Unable to find query status in cache."); + throw new NotFoundQueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH, "Unable to find query status in cache."); } } finally { statusLock.unlock(); From 626b3a943704e1c3c98f0837228ce6cf55113bce Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Mon, 14 Jun 2021 16:23:22 -0400 Subject: [PATCH 072/218] Adjusted dependencies to get embedded kafka tests working again. --- .../microservice/query/QueryManagementService.java | 10 +++++----- .../microservice/query/monitor/MonitorTask.java | 4 ++-- .../microservice/query/monitor/QueryMonitor.java | 2 +- .../datawave/microservice/query/runner/NextCall.java | 8 ++++---- .../query/status/QueryStatusUpdateHelper.java | 4 +--- .../web/filter/QueryMetricsEnrichmentFilterAdvice.java | 2 +- 6 files changed, 14 insertions(+), 16 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 251bbf73..8360d605 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -7,10 +7,6 @@ import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.authorization.util.AuthorizationsUtil; import datawave.microservice.common.audit.PrivateAuditConstants; -import datawave.microservice.query.storage.QueryQueueManager; -import datawave.microservice.query.storage.QueryStatus; -import datawave.microservice.query.storage.QueryStorageCache; -import datawave.microservice.query.storage.TaskKey; import datawave.microservice.query.config.QueryProperties; import datawave.microservice.query.logic.QueryLogic; import datawave.microservice.query.logic.QueryLogicFactory; @@ -18,6 +14,10 @@ import datawave.microservice.query.remote.QueryRequestHandler; import datawave.microservice.query.runner.NextCall; import datawave.microservice.query.status.QueryStatusUpdateHelper; +import datawave.microservice.query.storage.QueryQueueManager; +import datawave.microservice.query.storage.QueryStatus; +import datawave.microservice.query.storage.QueryStorageCache; +import datawave.microservice.query.storage.TaskKey; import datawave.microservice.query.util.QueryUtil; import datawave.security.util.ProxiedEntityUtils; import datawave.webservice.common.audit.AuditParameters; @@ -72,11 +72,11 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CANCELED; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CLOSED; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CREATED; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.DEFINED; -import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; @Service public class QueryManagementService implements QueryRequestHandler { diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java index ff2a1649..5d113fa4 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java @@ -1,12 +1,12 @@ package datawave.microservice.query.monitor; -import datawave.microservice.query.storage.QueryStatus; -import datawave.microservice.query.storage.QueryStorageCache; import datawave.microservice.query.QueryManagementService; import datawave.microservice.query.config.QueryExpirationProperties; import datawave.microservice.query.monitor.cache.MonitorStatus; import datawave.microservice.query.monitor.cache.MonitorStatusCache; import datawave.microservice.query.monitor.config.MonitorProperties; +import datawave.microservice.query.storage.QueryStatus; +import datawave.microservice.query.storage.QueryStorageCache; import datawave.webservice.query.exception.QueryException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java index fed1bbee..b969a364 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java @@ -1,11 +1,11 @@ package datawave.microservice.query.monitor; -import datawave.microservice.query.storage.QueryStorageCache; import datawave.microservice.query.QueryManagementService; import datawave.microservice.query.config.QueryExpirationProperties; import datawave.microservice.query.config.QueryProperties; import datawave.microservice.query.monitor.cache.MonitorStatusCache; import datawave.microservice.query.monitor.config.MonitorProperties; +import datawave.microservice.query.storage.QueryStorageCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index bd609d01..3f2072fe 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -1,14 +1,14 @@ package datawave.microservice.query.runner; +import datawave.microservice.query.config.NextCallProperties; +import datawave.microservice.query.config.QueryExpirationProperties; +import datawave.microservice.query.config.QueryProperties; +import datawave.microservice.query.logic.QueryLogic; import datawave.microservice.query.storage.QueryQueueListener; import datawave.microservice.query.storage.QueryQueueManager; import datawave.microservice.query.storage.QueryStatus; import datawave.microservice.query.storage.QueryStorageCache; import datawave.microservice.query.storage.Result; -import datawave.microservice.query.config.NextCallProperties; -import datawave.microservice.query.config.QueryExpirationProperties; -import datawave.microservice.query.config.QueryProperties; -import datawave.microservice.query.logic.QueryLogic; import datawave.webservice.query.cache.QueryMetricFactory; import datawave.webservice.query.cache.ResultsPage; import datawave.webservice.query.data.ObjectSizeOf; diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java index c3bc7dc6..570b7855 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java @@ -1,15 +1,13 @@ package datawave.microservice.query.status; +import datawave.microservice.query.config.QueryProperties; import datawave.microservice.query.storage.QueryStatus; import datawave.microservice.query.storage.QueryStorageCache; import datawave.microservice.query.storage.QueryStorageLock; -import datawave.microservice.query.config.QueryProperties; import datawave.webservice.query.exception.DatawaveErrorCode; import datawave.webservice.query.exception.NotFoundQueryException; import datawave.webservice.query.exception.QueryException; -import java.util.UUID; - import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CREATED; public class QueryStatusUpdateHelper { diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java index f18b0d64..73f3bbfe 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java @@ -1,8 +1,8 @@ package datawave.microservice.query.web.filter; +import datawave.microservice.query.logic.QueryLogicFactory; import datawave.microservice.query.storage.QueryState; import datawave.microservice.query.storage.QueryStorageCache; -import datawave.microservice.query.logic.QueryLogicFactory; import datawave.microservice.query.web.QueryMetrics; import datawave.microservice.query.web.annotation.EnrichQueryMetrics; import datawave.webservice.query.metric.BaseQueryMetric; From 155e089ab96c21cde9ccb880b7e74bc254955864 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 17 Jun 2021 11:03:02 -0400 Subject: [PATCH 073/218] Added query service tests for define and create --- .../query/DefaultQueryParameters.java | 4 +- .../query/QueryManagementService.java | 52 +- .../microservice/query/QueryServiceTest.java | 1006 ++++++++++++++++- .../resources/MyTestQueryLogicFactory.xml | 4 + .../src/test/resources/config/application.yml | 9 +- 5 files changed, 1008 insertions(+), 67 deletions(-) diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java index d0b9b71e..0476b011 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java @@ -19,7 +19,7 @@ public class DefaultQueryParameters implements QueryParameters { private static final List KNOWN_PARAMS = Arrays.asList(QUERY_STRING, QUERY_NAME, QUERY_PERSISTENCE, QUERY_PAGESIZE, QUERY_PAGETIMEOUT, QUERY_AUTHORIZATIONS, QUERY_EXPIRATION, QUERY_TRACE, QUERY_BEGIN, QUERY_END, QUERY_VISIBILITY, QUERY_LOGIC_NAME, QUERY_POOL, - QUERY_MAX_RESULTS_OVERRIDE); + QUERY_MAX_RESULTS_OVERRIDE, QUERY_MAX_CONCURRENT_TASKS); protected String query; protected String queryName; @@ -523,7 +523,7 @@ public void setMaxConcurrentTasks(int maxConcurrentTasks) { @Override public boolean isMaxConcurrentTasksOverridden() { - return false; + return isMaxConcurrentTasksOverridden; } @Override diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 8360d605..f84fc78c 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -43,6 +43,8 @@ import datawave.webservice.result.QueryImplListResponse; import datawave.webservice.result.QueryLogicResponse; import datawave.webservice.result.VoidResponse; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.cloud.bus.BusProperties; @@ -347,7 +349,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p * if there is an unknown error */ private TaskKey storeQuery(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser, boolean isCreateRequest, - String queryId) throws QueryException { + String queryId) throws BadRequestQueryException, QueryException { // validate query and get a query logic QueryLogic queryLogic = validateQuery(queryLogicName, parameters, currentUser); @@ -360,6 +362,15 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p Query query = createQuery(queryLogicName, parameters, userDn, currentUser.getDNs(), queryId); + // TODO: Downgrade the auths before or after auditing??? + // downgrade the auths + Set downgradedAuthorizations; + try { + downgradedAuthorizations = AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), currentUser); + } catch (Exception e) { + throw new BadRequestQueryException("Unable to downgrade authorizations", e, HttpStatus.SC_BAD_REQUEST + "-1"); + } + // if this is a create request, send an audit record to the auditor if (isCreateRequest) { audit(query, queryLogic, parameters, currentUser); @@ -373,7 +384,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p taskKey = queryStorageCache.createQuery( getPoolName(), query, - AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), currentUser), + downgradedAuthorizations, getMaxConcurrentTasks(queryLogic)); // @formatter:on @@ -384,7 +395,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p taskKey = queryStorageCache.defineQuery( getPoolName(), query, - AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), currentUser), + downgradedAuthorizations, getMaxConcurrentTasks(queryLogic)); // @formatter:on } @@ -497,7 +508,7 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th if (queryStatus.getQueryState() == CREATED) { return next(queryId, currentUser.getPrimaryUser().getRoles()); } else { - throw new BadRequestQueryException("Cannot call next on a query that is not running"); + throw new BadRequestQueryException("Cannot call next on a query that is not running", HttpStatus.SC_BAD_REQUEST + "-1"); } } catch (QueryException e) { throw e; @@ -747,7 +758,7 @@ private VoidResponse cancel(String queryId, ProxiedUserDetails currentUser, bool if (queryStatus.getQueryState() == CREATED || (queryStatus.getQueryState() == CLOSED && queryStatus.getConcurrentNextCount() > 0)) { cancel(queryId, true); } else { - throw new BadRequestQueryException(queryId + " cannot be canceled because it is not running."); + throw new BadRequestQueryException(queryId + " cannot be canceled because it is not running.", HttpStatus.SC_BAD_REQUEST + "-1"); } VoidResponse response = new VoidResponse(); @@ -938,7 +949,7 @@ private VoidResponse close(String queryId, ProxiedUserDetails currentUser, boole if (queryStatus.getQueryState() == CREATED) { close(queryId); } else { - throw new BadRequestQueryException(queryId + " cannot be closed because it is not running."); + throw new BadRequestQueryException(queryId + " cannot be closed because it is not running.", HttpStatus.SC_BAD_REQUEST + "-1"); } VoidResponse response = new VoidResponse(); @@ -1153,7 +1164,7 @@ private VoidResponse remove(String queryId, ProxiedUserDetails currentUser, bool throw new QueryException("Failed to remove " + queryId); } } else { - throw new BadRequestQueryException("Cannot remove a running query."); + throw new BadRequestQueryException("Cannot remove a running query.", HttpStatus.SC_BAD_REQUEST + "-1"); } VoidResponse response = new VoidResponse(); @@ -1265,7 +1276,7 @@ public GenericResponse update(String queryId, MultiValueMap update(String queryId, MultiValueMap parameterNames, MultiValueMa paramsUpdated = true; } } else { - throw new BadRequestQueryException("Cannot update a query parameter without a value: " + paramName); + throw new BadRequestQueryException("Cannot update a query parameter without a value: " + paramName, HttpStatus.SC_BAD_REQUEST + "-1"); } } return paramsUpdated; @@ -1611,7 +1622,6 @@ protected void audit(Query query, QueryLogic queryLogic, MultiValueMap createQueryLogic(String queryLogicName, ProxiedUserDetai */ protected void validateQueryLogic(QueryLogic queryLogic, MultiValueMap parameters, ProxiedUserDetails currentUser) throws BadRequestQueryException, UnauthorizedQueryException { - queryLogic.validate(parameters); + try { + queryLogic.validate(parameters); + } catch (IllegalArgumentException e) { + log.error("Unable to validate query parameters with query logic", e); + throw new BadRequestQueryException("Unable to validate query parameters with query logic.", e, HttpStatus.SC_BAD_REQUEST + "-1"); + } // always check against the max if (queryLogic.getMaxPageSize() > 0 && queryParameters.getPagesize() > queryLogic.getMaxPageSize()) { @@ -1900,7 +1920,8 @@ protected void validateQueryLogic(QueryLogic queryLogic, MultiValueMap= 0) { if (queryParameters.getMaxResultsOverride() < 0 || (queryLogic.getMaxResults() < queryParameters.getMaxResultsOverride())) { log.error("Invalid max results override: " + queryParameters.getMaxResultsOverride() + " vs " + queryLogic.getMaxResults()); - throw new BadRequestQueryException(DatawaveErrorCode.INVALID_MAX_RESULTS_OVERRIDE); + throw new BadRequestQueryException(DatawaveErrorCode.INVALID_MAX_RESULTS_OVERRIDE, + MessageFormat.format("Max = {0}.", queryLogic.getMaxResults())); } } @@ -1910,7 +1931,8 @@ protected void validateQueryLogic(QueryLogic queryLogic, MultiValueMap auditIds; + private MockRestServiceServer mockServer; + @Before public void setup() { + auditIds = new ArrayList<>(); + jwtRestTemplate = restTemplateBuilder.build(JWTRestTemplate.class); + jwtRestTemplate.setErrorHandler(new NoOpResponseErrorHandler()); DN = SubjectIssuerDNPair.of(userDN, "issuerDn"); + + RestTemplate auditorRestTemplate = (RestTemplate) new DirectFieldAccessor(auditClient).getPropertyValue("jwtRestTemplate"); + mockServer = MockRestServiceServer.createServer(auditorRestTemplate); } + @DirtiesContext @Test - public void testDefineQuery() { - Collection roles = Collections.singleton("AuthorizedUser"); - Collection auths = Collections.singleton("ALL"); - DatawaveUser uathDWUser = new DatawaveUser(DN, USER, auths, roles, null, System.currentTimeMillis()); - ProxiedUserDetails authUser = new ProxiedUserDetails(Collections.singleton(uathDWUser), uathDWUser.getCreationTime()); + public void testDefineSuccess() throws ParseException, IOException { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); - UriComponents uri = UriComponentsBuilder.newInstance().scheme("https").host("localhost").port(webServicePort).path("/query/v1/EventQuery/define") - .build(); + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - MultiValueMap map = new LinkedMultiValueMap<>(); - map.set(DefaultQueryParameters.QUERY_STRING, "FIELD:SOME_VALUE"); - map.set(DefaultQueryParameters.QUERY_NAME, "The Greatest Query in the World - Tribute"); - map.set(DefaultQueryParameters.QUERY_AUTHORIZATIONS, "ALL"); - map.set(DefaultQueryParameters.QUERY_BEGIN, "20000101 000000.000"); - map.set(DefaultQueryParameters.QUERY_END, "20500101 000000.000"); - map.set(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, "ALL"); - - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - try { - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, String.class); - - System.out.println("done!"); - } finally { - assertTrue("", true); - } + // setup a mock audit service + auditNotSentSetup(); + + long currentTimeMillis = System.currentTimeMillis(); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + false, + HttpStatus.Series.SUCCESSFUL, + resp); + // @formatter:on + + // verify that a query id was returned + String queryId = genericResponse.getResult(); + Assert.assertNotNull(queryId); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.DEFINED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that they query was created correctly + Query query = queryStatus.getQuery(); + // @formatter:off + assertQuery( + TEST_QUERY_STRING, + TEST_QUERY_NAME, + TEST_QUERY_AUTHORIZATIONS, + TEST_QUERY_BEGIN, + TEST_QUERY_END, + TEST_VISIBILITY_MARKING, + query); + // @formatter:on + + // verify that no audit message was sent + assertAuditNotSent(); + + // verify that query tasks weren't created + assertTasksNotCreated(queryId); + } + + @Test + public void testDefineFailure_paramValidation() { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + // remove the query param to induce a parameter validation failure + map.remove(QUERY_STRING); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertNull(genericResponse.getResult()); + + // verify that an exception was returned + Assert.assertEquals(1, genericResponse.getExceptions().size()); + + QueryExceptionType queryException = genericResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Missing one or more required QueryParameters", + "java.lang.IllegalArgumentException: Missing one or more required QueryParameters", + "400-1", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); } @Test - public void testCreateQuery() { - Collection roles = Collections.singleton("AuthorizedUser"); - Collection auths = Collections.singleton("ALL"); - DatawaveUser uathDWUser = new DatawaveUser(DN, USER, auths, roles, null, System.currentTimeMillis()); - ProxiedUserDetails authUser = new ProxiedUserDetails(Collections.singleton(uathDWUser), uathDWUser.getCreationTime()); + public void testDefineFailure_authValidation() { + ProxiedUserDetails authUser = createUserDetails(Collections.singleton("AuthorizedUser"), Collections.emptyList()); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertNull(genericResponse.getResult()); + + // verify that an exception was returned + Assert.assertEquals(1, genericResponse.getExceptions().size()); + + QueryExceptionType queryException = genericResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "User requested authorizations that they don't have. Missing: [ALL], Requested: [ALL], User: []", + "java.lang.IllegalArgumentException: User requested authorizations that they don't have. Missing: [ALL], Requested: [ALL], User: []", + "400-1", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @Test + public void testDefineFailure_queryLogicValidation() { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + // remove the beginDate param to induce a query logic validation failure + map.remove(BEGIN_DATE); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertNull(genericResponse.getResult()); + + // verify that an exception was returned + Assert.assertEquals(1, genericResponse.getExceptions().size()); + + QueryExceptionType queryException = genericResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Required parameter begin not found", + "java.lang.IllegalArgumentException: Required parameter begin not found", + "400-1", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @Test + public void testDefineFailure_maxPageSize() { + ProxiedUserDetails authUser = createUserDetails(Arrays.asList("AuthorizedUser", queryProperties.getPrivilegedRole()), null); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + // set an invalid page size override + map.set(QUERY_PAGESIZE, Integer.toString(Integer.MAX_VALUE)); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertNull(genericResponse.getResult()); + + // verify that an exception was returned + Assert.assertEquals(1, genericResponse.getExceptions().size()); + + QueryExceptionType queryException = genericResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Page size is larger than configured max. Max = 10,000.", + "Exception with no cause caught", + "400-6", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @Test + public void testDefineFailure_maxResultsOverride() { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + // set an invalid max results override + map.set(QUERY_MAX_RESULTS_OVERRIDE, Long.toString(Long.MAX_VALUE)); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertNull(genericResponse.getResult()); + + // verify that an exception was returned + Assert.assertEquals(1, genericResponse.getExceptions().size()); + + QueryExceptionType queryException = genericResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Invalid max results override value. Max = 1,000,000.", + "Exception with no cause caught", + "400-43", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @Test + public void testDefineFailure_maxConcurrentTasksOverride() { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + // add an invalid max results override + map.set(QUERY_MAX_CONCURRENT_TASKS, Integer.toString(Integer.MAX_VALUE)); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertNull(genericResponse.getResult()); + + // verify that an exception was returned + Assert.assertEquals(1, genericResponse.getExceptions().size()); + + QueryExceptionType queryException = genericResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Invalid max concurrent tasks override value. Max = 10.", + "Exception with no cause caught", + "400-44", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @Test + public void testDefineFailure_roleValidation() { + // create a user without the required role + ProxiedUserDetails authUser = createUserDetails(Collections.emptyList(), Collections.singletonList("ALL")); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertNull(genericResponse.getResult()); + + // verify that an exception was returned + Assert.assertEquals(1, genericResponse.getExceptions().size()); + + QueryExceptionType queryException = genericResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "User does not have the required roles.", + "datawave.webservice.query.exception.UnauthorizedQueryException: User does not have the required roles.", + "400-5", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @Test + public void testDefineFailure_markingValidation() { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + // remove the column visibility param to induce a security marking validation failure + map.remove(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertNull(genericResponse.getResult()); + + // verify that an exception was returned + Assert.assertEquals(1, genericResponse.getExceptions().size()); + + QueryExceptionType queryException = genericResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Required parameter columnVisibility not found", + "java.lang.IllegalArgumentException: Required parameter columnVisibility not found", + "400-4", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @DirtiesContext + @Test + public void testCreateSuccess() throws ParseException, IOException { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditSentSetup(); + + long currentTimeMillis = System.currentTimeMillis(); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + true, + HttpStatus.Series.SUCCESSFUL, + resp); + // @formatter:on + + // verify that a query id was returned + String queryId = genericResponse.getResult(); + Assert.assertNotNull(queryId); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that they query was created correctly + Query query = queryStatus.getQuery(); + + // @formatter:off + assertQuery( + TEST_QUERY_STRING, + TEST_QUERY_NAME, + TEST_QUERY_AUTHORIZATIONS, + TEST_QUERY_BEGIN, + TEST_QUERY_END, + TEST_VISIBILITY_MARKING, + query); + // @formatter:on + + // verify that an audit message was sent and the the audit id matches the query id + assertAuditSent(queryId); + + // verify that query tasks were created + assertTasksCreated(queryId); + + // TODO: Things to test + // successful query definition + // failed parameter validation + // query logic parameter validation + // missing required roles + // failed security marking validation + // failed query storage + // unknown error + } + + @Test + public void testCreateFailure_paramValidation() { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + // remove the query param to induce a parameter validation failure + map.remove(QUERY_STRING); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertNull(genericResponse.getResult()); + + // verify that an exception was returned + Assert.assertEquals(1, genericResponse.getExceptions().size()); - UriComponents uri = UriComponentsBuilder.newInstance().scheme("https").host("localhost").port(webServicePort).path("/query/v1/EventQuery/create") - .build(); + QueryExceptionType queryException = genericResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Missing one or more required QueryParameters", + "java.lang.IllegalArgumentException: Missing one or more required QueryParameters", + "400-1", + queryException); + // @formatter:on + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + @Test + public void testCreateFailure_authValidation() { + ProxiedUserDetails authUser = createUserDetails(Collections.singleton("AuthorizedUser"), Collections.emptyList()); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertNull(genericResponse.getResult()); + + // verify that an exception was returned + Assert.assertEquals(1, genericResponse.getExceptions().size()); + + QueryExceptionType queryException = genericResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "User requested authorizations that they don't have. Missing: [ALL], Requested: [ALL], User: []", + "java.lang.IllegalArgumentException: User requested authorizations that they don't have. Missing: [ALL], Requested: [ALL], User: []", + "400-1", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + @Test + public void testCreateFailure_queryLogicValidation() { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + // remove the beginDate param to induce a query logic validation failure + map.remove(BEGIN_DATE); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertNull(genericResponse.getResult()); + + // verify that an exception was returned + Assert.assertEquals(1, genericResponse.getExceptions().size()); + + QueryExceptionType queryException = genericResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Required parameter begin not found", + "java.lang.IllegalArgumentException: Required parameter begin not found", + "400-1", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + @Test + public void testCreateFailure_maxPageSize() { + ProxiedUserDetails authUser = createUserDetails(Arrays.asList("AuthorizedUser", queryProperties.getPrivilegedRole()), null); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + // set an invalid page size override + map.set(QUERY_PAGESIZE, Integer.toString(Integer.MAX_VALUE)); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertNull(genericResponse.getResult()); + + // verify that an exception was returned + Assert.assertEquals(1, genericResponse.getExceptions().size()); + + QueryExceptionType queryException = genericResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Page size is larger than configured max. Max = 10,000.", + "Exception with no cause caught", + "400-6", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + @Test + public void testCreateFailure_maxResultsOverride() { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + // set an invalid max results override + map.set(QUERY_MAX_RESULTS_OVERRIDE, Long.toString(Long.MAX_VALUE)); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertNull(genericResponse.getResult()); + + // verify that an exception was returned + Assert.assertEquals(1, genericResponse.getExceptions().size()); + + QueryExceptionType queryException = genericResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Invalid max results override value. Max = 1,000,000.", + "Exception with no cause caught", + "400-43", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + @Test + public void testCreateFailure_maxConcurrentTasksOverride() { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + // add an invalid max results override + map.set(QUERY_MAX_CONCURRENT_TASKS, Integer.toString(Integer.MAX_VALUE)); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertNull(genericResponse.getResult()); + + // verify that an exception was returned + Assert.assertEquals(1, genericResponse.getExceptions().size()); + + QueryExceptionType queryException = genericResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Invalid max concurrent tasks override value. Max = 10.", + "Exception with no cause caught", + "400-44", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + @Test + public void testCreateFailure_roleValidation() { + // create a user without the required role + ProxiedUserDetails authUser = createUserDetails(Collections.emptyList(), Collections.singletonList("ALL")); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertNull(genericResponse.getResult()); + + // verify that an exception was returned + Assert.assertEquals(1, genericResponse.getExceptions().size()); + + QueryExceptionType queryException = genericResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "User does not have the required roles.", + "datawave.webservice.query.exception.UnauthorizedQueryException: User does not have the required roles.", + "400-5", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + @Test + public void testCreateFailure_markingValidation() { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + // remove the column visibility param to induce a security marking validation failure + map.remove(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertNull(genericResponse.getResult()); + + // verify that an exception was returned + Assert.assertEquals(1, genericResponse.getExceptions().size()); + + QueryExceptionType queryException = genericResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Required parameter columnVisibility not found", + "java.lang.IllegalArgumentException: Required parameter columnVisibility not found", + "400-4", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + private ProxiedUserDetails createUserDetails() { + return createUserDetails(null, null); + } + + private ProxiedUserDetails createUserDetails(Collection roles, Collection auths) { + Collection userRoles = roles != null ? roles : Collections.singleton("AuthorizedUser"); + Collection userAuths = auths != null ? auths : Collections.singleton("ALL"); + DatawaveUser datawaveUser = new DatawaveUser(DN, USER, userAuths, userRoles, null, System.currentTimeMillis()); + return new ProxiedUserDetails(Collections.singleton(datawaveUser), datawaveUser.getCreationTime()); + } + + private UriComponents createUri(String path) { + return UriComponentsBuilder.newInstance().scheme("https").host("localhost").port(webServicePort).path("/query/v1/" + path).build(); + } + + private MultiValueMap createParams() { MultiValueMap map = new LinkedMultiValueMap<>(); - map.set(DefaultQueryParameters.QUERY_STRING, "FIELD:SOME_VALUE"); - map.set(DefaultQueryParameters.QUERY_NAME, "The Greatest Query in the World - Tribute"); - map.set(DefaultQueryParameters.QUERY_AUTHORIZATIONS, "ALL"); - map.set(DefaultQueryParameters.QUERY_BEGIN, "20000101 000000.000"); - map.set(DefaultQueryParameters.QUERY_END, "20500101 000000.000"); - map.set(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, "ALL"); - - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - try { - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, String.class); - - System.out.println("done!"); - } finally { - assertTrue("", true); + map.set(DefaultQueryParameters.QUERY_STRING, TEST_QUERY_STRING); + map.set(DefaultQueryParameters.QUERY_NAME, TEST_QUERY_NAME); + map.set(DefaultQueryParameters.QUERY_AUTHORIZATIONS, TEST_QUERY_AUTHORIZATIONS); + map.set(DefaultQueryParameters.QUERY_BEGIN, TEST_QUERY_BEGIN); + map.set(DefaultQueryParameters.QUERY_END, TEST_QUERY_END); + map.set(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, TEST_VISIBILITY_MARKING); + map.set(QUERY_MAX_CONCURRENT_TASKS, Integer.toString(1)); + map.set(QUERY_MAX_RESULTS_OVERRIDE, Long.toString(1234)); + map.set(QUERY_PAGESIZE, Long.toString(123)); + return map; + } + + private void assertQueryStatus(QueryStatus.QUERY_STATE queryState, long numResultsReturned, long numResultsGenerated, long concurrentNextCount, + long lastPageNumber, long lastCallTimeMillis, QueryStatus queryStatus) { + Assert.assertEquals(queryState, queryStatus.getQueryState()); + Assert.assertEquals(numResultsReturned, queryStatus.getNumResultsReturned()); + Assert.assertEquals(numResultsGenerated, queryStatus.getNumResultsGenerated()); + Assert.assertEquals(concurrentNextCount, queryStatus.getConcurrentNextCount()); + Assert.assertEquals(lastPageNumber, queryStatus.getLastPageNumber()); + Assert.assertTrue(queryStatus.getLastUsedMillis() > lastCallTimeMillis); + Assert.assertTrue(queryStatus.getLastUpdatedMillis() > lastCallTimeMillis); + } + + private void assertQuery(String queryString, String queryName, String authorizations, String begin, String end, String visibility, Query query) + throws ParseException { + SimpleDateFormat sdf = new SimpleDateFormat(DefaultQueryParameters.formatPattern); + Assert.assertEquals(queryString, query.getQuery()); + Assert.assertEquals(queryName, query.getQueryName()); + Assert.assertEquals(authorizations, query.getQueryAuthorizations()); + Assert.assertEquals(sdf.parse(begin), query.getBeginDate()); + Assert.assertEquals(sdf.parse(end), query.getEndDate()); + Assert.assertEquals(visibility, query.getColumnVisibility()); + } + + private void assertTasksCreated(String queryId) throws IOException { + // verify that the query task states were created + TaskStates taskStates = queryStorageCache.getTaskStates(queryId); + Assert.assertNotNull(taskStates); + + // verify that a query task was created + List taskKeys = queryStorageCache.getTasks(queryId); + Assert.assertFalse(taskKeys.isEmpty()); + } + + private void assertTasksNotCreated(String queryId) throws IOException { + // verify that the query task states were not created + TaskStates taskStates = queryStorageCache.getTaskStates(queryId); + Assert.assertNull(taskStates); + + // verify that a query task was not created + List taskKeys = queryStorageCache.getTasks(queryId); + Assert.assertTrue(taskKeys.isEmpty()); + } + + public RequestMatcher auditIdGrabber() { + return request -> { + List params = URLEncodedUtils.parse(request.getBody().toString(), Charset.defaultCharset()); + params.stream().filter(p -> p.getName().equals(AUDIT_ID)).forEach(p -> auditIds.add(p.getValue())); + }; + } + + private void auditSentSetup() { + mockServer.expect(requestTo(EXPECTED_AUDIT_URI)).andExpect(auditIdGrabber()).andRespond(withSuccess()); + } + + private void auditNotSentSetup() { + mockServer.expect(never(), requestTo(EXPECTED_AUDIT_URI)).andExpect(auditIdGrabber()).andRespond(withSuccess()); + } + + private void assertAuditSent(String queryId) { + mockServer.verify(); + Assert.assertEquals(1, auditIds.size()); + Assert.assertEquals(queryId, auditIds.get(0)); + } + + private void assertAuditNotSent() { + mockServer.verify(); + Assert.assertEquals(0, auditIds.size()); + } + + private void assertQueryException(String message, String cause, String code, QueryExceptionType queryException) { + Assert.assertEquals(message, queryException.getMessage()); + Assert.assertEquals(cause, queryException.getCause()); + Assert.assertEquals(code, queryException.getCode()); + } + + @SuppressWarnings("unchecked") + private GenericResponse assertGenericResponse(boolean hasResults, HttpStatus.Series series, ResponseEntity response) { + Assert.assertEquals(series, response.getStatusCode().series()); + Assert.assertNotNull(response); + GenericResponse genericResponse = (GenericResponse) response.getBody(); + Assert.assertNotNull(genericResponse); + Assert.assertEquals(hasResults, genericResponse.getHasResults()); + return genericResponse; + } + + private static class NoOpResponseErrorHandler extends DefaultResponseErrorHandler { + @Override + public void handleError(ClientHttpResponse response) throws IOException { + // do nothing } } diff --git a/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml b/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml index bb02a338..1c644274 100644 --- a/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml +++ b/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml @@ -147,6 +147,10 @@ + + + + diff --git a/query-microservices/query-service/service/src/test/resources/config/application.yml b/query-microservices/query-service/service/src/test/resources/config/application.yml index 60b828df..4800ec60 100644 --- a/query-microservices/query-service/service/src/test/resources/config/application.yml +++ b/query-microservices/query-service/service/src/test/resources/config/application.yml @@ -40,7 +40,7 @@ management: logging: level: - datawave.microservice: DEBUG + datawave.microservice: FATAL warehouse: accumulo: @@ -142,7 +142,7 @@ datawave: metadataTableName: ${warehouse.tables.metadata.name} indexTableName: ${warehouse.tables.index.name} reverseIndexTableName: ${warehouse.tables.reverseIndex.name} - maxResults: -1 + maxResults: 1000000 queryThreads: ${warehouse.defaults.queryThreads} indexLookupThreads: ${warehouse.defaults.indexLookupThreads} dateIndexThreads: ${warehouse.defaults.dateIndexThreads} @@ -170,7 +170,7 @@ datawave: - 'foo.bar' filterOptions: 'bar': "foo" - auditType: "NONE" + auditType: "ACTIVE" logicDescription: "Retrieve sharded events/documents, leveraging the global index tables as needed" eventPerDayThreshold: ${warehouse.defaults.eventPerDayThreshold} shardsPerDayThreshold: ${warehouse.defaults.shardsPerDayThreshold} @@ -219,6 +219,9 @@ datawave: statsdHost: ${warehouse.statsd.host} statsdPort: ${warehouse.statsd.port} evaluationOnlyFields: "" + maxConcurrentTasks: 10 + requiredRoles: + - "AuthorizedUser" audit-client: discovery: From 54e956fa150e6583d5e4d7ce0c63c43d18733e82 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 17 Jun 2021 11:55:04 -0400 Subject: [PATCH 074/218] Updated next call logic to be able to determine when all results have been retrieved for a query. --- .../microservice/query/runner/NextCall.java | 85 ++++++++++++++----- 1 file changed, 63 insertions(+), 22 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 3f2072fe..1dcb4bf7 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -9,6 +9,7 @@ import datawave.microservice.query.storage.QueryStatus; import datawave.microservice.query.storage.QueryStorageCache; import datawave.microservice.query.storage.Result; +import datawave.microservice.query.storage.TaskStates; import datawave.webservice.query.cache.QueryMetricFactory; import datawave.webservice.query.cache.ResultsPage; import datawave.webservice.query.data.ObjectSizeOf; @@ -54,8 +55,12 @@ public class NextCall implements Callable> { private long startTimeMillis; private ResultsPage.Status status = ResultsPage.Status.COMPLETE; - private long lastStatusUpdateTime = 0L; + private long lastQueryStatusUpdateTime = 0L; private QueryStatus queryStatus; + private long lastTaskStatesUpdateTime = 0L; + private TaskStates taskStates; + private boolean tasksFinished = false; + private long prevResultCount = 0L; private final BaseQueryMetric metric; @@ -108,9 +113,6 @@ public ResultsPage call() { while (!isFinished(queryId)) { Message message = resultListener.receive(nextCallProperties.getResultPollIntervalMillis()); if (message != null) { - - // TODO: In the past, if we got a null result we would mark the next call as finished - // Should we continue to do that, or something else? Object result = message.getPayload().getPayload(); if (result != null) { results.add(result); @@ -132,23 +134,25 @@ private boolean isFinished(String queryId) { boolean finished = false; long callTimeMillis = System.currentTimeMillis() - startTimeMillis; - // 1) was this query canceled? - if (canceled) { - log.info("Query [{}]: query cancelled, aborting next call", queryId); + // 1) have we hit the user's results-per-page limit? + if (results.size() >= userResultsPerPage) { + log.info("Query [{}]: user requested max page size has been reached, aborting next call", queryId); finished = true; } - // 2) have we hit the user's results-per-page limit? - if (!finished && results.size() >= userResultsPerPage) { - log.info("Query [{}]: user requested max page size has been reached, aborting next call", queryId); + // 2) have we hit the query logic's results-per-page limit? + if (!finished && results.size() >= logicResultsPerPage) { + log.info("Query [{}]: query logic max page size has been reached, aborting next call", queryId); finished = true; } - // 3) have we hit the query logic's results-per-page limit? - if (!finished && results.size() >= logicResultsPerPage) { - log.info("Query [{}]: query logic max page size has been reached, aborting next call", queryId); + // 3) was this query canceled? + if (!finished && canceled) { + log.info("Query [{}]: query cancelled, aborting next call", queryId); + + status = ResultsPage.Status.PARTIAL; finished = true; } @@ -162,7 +166,24 @@ private boolean isFinished(String queryId) { finished = true; } - // 5) have we hit the max results (or the max results override)? + // 5) have we retrieved all of the results? + if (!finished && !getTaskStates().hasUnfinishedTasks()) { + // if all tasks have completed (or failed), start keeping track of the result count + if (tasksFinished) { + // if we stop getting results, we are done + if (prevResultCount == results.size()) { + log.info("Query [{}]: all query tasks complete, and all results retrieved, aborting next call", queryId); + + status = ResultsPage.Status.PARTIAL; + + finished = true; + } + } else { + tasksFinished = true; + } + } + + // 6) have we hit the max results (or the max results override)? if (!finished) { long numResultsReturned = getQueryStatus().getNumResultsReturned(); long numResults = numResultsReturned + results.size(); @@ -173,6 +194,8 @@ private boolean isFinished(String queryId) { // TODO: Figure out query metrics metric.setLifecycle(QueryMetric.Lifecycle.MAXRESULTS); + status = ResultsPage.Status.PARTIAL; + finished = true; } } else if (maxResults >= 0 && numResults >= maxResults) { @@ -181,23 +204,27 @@ private boolean isFinished(String queryId) { // TODO: Figure out query metrics metric.setLifecycle(QueryMetric.Lifecycle.MAXRESULTS); + status = ResultsPage.Status.PARTIAL; + finished = true; } } // TODO: Do I need to pull query metrics to get the next/seek count? // This used to come from the query logic transform iterator - // 6) have we reached the "max work" limit? (i.e. next count + seek count) + // 7) have we reached the "max work" limit? (i.e. next count + seek count) if (!finished && logicMaxWork >= 0 && (metric.getNextCount() + metric.getSeekCount()) >= logicMaxWork) { log.info("Query [{}]: logic max work has been reached, aborting next call", queryId); // TODO: Figure out query metrics metric.setLifecycle(BaseQueryMetric.Lifecycle.MAXWORK); + status = ResultsPage.Status.PARTIAL; + finished = true; } - // 7) are we going to timeout before getting a full page? if so, return partial results + // 8) are we going to timeout before getting a full page? if so, return partial results if (!finished && shortCircuitTimeout(callTimeMillis)) { log.info("Query [{}]: logic max expire before page is full, returning existing results: {} of {} results in {}ms", queryId, results.size(), maxResultsPerPage, callTimeMillis); @@ -205,22 +232,24 @@ private boolean isFinished(String queryId) { status = ResultsPage.Status.PARTIAL; finished = true; - } - // 8) have we been in this next call too long? if so, return + // 9) have we been in this next call too long? if so, return if (!finished && callExpiredTimeout(callTimeMillis)) { log.info("Query [{}]: max call time reached, returning existing results: {} of {} results in {}ms", queryId, results.size(), maxResultsPerPage, callTimeMillis); - status = ResultsPage.Status.PARTIAL; - // TODO: Figure out query metrics metric.setLifecycle(BaseQueryMetric.Lifecycle.NEXTTIMEOUT); + status = ResultsPage.Status.PARTIAL; + finished = true; } + // save the previous result count + prevResultCount = results.size(); + return finished; } @@ -254,14 +283,26 @@ private boolean callExpiredTimeout(long callTimeMillis) { private QueryStatus getQueryStatus() { if (queryStatus == null || isQueryStatusExpired()) { - lastStatusUpdateTime = System.currentTimeMillis(); + lastQueryStatusUpdateTime = System.currentTimeMillis(); queryStatus = queryStorageCache.getQueryStatus(queryId); } return queryStatus; } + private TaskStates getTaskStates() { + if (taskStates == null || isTaskStatesExpired()) { + lastTaskStatesUpdateTime = System.currentTimeMillis(); + taskStates = queryStorageCache.getTaskStates(queryId); + } + return taskStates; + } + private boolean isQueryStatusExpired() { - return (System.currentTimeMillis() - lastStatusUpdateTime) > nextCallProperties.getStatusUpdateIntervalMillis(); + return (System.currentTimeMillis() - lastQueryStatusUpdateTime) > nextCallProperties.getStatusUpdateIntervalMillis(); + } + + private boolean isTaskStatesExpired() { + return (System.currentTimeMillis() - lastTaskStatesUpdateTime) > nextCallProperties.getStatusUpdateIntervalMillis(); } public boolean isCanceled() { From bc4b388c30c45676c0d8e49cd631dfba81b7e75b Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 17 Jun 2021 17:27:42 -0400 Subject: [PATCH 075/218] Working on testing the next call for the query service. --- .../query/QueryManagementService.java | 18 ++- .../microservice/query/runner/NextCall.java | 17 +-- .../microservice/query/QueryServiceTest.java | 144 ++++++++++++++++-- 3 files changed, 150 insertions(+), 29 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index f84fc78c..284193d8 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -50,6 +50,7 @@ import org.springframework.cloud.bus.BusProperties; import org.springframework.cloud.bus.event.RemoteQueryRequestEvent; import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.core.task.TaskRejectedException; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.stereotype.Service; @@ -89,6 +90,7 @@ public class QueryManagementService implements QueryRequestHandler { private final QueryProperties queryProperties; private final ApplicationContext appCtx; + private final ApplicationEventPublisher eventPublisher; private final BusProperties busProperties; // Note: QueryParameters need to be request scoped @@ -108,12 +110,13 @@ public class QueryManagementService implements QueryRequestHandler { private final QueryStatusUpdateHelper queryStatusUpdateHelper; private final MultiValueMap nextCallMap = new LinkedMultiValueMap<>(); - public QueryManagementService(QueryProperties queryProperties, ApplicationContext appCtx, BusProperties busProperties, QueryParameters queryParameters, - SecurityMarking securityMarking, QueryLogicFactory queryLogicFactory, QueryMetricFactory queryMetricFactory, - ResponseObjectFactory responseObjectFactory, QueryStorageCache queryStorageCache, QueryQueueManager queryQueueManager, - AuditClient auditClient, ThreadPoolTaskExecutor nextCallExecutor) { + public QueryManagementService(QueryProperties queryProperties, ApplicationContext appCtx, ApplicationEventPublisher eventPublisher, + BusProperties busProperties, QueryParameters queryParameters, SecurityMarking securityMarking, QueryLogicFactory queryLogicFactory, + QueryMetricFactory queryMetricFactory, ResponseObjectFactory responseObjectFactory, QueryStorageCache queryStorageCache, + QueryQueueManager queryQueueManager, AuditClient auditClient, ThreadPoolTaskExecutor nextCallExecutor) { this.queryProperties = queryProperties; this.appCtx = appCtx; + this.eventPublisher = eventPublisher; this.busProperties = busProperties; this.queryParameters = queryParameters; this.securityMarking = securityMarking; @@ -565,7 +568,6 @@ private BaseQueryResponse next(String queryId, Collection userRoles) thr .setQueryMetricFactory(queryMetricFactory) .setQueryId(queryId) .setQueryLogic(queryLogic) - .setIdentifier(identifier) .build(); // @formatter:on @@ -1549,7 +1551,7 @@ private QueryStatus validateRequest(String queryId, ProxiedUserDetails currentUs // admins requests can operate on any query, regardless of ownership if (!adminOverride) { // does the current user own this query? - String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getDn().subjectDN()); Query query = queryStatus.getQuery(); if (!query.getOwner().equals(userId)) { throw new UnauthorizedQueryException(DatawaveErrorCode.QUERY_OWNER_MISMATCH, MessageFormat.format("{0} != {1}", userId, query.getOwner())); @@ -1678,7 +1680,7 @@ public void publishNextEvent(String queryId, String queryPool) { private void publishExecutorEvent(QueryRequest queryRequest, String queryPool) { // @formatter:off - appCtx.publishEvent( + eventPublisher.publishEvent( new RemoteQueryRequestEvent( this, busProperties.getId(), @@ -1689,7 +1691,7 @@ private void publishExecutorEvent(QueryRequest queryRequest, String queryPool) { private void publishSelfEvent(QueryRequest queryRequest) { // @formatter:off - appCtx.publishEvent( + eventPublisher.publishEvent( new RemoteQueryRequestEvent( this, busProperties.getId(), diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 1dcb4bf7..49c5f50a 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -21,6 +21,7 @@ import java.util.LinkedList; import java.util.List; +import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -32,7 +33,6 @@ public class NextCall implements Callable> { private final QueryQueueManager queryQueueManager; private final QueryStorageCache queryStorageCache; private final String queryId; - private final String identifier; private volatile boolean canceled = false; private volatile Future> future = null; @@ -69,7 +69,6 @@ private NextCall(Builder builder) { this.queryQueueManager = builder.queryQueueManager; this.queryStorageCache = builder.queryStorageCache; this.queryId = builder.queryId; - this.identifier = builder.identifier; QueryStatus status = getQueryStatus(); long pageTimeoutMillis = TimeUnit.MINUTES.toMillis(status.getQuery().getPageTimeout()); @@ -107,7 +106,7 @@ private NextCall(Builder builder) { public ResultsPage call() { startTimeMillis = System.currentTimeMillis(); - QueryQueueListener resultListener = queryQueueManager.createListener(identifier, queryId); + QueryQueueListener resultListener = queryQueueManager.createListener(UUID.randomUUID().toString(), queryId); // keep waiting for results until we're finished while (!isFinished(queryId)) { @@ -142,7 +141,7 @@ private boolean isFinished(String queryId) { } // 2) have we hit the query logic's results-per-page limit? - if (!finished && results.size() >= logicResultsPerPage) { + if (!finished && logicResultsPerPage > 0 && results.size() >= logicResultsPerPage) { log.info("Query [{}]: query logic max page size has been reached, aborting next call", queryId); finished = true; @@ -158,7 +157,7 @@ private boolean isFinished(String queryId) { } // 4) have we hit the query logic's bytes-per-page limit? - if (!finished && pageSizeBytes >= logicBytesPerPage) { + if (!finished && logicBytesPerPage > 0 && pageSizeBytes >= logicBytesPerPage) { log.info("Query [{}]: query logic max page byte size has been reached, aborting next call", queryId); status = ResultsPage.Status.PARTIAL; @@ -213,7 +212,7 @@ private boolean isFinished(String queryId) { // TODO: Do I need to pull query metrics to get the next/seek count? // This used to come from the query logic transform iterator // 7) have we reached the "max work" limit? (i.e. next count + seek count) - if (!finished && logicMaxWork >= 0 && (metric.getNextCount() + metric.getSeekCount()) >= logicMaxWork) { + if (!finished && logicMaxWork > 0 && (metric.getNextCount() + metric.getSeekCount()) >= logicMaxWork) { log.info("Query [{}]: logic max work has been reached, aborting next call", queryId); // TODO: Figure out query metrics @@ -333,7 +332,6 @@ public static class Builder { private QueryMetricFactory queryMetricFactory; private String queryId; private QueryLogic queryLogic; - private String identifier; public Builder setQueryProperties(QueryProperties queryProperties) { this.nextCallProperties = queryProperties.getNextCall(); @@ -376,11 +374,6 @@ public Builder setQueryLogic(QueryLogic queryLogic) { return this; } - public Builder setIdentifier(String identifier) { - this.identifier = identifier; - return this; - } - public NextCall build() { return new NextCall(this); } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java index bdcc7c18..a8ac2625 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java @@ -6,19 +6,24 @@ import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.query.config.QueryProperties; +import datawave.microservice.query.remote.QueryRequest; import datawave.microservice.query.storage.QueryStatus; import datawave.microservice.query.storage.QueryStorageCache; +import datawave.microservice.query.storage.Result; import datawave.microservice.query.storage.TaskKey; import datawave.microservice.query.storage.TaskStates; +import datawave.microservice.query.storage.queue.TestQueryQueueManager; import datawave.security.authorization.DatawaveUser; import datawave.security.authorization.SubjectIssuerDNPair; import datawave.webservice.query.Query; import datawave.webservice.query.exception.QueryExceptionType; +import datawave.webservice.result.BaseQueryResponse; import datawave.webservice.result.GenericResponse; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URLEncodedUtils; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.DirectFieldAccessor; @@ -26,8 +31,13 @@ import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.boot.web.server.LocalServerPort; +import org.springframework.cloud.bus.event.RemoteQueryRequestEvent; +import org.springframework.context.ApplicationEvent; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; import org.springframework.context.annotation.Profile; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; @@ -54,7 +64,10 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.LinkedList; import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import static datawave.microservice.query.QueryParameters.QUERY_MAX_CONCURRENT_TASKS; import static datawave.microservice.query.QueryParameters.QUERY_MAX_RESULTS_OVERRIDE; @@ -64,6 +77,7 @@ import static datawave.webservice.common.audit.AuditParameters.QUERY_STRING; import static datawave.webservice.query.QueryImpl.BEGIN_DATE; import static org.springframework.test.web.client.ExpectedCount.never; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.anything; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; @@ -93,12 +107,18 @@ public class QueryServiceTest { @Autowired private QueryStorageCache queryStorageCache; + @Autowired + private TestQueryQueueManager queryQueueManager; + @Autowired private AuditClient auditClient; @Autowired private QueryProperties queryProperties; + @Autowired + private LinkedList queryRequestEvents; + private List auditIds; private MockRestServiceServer mockServer; @@ -112,6 +132,8 @@ public void setup() { RestTemplate auditorRestTemplate = (RestTemplate) new DirectFieldAccessor(auditClient).getPropertyValue("jwtRestTemplate"); mockServer = MockRestServiceServer.createServer(auditorRestTemplate); + + queryRequestEvents.clear(); } @DirtiesContext @@ -505,6 +527,16 @@ public void testCreateSuccess() throws ParseException, IOException { String queryId = genericResponse.getResult(); Assert.assertNotNull(queryId); + // verify that the create event was published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + // verify that query status was created correctly QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); @@ -538,15 +570,6 @@ public void testCreateSuccess() throws ParseException, IOException { // verify that query tasks were created assertTasksCreated(queryId); - - // TODO: Things to test - // successful query definition - // failed parameter validation - // query logic parameter validation - // missing required roles - // failed security marking validation - // failed query storage - // unknown error } @Test @@ -904,6 +927,74 @@ public void testCreateFailure_markingValidation() { assertAuditNotSent(); } + // Next tests + // successful next + // query not found + // query not running + // query ownership failure + // query lock failure + // interrupted next call + // next call timeout + // no results + // executor task rejection + // more + @Ignore + @Test + public void testNextSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser); + + UriComponents uri = createUri(queryId + "/next"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + Future> future = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, BaseQueryResponse.class)); + + // pump enough results into the queue to trigger a complete page + int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); + for (int resultId = 0; resultId < pageSize; resultId++) { + queryQueueManager.sendMessage(queryId, new Result(Integer.toString(resultId), new String[] {"this", "that"})); + } + + // the response should come back right away + ResponseEntity response = future.get(); + + // TODO: Work through the missing classpath entries needed for DocumentTransformer.java + // TODO: Update the result payload to be BaseEvent + + // verify that the next event was published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + + System.out.println("done"); + + } + + private String createQuery(ProxiedUserDetails authUser) { + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + // remove the create event + queryRequestEvents.clear(); + + return (String) resp.getBody().getResult(); + } + private ProxiedUserDetails createUserDetails() { return createUserDetails(null, null); } @@ -933,6 +1024,12 @@ private MultiValueMap createParams() { return map; } + private void assertQueryRequestEvent(String destination, QueryRequest.Method method, String queryId, RemoteQueryRequestEvent queryRequestEvent) { + Assert.assertEquals(destination, queryRequestEvent.getDestinationService()); + Assert.assertEquals(queryId, queryRequestEvent.getRequest().getQueryId()); + Assert.assertEquals(method, queryRequestEvent.getRequest().getMethod()); + } + private void assertQueryStatus(QueryStatus.QUERY_STATE queryState, long numResultsReturned, long numResultsGenerated, long concurrentNextCount, long lastPageNumber, long lastCallTimeMillis, QueryStatus queryStatus) { Assert.assertEquals(queryState, queryStatus.getQueryState()); @@ -982,6 +1079,10 @@ public RequestMatcher auditIdGrabber() { }; } + private void auditIgnoreSetup() { + mockServer.expect(anything()).andRespond(withSuccess()); + } + private void auditSentSetup() { mockServer.expect(requestTo(EXPECTED_AUDIT_URI)).andExpect(auditIdGrabber()).andRespond(withSuccess()); } @@ -1028,6 +1129,31 @@ public void handleError(ClientHttpResponse response) throws IOException { @Profile("QueryServiceTest") @ComponentScan(basePackages = "datawave.microservice") public static class QueryServiceTestConfiguration { + @Bean + public LinkedList queryRequestEvents() { + return new LinkedList<>(); + } + @Bean + @Primary + public ApplicationEventPublisher eventPublisher(ApplicationEventPublisher eventPublisher) { + return new ApplicationEventPublisher() { + @Override + public void publishEvent(ApplicationEvent event) { + saveEvent(event); + } + + @Override + public void publishEvent(Object event) { + saveEvent(event); + } + + private void saveEvent(Object event) { + if (event instanceof RemoteQueryRequestEvent) { + queryRequestEvents().push(((RemoteQueryRequestEvent) event)); + } + } + }; + } } } From 7a3583cb0f8f720f34b3e9c978f5a6bb5ca30742 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Fri, 25 Jun 2021 15:53:09 +0000 Subject: [PATCH 076/218] Updated to encorporate consumer acknowledgements for the results --- .../query/QueryManagementService.java | 16 ++++++++-------- .../query/monitor/cache/MonitorStatusCache.java | 5 +++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 284193d8..e75a4846 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -322,8 +322,8 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p * Validates the query request, creates an entry in the query storage cache, and publishes a create event to the executor service. *

* Validation is run against the requested logic, the parameters, and the security markings in {@link #validateQuery}.
- * Auditing is performed when {@param isCreateRequest} is true using {@link #audit}.
- * If {@param queryId} is null, a query id will be generated automatically. + * Auditing is performed when {@code isCreateRequest} is true using {@link #audit}.
+ * If {@code queryId} is null, a query id will be generated automatically. * * @param queryLogicName * the requested query logic, not null @@ -729,7 +729,7 @@ public VoidResponse adminCancelAll(ProxiedUserDetails currentUser) throws QueryE * Cancel can only be called on a running query, or a query that is in the process of closing.
* Outstanding next calls will be stopped immediately, but will return partial results if applicable.
* Publishes a cancel event to the query and executor services.
- * Query ownership will only be validated when {@param adminOverride} is set to true. + * Query ownership will only be validated when {@code adminOverride} is set to true. * * @param queryId * the query id, not null @@ -779,9 +779,9 @@ private VoidResponse cancel(String queryId, ProxiedUserDetails currentUser, bool * Cancels the specified query, and optionally publishes a cancel event to the query and executor services. *

* Cancels any locally-running next calls.
- * Called with {@param publishEvent} set to true when the user calls cancel.
- * When {@param publishEvent} is true, changes the query state to {@link QueryStatus.QUERY_STATE#CANCELED}.
- * Called with {@param publishEvent} set to false when handling a remote cancel event. + * Called with {@code publishEvent} set to true when the user calls cancel.
+ * When {@code publishEvent} is true, changes the query state to {@link QueryStatus.QUERY_STATE#CANCELED}.
+ * Called with {@code publishEvent} set to false when handling a remote cancel event. * * @param queryId * the query id, not null @@ -921,7 +921,7 @@ public VoidResponse adminCloseAll(ProxiedUserDetails currentUser) throws QueryEx *

* Close can only be called on a running query.
* Outstanding next calls will be allowed to run until they can return a full page, or they timeout.
- * Query ownership will only be validated when {@param adminOverride} is set to true. + * Query ownership will only be validated when {@code adminOverride} is set to true. * * @param queryId * the query id, not null @@ -1137,7 +1137,7 @@ public VoidResponse adminRemoveAll(ProxiedUserDetails currentUser) throws QueryE * Removes the specified query from query storage. *

* Remove can only be called on a query that is not running.
- * Query ownership will only be validated when {@param adminOverride} is set to true. + * Query ownership will only be validated when {@code adminOverride} is set to true. * * @param queryId * the query id, not null diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java index 914f607e..0a7b2c67 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java @@ -21,7 +21,7 @@ public MonitorStatusCache(LockableCacheInspector cacheInspector) { /** * Get the query monitor status * - * @return + * @return the stored monitor status */ public MonitorStatus getStatus() { return cacheInspector.list(CACHE_NAME, MonitorStatus.class, CACHE_KEY); @@ -31,7 +31,8 @@ public MonitorStatus getStatus() { * Store the query monitor status * * @param monitorStatus - * @return + * The monitor status to store + * @return the stored monitor status */ @CachePut(key = CACHE_KEY) public MonitorStatus setStatus(MonitorStatus monitorStatus) { From be846244f17882229b4c2fdb9c55c8c2be57d55d Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Fri, 25 Jun 2021 18:19:17 +0000 Subject: [PATCH 077/218] Updated to generate results if numConcurrentNextCalls>0 when a query is closed. Updated to adjust the number of results to generate capped by max results. Added test cases to verify we generate results as appropriate (ShouldGenerateResultsTest) Updated a bunch of javadoc to remove all warnings --- .../datawave/microservice/query/DefaultQueryParameters.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java index 0476b011..1e1b53ee 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java @@ -68,6 +68,7 @@ public DefaultQueryParameters() { * @param parameters * - a Map of QueryParameters * @throws IllegalArgumentException + * if not exactly 1 value per parameter */ public void validate(Map> parameters) throws IllegalArgumentException { for (String param : KNOWN_PARAMS) { @@ -302,14 +303,16 @@ public static synchronized Date parseDate(String s, String defaultTime, String d * @param pagesize * - * @param pageTimeout + * - * @param maxResultsOverride + * - * @param persistenceMode * - * @param parameters * - additional parameters passed in as map * @param trace * - - * @return + * @return the parameters * @throws ParseException * on date parse/format error */ From ef1a7ffed28b1324ae3a9f79c923931a2bcff81e Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 15 Jul 2021 16:07:51 -0400 Subject: [PATCH 078/218] Resolved missing class errors when running query next test. --- .../microservice/query/runner/NextCall.java | 22 +++++++++++-------- .../microservice/query/QueryServiceTest.java | 15 +++++++++++-- .../src/test/resources/config/application.yml | 3 ++- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 49c5f50a..d86a7e66 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -112,16 +112,20 @@ public ResultsPage call() { while (!isFinished(queryId)) { Message message = resultListener.receive(nextCallProperties.getResultPollIntervalMillis()); if (message != null) { - Object result = message.getPayload().getPayload(); - if (result != null) { - results.add(result); - - if (logicBytesPerPage > 0) { - pageSizeBytes += ObjectSizeOf.Sizer.getObjectSize(result); + for (Object result : message.getPayload().getPayload()) { + // have to check to make sure we haven't reached the page size + if (!isFinished(queryId)) { + if (result != null) { + results.add(result); + + if (logicBytesPerPage > 0) { + pageSizeBytes += ObjectSizeOf.Sizer.getObjectSize(result); + } + } else { + log.debug("Null result encountered, no more results"); + break; + } } - } else { - log.debug("Null result encountered, no more results"); - break; } } } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java index a8ac2625..7fd38a93 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java @@ -17,6 +17,8 @@ import datawave.security.authorization.SubjectIssuerDNPair; import datawave.webservice.query.Query; import datawave.webservice.query.exception.QueryExceptionType; +import datawave.webservice.query.result.event.DefaultEvent; +import datawave.webservice.query.result.event.DefaultField; import datawave.webservice.result.BaseQueryResponse; import datawave.webservice.result.GenericResponse; import org.apache.http.NameValuePair; @@ -938,7 +940,8 @@ public void testCreateFailure_markingValidation() { // no results // executor task rejection // more - @Ignore + // @Ignore + @DirtiesContext @Test public void testNextSuccess() throws Exception { ProxiedUserDetails authUser = createUserDetails(); @@ -956,7 +959,15 @@ public void testNextSuccess() throws Exception { // pump enough results into the queue to trigger a complete page int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); for (int resultId = 0; resultId < pageSize; resultId++) { - queryQueueManager.sendMessage(queryId, new Result(Integer.toString(resultId), new String[] {"this", "that"})); + DefaultEvent[] events = new DefaultEvent[1]; + events[0] = new DefaultEvent(); + long currentTime = System.currentTimeMillis(); + // @formatter:off + events[0].setFields(Arrays.asList( + new DefaultField("LOKI", "ALL", currentTime, "ALLIGATOR"), + new DefaultField("LOKI", "ALL", currentTime, "CLASSIC"))); + // @formatter:on + queryQueueManager.sendMessage(queryId, new Result(Integer.toString(resultId), events)); } // the response should come back right away diff --git a/query-microservices/query-service/service/src/test/resources/config/application.yml b/query-microservices/query-service/service/src/test/resources/config/application.yml index 4800ec60..8b491193 100644 --- a/query-microservices/query-service/service/src/test/resources/config/application.yml +++ b/query-microservices/query-service/service/src/test/resources/config/application.yml @@ -85,7 +85,8 @@ warehouse: maxEvaluationPipelines: 16 maxPipelineCachedResults: 16 hdfsSiteConfigURLs: 'file:///etc/hadoop/conf/core-site.xml,file:///etc/hadoop/conf/hdfs-site.xml' - ivaratorFstHdfsBaseURIs: 'hdfs:///IvaratorCache' + ivaratorFstHdfsBaseURIs: + - basePathURI: 'hdfs:///IvaratorCache' ivaratorCacheBufferSize: 10000 ivaratorMaxOpenFiles: 100 ivaratorCacheScanPersistThreshold: 100000 From 2ad4b968aa9833f3c711ab667495db30468ddd8d Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 22 Jul 2021 17:07:54 -0400 Subject: [PATCH 079/218] Added success and failure tests for the query service next call. --- .../query/config/NextCallProperties.java | 4 +- .../microservice/query/runner/NextCall.java | 18 +- .../microservice/query/QueryServiceTest.java | 741 +++++++++++++++--- .../src/test/resources/config/application.yml | 14 +- .../src/test/resources/log4j2-test.xml | 4 +- 5 files changed, 656 insertions(+), 125 deletions(-) diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/NextCallProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/NextCallProperties.java index 52deda82..53262c6b 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/NextCallProperties.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/NextCallProperties.java @@ -10,13 +10,13 @@ @Validated public class NextCallProperties { @PositiveOrZero - private long resultPollInterval = TimeUnit.SECONDS.toMillis(30); + private long resultPollInterval = TimeUnit.SECONDS.toMillis(6); @NotNull private TimeUnit resultPollTimeUnit = TimeUnit.MILLISECONDS; @Positive private int concurrency = 1; @PositiveOrZero - private long statusUpdateInterval = TimeUnit.SECONDS.toMillis(30); + private long statusUpdateInterval = TimeUnit.SECONDS.toMillis(6); @NotNull private TimeUnit statusUpdateTimeUnit = TimeUnit.MILLISECONDS; diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index d86a7e66..93e7dd83 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -112,7 +112,10 @@ public ResultsPage call() { while (!isFinished(queryId)) { Message message = resultListener.receive(nextCallProperties.getResultPollIntervalMillis()); if (message != null) { - for (Object result : message.getPayload().getPayload()) { + Object[] payload = message.getPayload().getPayload(); + for (int resultIdx = 0; resultIdx < payload.length; resultIdx++) { + Object result = payload[resultIdx]; + // have to check to make sure we haven't reached the page size if (!isFinished(queryId)) { if (result != null) { @@ -125,17 +128,26 @@ public ResultsPage call() { log.debug("Null result encountered, no more results"); break; } + } else { + // TODO: This will no longer be possible once the result queues are updated to pass single results instead of arrays + if ((resultIdx + 1) != payload.length) { + log.error("Next call is dropping " + (payload.length - (resultIdx + 1)) + " results"); + } } } } } + // stop the result listener + resultListener.stop(); + return new ResultsPage<>(results, status); } private boolean isFinished(String queryId) { boolean finished = false; long callTimeMillis = System.currentTimeMillis() - startTimeMillis; + final QueryStatus queryStatus = getQueryStatus(); // 1) have we hit the user's results-per-page limit? if (results.size() >= userResultsPerPage) { @@ -152,7 +164,7 @@ private boolean isFinished(String queryId) { } // 3) was this query canceled? - if (!finished && canceled) { + if (!finished && (canceled || queryStatus.getQueryState() == QueryStatus.QUERY_STATE.CANCELED)) { log.info("Query [{}]: query cancelled, aborting next call", queryId); status = ResultsPage.Status.PARTIAL; @@ -188,7 +200,7 @@ private boolean isFinished(String queryId) { // 6) have we hit the max results (or the max results override)? if (!finished) { - long numResultsReturned = getQueryStatus().getNumResultsReturned(); + long numResultsReturned = queryStatus.getNumResultsReturned(); long numResults = numResultsReturned + results.size(); if (this.maxResultsOverridden) { if (maxResultsOverride >= 0 && numResults >= maxResultsOverride) { diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java index 7fd38a93..a80e4b0a 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java @@ -1,5 +1,6 @@ package datawave.microservice.query; +import com.google.common.collect.Iterables; import datawave.marking.ColumnVisibilitySecurityMarking; import datawave.microservice.audit.AuditClient; import datawave.microservice.authorization.jwt.JWTRestTemplate; @@ -19,13 +20,14 @@ import datawave.webservice.query.exception.QueryExceptionType; import datawave.webservice.query.result.event.DefaultEvent; import datawave.webservice.query.result.event.DefaultField; -import datawave.webservice.result.BaseQueryResponse; +import datawave.webservice.result.BaseResponse; +import datawave.webservice.result.DefaultEventQueryResponse; import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.VoidResponse; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URLEncodedUtils; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.DirectFieldAccessor; @@ -68,8 +70,12 @@ import java.util.Collections; import java.util.LinkedList; import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.stream.Collectors; import static datawave.microservice.query.QueryParameters.QUERY_MAX_CONCURRENT_TASKS; import static datawave.microservice.query.QueryParameters.QUERY_MAX_RESULTS_OVERRIDE; @@ -106,6 +112,9 @@ public class QueryServiceTest { private SubjectIssuerDNPair DN; private String userDN = "userDn"; + private SubjectIssuerDNPair altDN; + private String altUserDN = "altUserDN"; + @Autowired private QueryStorageCache queryStorageCache; @@ -131,6 +140,7 @@ public void setup() { jwtRestTemplate = restTemplateBuilder.build(JWTRestTemplate.class); jwtRestTemplate.setErrorHandler(new NoOpResponseErrorHandler()); DN = SubjectIssuerDNPair.of(userDN, "issuerDn"); + altDN = SubjectIssuerDNPair.of(altUserDN, "issuerDN"); RestTemplate auditorRestTemplate = (RestTemplate) new DirectFieldAccessor(auditClient).getPropertyValue("jwtRestTemplate"); mockServer = MockRestServiceServer.createServer(auditorRestTemplate); @@ -208,22 +218,22 @@ public void testDefineFailure_paramValidation() { RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); // @formatter:off - GenericResponse genericResponse = assertGenericResponse( + BaseResponse baseResponse = assertBaseResponse( false, HttpStatus.Series.CLIENT_ERROR, resp); // @formatter:on // verify that there is no result - Assert.assertNull(genericResponse.getResult()); + Assert.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, genericResponse.getExceptions().size()); + Assert.assertEquals(1, baseResponse.getExceptions().size()); - QueryExceptionType queryException = genericResponse.getExceptions().get(0); + QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off assertQueryException( "Missing one or more required QueryParameters", @@ -244,22 +254,22 @@ public void testDefineFailure_authValidation() { RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); // @formatter:off - GenericResponse genericResponse = assertGenericResponse( + BaseResponse baseResponse = assertBaseResponse( false, HttpStatus.Series.CLIENT_ERROR, resp); // @formatter:on // verify that there is no result - Assert.assertNull(genericResponse.getResult()); + Assert.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, genericResponse.getExceptions().size()); + Assert.assertEquals(1, baseResponse.getExceptions().size()); - QueryExceptionType queryException = genericResponse.getExceptions().get(0); + QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off assertQueryException( "User requested authorizations that they don't have. Missing: [ALL], Requested: [ALL], User: []", @@ -283,22 +293,22 @@ public void testDefineFailure_queryLogicValidation() { RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); // @formatter:off - GenericResponse genericResponse = assertGenericResponse( + BaseResponse baseResponse = assertBaseResponse( false, HttpStatus.Series.CLIENT_ERROR, resp); // @formatter:on // verify that there is no result - Assert.assertNull(genericResponse.getResult()); + Assert.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, genericResponse.getExceptions().size()); + Assert.assertEquals(1, baseResponse.getExceptions().size()); - QueryExceptionType queryException = genericResponse.getExceptions().get(0); + QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off assertQueryException( "Required parameter begin not found", @@ -322,22 +332,22 @@ public void testDefineFailure_maxPageSize() { RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); // @formatter:off - GenericResponse genericResponse = assertGenericResponse( + BaseResponse baseResponse = assertBaseResponse( false, HttpStatus.Series.CLIENT_ERROR, resp); // @formatter:on // verify that there is no result - Assert.assertNull(genericResponse.getResult()); + Assert.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, genericResponse.getExceptions().size()); + Assert.assertEquals(1, baseResponse.getExceptions().size()); - QueryExceptionType queryException = genericResponse.getExceptions().get(0); + QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off assertQueryException( "Page size is larger than configured max. Max = 10,000.", @@ -361,25 +371,25 @@ public void testDefineFailure_maxResultsOverride() { RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); // @formatter:off - GenericResponse genericResponse = assertGenericResponse( + BaseResponse baseResponse = assertBaseResponse( false, HttpStatus.Series.CLIENT_ERROR, resp); // @formatter:on // verify that there is no result - Assert.assertNull(genericResponse.getResult()); + Assert.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, genericResponse.getExceptions().size()); + Assert.assertEquals(1, baseResponse.getExceptions().size()); - QueryExceptionType queryException = genericResponse.getExceptions().get(0); + QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off assertQueryException( - "Invalid max results override value. Max = 1,000,000.", + "Invalid max results override value. Max = 369.", "Exception with no cause caught", "400-43", queryException); @@ -400,22 +410,22 @@ public void testDefineFailure_maxConcurrentTasksOverride() { RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); // @formatter:off - GenericResponse genericResponse = assertGenericResponse( + BaseResponse baseResponse = assertBaseResponse( false, HttpStatus.Series.CLIENT_ERROR, resp); // @formatter:on // verify that there is no result - Assert.assertNull(genericResponse.getResult()); + Assert.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, genericResponse.getExceptions().size()); + Assert.assertEquals(1, baseResponse.getExceptions().size()); - QueryExceptionType queryException = genericResponse.getExceptions().get(0); + QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off assertQueryException( "Invalid max concurrent tasks override value. Max = 10.", @@ -437,22 +447,22 @@ public void testDefineFailure_roleValidation() { RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); // @formatter:off - GenericResponse genericResponse = assertGenericResponse( + BaseResponse baseResponse = assertBaseResponse( false, HttpStatus.Series.CLIENT_ERROR, resp); // @formatter:on // verify that there is no result - Assert.assertNull(genericResponse.getResult()); + Assert.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, genericResponse.getExceptions().size()); + Assert.assertEquals(1, baseResponse.getExceptions().size()); - QueryExceptionType queryException = genericResponse.getExceptions().get(0); + QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off assertQueryException( "User does not have the required roles.", @@ -476,22 +486,22 @@ public void testDefineFailure_markingValidation() { RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); // @formatter:off - GenericResponse genericResponse = assertGenericResponse( + BaseResponse baseResponse = assertBaseResponse( false, HttpStatus.Series.CLIENT_ERROR, resp); // @formatter:on // verify that there is no result - Assert.assertNull(genericResponse.getResult()); + Assert.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, genericResponse.getExceptions().size()); + Assert.assertEquals(1, baseResponse.getExceptions().size()); - QueryExceptionType queryException = genericResponse.getExceptions().get(0); + QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off assertQueryException( "Required parameter columnVisibility not found", @@ -588,22 +598,22 @@ public void testCreateFailure_paramValidation() { // setup a mock audit service auditNotSentSetup(); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); // @formatter:off - GenericResponse genericResponse = assertGenericResponse( + BaseResponse baseResponse = assertBaseResponse( false, HttpStatus.Series.CLIENT_ERROR, resp); // @formatter:on // verify that there is no result - Assert.assertNull(genericResponse.getResult()); + Assert.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, genericResponse.getExceptions().size()); + Assert.assertEquals(1, baseResponse.getExceptions().size()); - QueryExceptionType queryException = genericResponse.getExceptions().get(0); + QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off assertQueryException( "Missing one or more required QueryParameters", @@ -630,22 +640,22 @@ public void testCreateFailure_authValidation() { // setup a mock audit service auditNotSentSetup(); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); // @formatter:off - GenericResponse genericResponse = assertGenericResponse( + BaseResponse baseResponse = assertBaseResponse( false, HttpStatus.Series.CLIENT_ERROR, resp); // @formatter:on // verify that there is no result - Assert.assertNull(genericResponse.getResult()); + Assert.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, genericResponse.getExceptions().size()); + Assert.assertEquals(1, baseResponse.getExceptions().size()); - QueryExceptionType queryException = genericResponse.getExceptions().get(0); + QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off assertQueryException( "User requested authorizations that they don't have. Missing: [ALL], Requested: [ALL], User: []", @@ -675,22 +685,22 @@ public void testCreateFailure_queryLogicValidation() { // setup a mock audit service auditNotSentSetup(); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); // @formatter:off - GenericResponse genericResponse = assertGenericResponse( + BaseResponse baseResponse = assertBaseResponse( false, HttpStatus.Series.CLIENT_ERROR, resp); // @formatter:on // verify that there is no result - Assert.assertNull(genericResponse.getResult()); + Assert.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, genericResponse.getExceptions().size()); + Assert.assertEquals(1, baseResponse.getExceptions().size()); - QueryExceptionType queryException = genericResponse.getExceptions().get(0); + QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off assertQueryException( "Required parameter begin not found", @@ -720,22 +730,22 @@ public void testCreateFailure_maxPageSize() { // setup a mock audit service auditNotSentSetup(); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); // @formatter:off - GenericResponse genericResponse = assertGenericResponse( + BaseResponse baseResponse = assertBaseResponse( false, HttpStatus.Series.CLIENT_ERROR, resp); // @formatter:on // verify that there is no result - Assert.assertNull(genericResponse.getResult()); + Assert.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, genericResponse.getExceptions().size()); + Assert.assertEquals(1, baseResponse.getExceptions().size()); - QueryExceptionType queryException = genericResponse.getExceptions().get(0); + QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off assertQueryException( "Page size is larger than configured max. Max = 10,000.", @@ -765,25 +775,25 @@ public void testCreateFailure_maxResultsOverride() { // setup a mock audit service auditNotSentSetup(); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); // @formatter:off - GenericResponse genericResponse = assertGenericResponse( + BaseResponse baseResponse = assertBaseResponse( false, HttpStatus.Series.CLIENT_ERROR, resp); // @formatter:on // verify that there is no result - Assert.assertNull(genericResponse.getResult()); + Assert.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, genericResponse.getExceptions().size()); + Assert.assertEquals(1, baseResponse.getExceptions().size()); - QueryExceptionType queryException = genericResponse.getExceptions().get(0); + QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off assertQueryException( - "Invalid max results override value. Max = 1,000,000.", + "Invalid max results override value. Max = 369.", "Exception with no cause caught", "400-43", queryException); @@ -810,22 +820,22 @@ public void testCreateFailure_maxConcurrentTasksOverride() { // setup a mock audit service auditNotSentSetup(); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); // @formatter:off - GenericResponse genericResponse = assertGenericResponse( + BaseResponse baseResponse = assertBaseResponse( false, HttpStatus.Series.CLIENT_ERROR, resp); // @formatter:on // verify that there is no result - Assert.assertNull(genericResponse.getResult()); + Assert.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, genericResponse.getExceptions().size()); + Assert.assertEquals(1, baseResponse.getExceptions().size()); - QueryExceptionType queryException = genericResponse.getExceptions().get(0); + QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off assertQueryException( "Invalid max concurrent tasks override value. Max = 10.", @@ -853,22 +863,22 @@ public void testCreateFailure_roleValidation() { // setup a mock audit service auditNotSentSetup(); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); // @formatter:off - GenericResponse genericResponse = assertGenericResponse( + BaseResponse baseResponse = assertBaseResponse( false, HttpStatus.Series.CLIENT_ERROR, resp); // @formatter:on // verify that there is no result - Assert.assertNull(genericResponse.getResult()); + Assert.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, genericResponse.getExceptions().size()); + Assert.assertEquals(1, baseResponse.getExceptions().size()); - QueryExceptionType queryException = genericResponse.getExceptions().get(0); + QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off assertQueryException( "User does not have the required roles.", @@ -898,22 +908,22 @@ public void testCreateFailure_markingValidation() { // setup a mock audit service auditNotSentSetup(); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); // @formatter:off - GenericResponse genericResponse = assertGenericResponse( + BaseResponse baseResponse = assertBaseResponse( false, HttpStatus.Series.CLIENT_ERROR, resp); // @formatter:on // verify that there is no result - Assert.assertNull(genericResponse.getResult()); + Assert.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, genericResponse.getExceptions().size()); + Assert.assertEquals(1, baseResponse.getExceptions().size()); - QueryExceptionType queryException = genericResponse.getExceptions().get(0); + QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off assertQueryException( "Required parameter columnVisibility not found", @@ -929,52 +939,368 @@ public void testCreateFailure_markingValidation() { assertAuditNotSent(); } - // Next tests - // successful next - // query not found - // query not running - // query ownership failure - // query lock failure - // interrupted next call - // next call timeout - // no results - // executor task rejection - // more - // @Ignore @DirtiesContext @Test public void testNextSuccess() throws Exception { ProxiedUserDetails authUser = createUserDetails(); // create a valid query - String queryId = createQuery(authUser); + String queryId = createQuery(authUser, createParams()); - UriComponents uri = createUri(queryId + "/next"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + // pump enough results into the queue to trigger a complete page + int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("LOKI", "ALLIGATOR"); + fieldValues.add("LOKI", "CLASSIC"); + + // @formatter:off + publishEventsToQueue( + queryId, + (int)(1.5*pageSize), + fieldValues, + "ALL"); + // @formatter:on // make the next call asynchronously - Future> future = Executors.newSingleThreadExecutor() - .submit(() -> jwtRestTemplate.exchange(requestEntity, BaseQueryResponse.class)); + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // verify some headers + Assert.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + + DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); + + // verify the query response + // @formatter:off + assertQueryResponse( + queryId, + "EventQuery", + 1, + false, + Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-OperationTimeInMS")))), + 1, + Collections.singletonList("LOKI"), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Arrays.asList("LOKI", "LOKI"), + Arrays.asList("ALLIGATOR", "CLASSIC"), + event); + // @formatter:on + + // verify that the next event was published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testNextSuccess_multiplePages() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // pump enough results into the queue to trigger two complete pages + int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("LOKI", "ALLIGATOR"); + fieldValues.add("LOKI", "CLASSIC"); + + for (int page = 1; page <= 2; page++) { + // TODO: We have to generate the results in between next calls because the test queue manager does not handle requeueing of unused messages :( + // @formatter:off + publishEventsToQueue( + queryId, + pageSize, + fieldValues, + "ALL"); + // @formatter:on + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // verify some headers + Assert.assertEquals(Integer.toString(page), Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + + DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); + + // verify the query response + // @formatter:off + assertQueryResponse( + queryId, + "EventQuery", + page, + false, + Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-OperationTimeInMS")))), + 1, + Collections.singletonList("LOKI"), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Arrays.asList("LOKI", "LOKI"), + Arrays.asList("ALLIGATOR", "CLASSIC"), + event); + // @formatter:on + + // verify that the next event was published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + } + + @DirtiesContext + @Test + public void testNextSuccess_cancelPartialResults() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); // pump enough results into the queue to trigger a complete page int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); - for (int resultId = 0; resultId < pageSize; resultId++) { - DefaultEvent[] events = new DefaultEvent[1]; - events[0] = new DefaultEvent(); - long currentTime = System.currentTimeMillis(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("LOKI", "ALLIGATOR"); + fieldValues.add("LOKI", "CLASSIC"); + + int numEvents = (int) (0.5 * pageSize); + + // @formatter:off + publishEventsToQueue( + queryId, + numEvents, + fieldValues, + "ALL"); + // @formatter:on + + // make the next call asynchronously + Future> nextFuture = nextQuery(authUser, queryId); + + // make sure all events were consumed before canceling + while (queryQueueManager.getQueueSize(queryId) != 0) { + Thread.sleep(100); + } + + // cancel the query so that it returns partial results + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // the response should come back right away + ResponseEntity nextResponse = nextFuture.get(); + + Assert.assertEquals(200, nextResponse.getStatusCodeValue()); + + // verify some headers + Assert.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-query-page-number")))); + Assert.assertEquals("true", Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-Partial-Results")))); + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-query-last-page")))); + + DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) nextResponse.getBody(); + + // verify the query response + // @formatter:off + assertQueryResponse( + queryId, + "EventQuery", + 1, + true, + Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-OperationTimeInMS")))), + 1, + Collections.singletonList("LOKI"), + numEvents, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Arrays.asList("LOKI", "LOKI"), + Arrays.asList("ALLIGATOR", "CLASSIC"), + event); + // @formatter:on + + // verify that the next events were published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testNextSuccess_maxResults() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // pump enough results into the queue to trigger two complete pages + int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("LOKI", "ALLIGATOR"); + fieldValues.add("LOKI", "CLASSIC"); + + for (int page = 1; page <= 3; page++) { + // TODO: We have to generate the results in between next calls because the test queue manager does not handle requeueing of unused messages :( // @formatter:off - events[0].setFields(Arrays.asList( - new DefaultField("LOKI", "ALL", currentTime, "ALLIGATOR"), - new DefaultField("LOKI", "ALL", currentTime, "CLASSIC"))); + publishEventsToQueue( + queryId, + pageSize, + fieldValues, + "ALL"); + // @formatter:on + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // verify some headers + Assert.assertEquals(Integer.toString(page), Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + + if (page != 4) { + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + } else { + Assert.assertEquals("true", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + } + + DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); + + // verify the query response + // @formatter:off + assertQueryResponse( + queryId, + "EventQuery", + page, + false, + Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-OperationTimeInMS")))), + 1, + Collections.singletonList("LOKI"), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Arrays.asList("LOKI", "LOKI"), + Arrays.asList("ALLIGATOR", "CLASSIC"), + event); + // @formatter:on + + // verify that the next event was published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); // @formatter:on - queryQueueManager.sendMessage(queryId, new Result(Integer.toString(resultId), events)); } + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + // the response should come back right away - ResponseEntity response = future.get(); + ResponseEntity response = future.get(); - // TODO: Work through the missing classpath entries needed for DocumentTransformer.java - // TODO: Update the result payload to be BaseEvent + Assert.assertEquals(404, response.getStatusCodeValue()); + + assertQueryException("No results found for query. " + queryId, "Exception with no cause caught", "404-4", + Iterables.getOnlyElement(response.getBody().getExceptions())); + } + + @DirtiesContext + @Test + public void testNextSuccess_noResults() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // remove the task states to make it appear that the executor has finished + TaskStates taskStates = queryStorageCache.getTaskStates(queryId); + taskStates.getTaskStates().remove(TaskStates.TASK_STATE.READY); + queryStorageCache.updateTaskStates(taskStates); + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(404, response.getStatusCodeValue()); + + assertQueryException("No results found for query. " + queryId, "Exception with no cause caught", "404-4", + Iterables.getOnlyElement(response.getBody().getExceptions())); // verify that the next event was published Assert.assertEquals(1, queryRequestEvents.size()); @@ -985,14 +1311,144 @@ public void testNextSuccess() throws Exception { queryId, queryRequestEvents.removeLast()); // @formatter:on + } + + @Test + public void testNextFailure_queryNotFound() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(404, response.getStatusCodeValue()); - System.out.println("done"); + assertQueryException("No query object matches this id. " + queryId, "Exception with no cause caught", "404-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); } - private String createQuery(ProxiedUserDetails authUser) { + @DirtiesContext + @Test + public void testNextFailure_queryNotRunning() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // cancel the query so that it returns partial results + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + assertQueryException("Cannot call next on a query that is not running", "Exception with no cause caught", "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + + // verify that the next events were published + Assert.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testNextFailure_ownershipFailure() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails altAuthUser = createAltUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // make the next call as an alternate user asynchronously + Future> future = nextQuery(altAuthUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(401, response.getStatusCodeValue()); + + assertQueryException("Current user does not match user that defined query. altuserdn != userdn", "Exception with no cause caught", "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + + // verify that the next events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @DirtiesContext + @Test + public void testNextFailure_timeout() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back after the configured timeout (5 seconds) + ResponseEntity response = future.get(); + + Assert.assertEquals(500, response.getStatusCodeValue()); + + assertQueryException("Query timed out. " + queryId + " timed out.", "Exception with no cause caught", "500-27", + Iterables.getOnlyElement(response.getBody().getExceptions())); + + // verify that the next events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + private void publishEventsToQueue(String queryId, int numEvents, MultiValueMap fieldValues, String visibility) throws Exception { + for (int resultId = 0; resultId < numEvents; resultId++) { + DefaultEvent[] events = new DefaultEvent[1]; + events[0] = new DefaultEvent(); + long currentTime = System.currentTimeMillis(); + List fields = new ArrayList<>(); + for (Map.Entry> entry : fieldValues.entrySet()) { + for (String value : entry.getValue()) { + fields.add(new DefaultField(entry.getKey(), visibility, currentTime, value)); + } + } + events[0].setFields(fields); + queryQueueManager.sendMessage(queryId, new Result(Integer.toString(resultId), events)); + } + } + + private String createQuery(ProxiedUserDetails authUser, MultiValueMap map) { UriComponents uri = createUri("EventQuery/create"); - MultiValueMap map = createParams(); // not testing audit with this method auditIgnoreSetup(); @@ -1006,6 +1462,22 @@ private String createQuery(ProxiedUserDetails authUser) { return (String) resp.getBody().getResult(); } + private Future> nextQuery(ProxiedUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId + "/next"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, BaseResponse.class)); + } + + private Future> cancelQuery(ProxiedUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId + "/cancel"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + private ProxiedUserDetails createUserDetails() { return createUserDetails(null, null); } @@ -1017,6 +1489,17 @@ private ProxiedUserDetails createUserDetails(Collection roles, Collectio return new ProxiedUserDetails(Collections.singleton(datawaveUser), datawaveUser.getCreationTime()); } + private ProxiedUserDetails createAltUserDetails() { + return createAltUserDetails(null, null); + } + + private ProxiedUserDetails createAltUserDetails(Collection roles, Collection auths) { + Collection userRoles = roles != null ? roles : Collections.singleton("AuthorizedUser"); + Collection userAuths = auths != null ? auths : Collections.singleton("ALL"); + DatawaveUser datawaveUser = new DatawaveUser(altDN, USER, userAuths, userRoles, null, System.currentTimeMillis()); + return new ProxiedUserDetails(Collections.singleton(datawaveUser), datawaveUser.getCreationTime()); + } + private UriComponents createUri(String path) { return UriComponentsBuilder.newInstance().scheme("https").host("localhost").port(webServicePort).path("/query/v1/" + path).build(); } @@ -1030,11 +1513,28 @@ private MultiValueMap createParams() { map.set(DefaultQueryParameters.QUERY_END, TEST_QUERY_END); map.set(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, TEST_VISIBILITY_MARKING); map.set(QUERY_MAX_CONCURRENT_TASKS, Integer.toString(1)); - map.set(QUERY_MAX_RESULTS_OVERRIDE, Long.toString(1234)); + map.set(QUERY_MAX_RESULTS_OVERRIDE, Long.toString(369)); map.set(QUERY_PAGESIZE, Long.toString(123)); return map; } + private void assertDefaultEvent(List fields, List values, DefaultEvent event) { + Assert.assertEquals(fields, event.getFields().stream().map(DefaultField::getName).collect(Collectors.toList())); + Assert.assertEquals(values, event.getFields().stream().map(DefaultField::getValueString).collect(Collectors.toList())); + } + + private void assertQueryResponse(String queryId, String logicName, long pageNumber, boolean partialResults, long operationTimeInMS, int numFields, + List fieldNames, int numEvents, DefaultEventQueryResponse queryResponse) { + Assert.assertEquals(queryId, queryResponse.getQueryId()); + Assert.assertEquals(logicName, queryResponse.getLogicName()); + Assert.assertEquals(pageNumber, queryResponse.getPageNumber()); + Assert.assertEquals(partialResults, queryResponse.isPartialResults()); + Assert.assertEquals(operationTimeInMS, queryResponse.getOperationTimeMS()); + Assert.assertEquals(numFields, queryResponse.getFields().size()); + Assert.assertEquals(fieldNames, queryResponse.getFields()); + Assert.assertEquals(numEvents, queryResponse.getEvents().size()); + } + private void assertQueryRequestEvent(String destination, QueryRequest.Method method, String queryId, RemoteQueryRequestEvent queryRequestEvent) { Assert.assertEquals(destination, queryRequestEvent.getDestinationService()); Assert.assertEquals(queryId, queryRequestEvent.getRequest().getQueryId()); @@ -1119,6 +1619,15 @@ private void assertQueryException(String message, String cause, String code, Que Assert.assertEquals(code, queryException.getCode()); } + private BaseResponse assertBaseResponse(boolean hasResults, HttpStatus.Series series, ResponseEntity response) { + Assert.assertEquals(series, response.getStatusCode().series()); + Assert.assertNotNull(response); + BaseResponse baseResponse = response.getBody(); + Assert.assertNotNull(baseResponse); + Assert.assertEquals(hasResults, baseResponse.getHasResults()); + return baseResponse; + } + @SuppressWarnings("unchecked") private GenericResponse assertGenericResponse(boolean hasResults, HttpStatus.Series series, ResponseEntity response) { Assert.assertEquals(series, response.getStatusCode().series()); diff --git a/query-microservices/query-service/service/src/test/resources/config/application.yml b/query-microservices/query-service/service/src/test/resources/config/application.yml index 8b491193..8e595937 100644 --- a/query-microservices/query-service/service/src/test/resources/config/application.yml +++ b/query-microservices/query-service/service/src/test/resources/config/application.yml @@ -1,4 +1,7 @@ spring: + application: + name: query + autoconfigure: exclude: org.springframework.boot.autoconfigure.web.servlet.error.ErrorMvcAutoConfiguration @@ -40,7 +43,8 @@ management: logging: level: - datawave.microservice: FATAL + root: WARN + datawave.microservice.query: WARN warehouse: accumulo: @@ -106,6 +110,12 @@ datawave: "[datawave.data.type.DateType]": "datawave.data.type.RawDateType" query: + nextCall: + resultPollInterval: 500 + statusUpdateInterval: 500 + expiration: + callTimeout: 5 + callTimeUnit: SECONDS parser: skipTokenizeUnfieldedFields: - "DOMETA" @@ -143,7 +153,7 @@ datawave: metadataTableName: ${warehouse.tables.metadata.name} indexTableName: ${warehouse.tables.index.name} reverseIndexTableName: ${warehouse.tables.reverseIndex.name} - maxResults: 1000000 + maxResults: 369 queryThreads: ${warehouse.defaults.queryThreads} indexLookupThreads: ${warehouse.defaults.indexLookupThreads} dateIndexThreads: ${warehouse.defaults.dateIndexThreads} diff --git a/query-microservices/query-service/service/src/test/resources/log4j2-test.xml b/query-microservices/query-service/service/src/test/resources/log4j2-test.xml index 0600e42c..43cd58ff 100644 --- a/query-microservices/query-service/service/src/test/resources/log4j2-test.xml +++ b/query-microservices/query-service/service/src/test/resources/log4j2-test.xml @@ -10,8 +10,8 @@ - - + + \ No newline at end of file From 70f7a2ff19cbc2eddddf3f88efa4e58a3d66f152 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 30 Jul 2021 00:03:36 -0400 Subject: [PATCH 080/218] Added query service tests for next, cancel, close, and reset. --- .../query/QueryManagementService.java | 63 +- .../query/monitor/MonitorTask.java | 45 +- .../query/monitor/QueryMonitor.java | 6 +- .../query/status/QueryStatusUpdateHelper.java | 22 +- .../microservice/query/QueryServiceTest.java | 1645 ++++++++++++++++- .../src/test/resources/config/application.yml | 4 +- 6 files changed, 1681 insertions(+), 104 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index e75a4846..8f819a68 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -59,8 +59,6 @@ import java.io.IOException; import java.lang.reflect.Method; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; @@ -80,6 +78,11 @@ import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CLOSED; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CREATED; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.DEFINED; +import static datawave.webservice.query.QueryImpl.DN_LIST; +import static datawave.webservice.query.QueryImpl.QUERY_ID; +import static datawave.webservice.query.QueryImpl.USER_DN; + +// TODO: Make sure close is called automatically after a query finishes. I think this is more of a query metrics thing than anything. @Service public class QueryManagementService implements QueryRequestHandler { @@ -105,7 +108,6 @@ public class QueryManagementService implements QueryRequestHandler { private final QueryQueueManager queryQueueManager; private final AuditClient auditClient; private final ThreadPoolTaskExecutor nextCallExecutor; - private final String identifier; private final QueryStatusUpdateHelper queryStatusUpdateHelper; private final MultiValueMap nextCallMap = new LinkedMultiValueMap<>(); @@ -127,15 +129,6 @@ public QueryManagementService(QueryProperties queryProperties, ApplicationContex this.queryQueueManager = queryQueueManager; this.auditClient = auditClient; this.nextCallExecutor = nextCallExecutor; - - String hostname; - try { - hostname = InetAddress.getLocalHost().getHostName(); - } catch (UnknownHostException e) { - hostname = "UNKNOWN"; - } - this.identifier = hostname; - this.queryStatusUpdateHelper = new QueryStatusUpdateHelper(this.queryProperties, this.queryStorageCache); } @@ -551,7 +544,7 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th private BaseQueryResponse next(String queryId, Collection userRoles) throws InterruptedException, QueryException { // before we spin up a separate thread, make sure we are allowed to call next boolean success = false; - QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(queryId, queryStatusUpdateHelper::claimConcurrentNext); + QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(queryId, queryStatusUpdateHelper::claimNextCall); try { // publish a next event to the executor pool publishNextEvent(queryId, queryStatus.getQueryKey().getQueryPool()); @@ -585,7 +578,7 @@ private BaseQueryResponse next(String queryId, Collection userRoles) thr // after all of our work is done, perform our final query status update for this next call queryStatus = queryStatusUpdateHelper.lockedUpdate(queryId, status -> { - queryStatusUpdateHelper.releaseConcurrentNext(status); + queryStatusUpdateHelper.releaseNextCall(status, queryQueueManager); status.setLastPageNumber(status.getLastPageNumber() + 1); status.setNumResultsReturned(status.getNumResultsReturned() + resultsPage.getResults().size()); }); @@ -620,7 +613,7 @@ private BaseQueryResponse next(String queryId, Collection userRoles) thr } finally { // update query status if we failed if (!success) { - queryStatusUpdateHelper.lockedUpdate(queryId, queryStatusUpdateHelper::releaseConcurrentNext); + queryStatusUpdateHelper.lockedUpdate(queryId, status -> queryStatusUpdateHelper.releaseNextCall(status, queryQueueManager)); } } } @@ -757,10 +750,10 @@ private VoidResponse cancel(String queryId, ProxiedUserDetails currentUser, bool QueryStatus queryStatus = validateRequest(queryId, currentUser, adminOverride); // if the query is running, or if the query is closing and finishing up a next call - if (queryStatus.getQueryState() == CREATED || (queryStatus.getQueryState() == CLOSED && queryStatus.getConcurrentNextCount() > 0)) { + if (queryStatus.isRunning()) { cancel(queryId, true); } else { - throw new BadRequestQueryException(queryId + " cannot be canceled because it is not running.", HttpStatus.SC_BAD_REQUEST + "-1"); + throw new BadRequestQueryException("Cannot call cancel on a query that is not running", HttpStatus.SC_BAD_REQUEST + "-1"); } VoidResponse response = new VoidResponse(); @@ -808,6 +801,11 @@ public void cancel(String queryId, boolean publishEvent) throws InterruptedExcep status.setQueryState(CANCELED); }); + // delete the results queue + queryQueueManager.deleteQueue(queryId); + + // TODO: Delete the tasks as well? + QueryRequest cancelRequest = QueryRequest.cancel(queryId); // publish a cancel event to all of the query services @@ -951,7 +949,7 @@ private VoidResponse close(String queryId, ProxiedUserDetails currentUser, boole if (queryStatus.getQueryState() == CREATED) { close(queryId); } else { - throw new BadRequestQueryException(queryId + " cannot be closed because it is not running.", HttpStatus.SC_BAD_REQUEST + "-1"); + throw new BadRequestQueryException("Cannot call close on a query that is not running", HttpStatus.SC_BAD_REQUEST + "-1"); } VoidResponse response = new VoidResponse(); @@ -986,6 +984,18 @@ public void close(String queryId) throws InterruptedException, QueryException { status.setQueryState(CLOSED); }); + // if the query has no active next calls, delete the results queue + if (queryStatus.getActiveNextCalls() == 0) { + queryQueueManager.deleteQueue(queryId); + } + + // TODO: If we are running a next call, don't delete tasks, task states, or delete query queues + // TODO: If we aren't running a next call, do what we would do for a cancel + + // TODO: update tasks, task states, and delete query queues + // TODO: Tasks should be + // TODO: Figure out when to delete the queue for a close. After the last page is returned? + // publish a close event to the executor pool publishExecutorEvent(QueryRequest.close(queryId), queryStatus.getQueryKey().getQueryPool()); } @@ -1033,7 +1043,7 @@ public GenericResponse reset(String queryId, ProxiedUserDetails currentU QueryStatus queryStatus = validateRequest(queryId, currentUser); // cancel the query if it is running - if (queryStatus.getQueryState() == CREATED) { + if (queryStatus.isRunning()) { cancel(queryStatus.getQueryKey().getQueryId(), true); } @@ -1119,7 +1129,7 @@ public VoidResponse adminRemoveAll(ProxiedUserDetails currentUser) throws QueryE log.info("Remove All called by admin: " + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); try { List queryStatuses = queryStorageCache.getQueryStatus(); - queryStatuses.removeIf(s -> s.getQueryState() == CREATED); + queryStatuses.removeIf(QueryStatus::isRunning); VoidResponse response = new VoidResponse(); for (QueryStatus queryStatus : queryStatuses) { @@ -1251,7 +1261,7 @@ public GenericResponse update(String queryId, MultiValueMap ignoredParams = new ArrayList<>(parameters.keySet()); List safeParams = new ArrayList<>(queryProperties.getUpdatableParams()); @@ -1394,11 +1404,14 @@ public GenericResponse duplicate(String queryId, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { - // TODO: Are we losing anything by recreating the parameters this way? // recreate the query parameters MultiValueMap currentParams = new LinkedMultiValueMap<>(); - currentParams.addAll(queryStatus.getQuery().getOptionalQueryParameters()); - queryStatus.getQuery().getParameters().forEach(x -> currentParams.add(x.getParameterName(), x.getParameterValue())); + currentParams.addAll(queryStatus.getQuery().toMap()); + + // remove some of the copied params + currentParams.remove(QUERY_ID); + currentParams.remove(USER_DN); + currentParams.remove(DN_LIST); // updated all of the passed in parameters updateParameters(parameters, currentParams); @@ -1810,7 +1823,7 @@ protected QueryLogic validateQuery(String queryLogicName, MultiValueMap parameters) throws BadRequestQueryException { // add query logic name to parameters - parameters.add(QUERY_LOGIC_NAME, queryLogicName); + parameters.set(QUERY_LOGIC_NAME, queryLogicName); log.debug(writeValueAsString(parameters)); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java index 5d113fa4..983060f2 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java @@ -5,6 +5,7 @@ import datawave.microservice.query.monitor.cache.MonitorStatus; import datawave.microservice.query.monitor.cache.MonitorStatusCache; import datawave.microservice.query.monitor.config.MonitorProperties; +import datawave.microservice.query.storage.QueryQueueManager; import datawave.microservice.query.storage.QueryStatus; import datawave.microservice.query.storage.QueryStorageCache; import datawave.webservice.query.exception.QueryException; @@ -14,9 +15,6 @@ import java.io.IOException; import java.util.concurrent.Callable; -import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CLOSED; -import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CREATED; - public class MonitorTask implements Callable { private final Logger log = LoggerFactory.getLogger(this.getClass()); @@ -24,14 +22,16 @@ public class MonitorTask implements Callable { private final QueryExpirationProperties expirationProperties; private final MonitorStatusCache monitorStatusCache; private final QueryStorageCache queryStorageCache; + private final QueryQueueManager queryQueueManager; private final QueryManagementService queryManagementService; public MonitorTask(MonitorProperties monitorProperties, QueryExpirationProperties expirationProperties, MonitorStatusCache monitorStatusCache, - QueryStorageCache queryStorageCache, QueryManagementService queryManagementService) { + QueryStorageCache queryStorageCache, QueryQueueManager queryQueueManager, QueryManagementService queryManagementService) { this.monitorProperties = monitorProperties; this.expirationProperties = expirationProperties; this.monitorStatusCache = monitorStatusCache; this.queryStorageCache = queryStorageCache; + this.queryQueueManager = queryQueueManager; this.queryManagementService = queryManagementService; } @@ -64,20 +64,31 @@ public Void call() throws Exception { // 3) Are there any other conditions that we should check for? private void monitor(long currentTimeMillis) { for (QueryStatus status : queryStorageCache.getQueryStatus()) { - String queryId = status.getQueryKey().getQueryId().toString(); + String queryId = status.getQueryKey().getQueryId(); - // if the query is not running, and has been inactive for too long, evict it - if (status.getQueryState() != CREATED && status.isInactive(currentTimeMillis, monitorProperties.getInactiveQueryTimeToLiveMillis())) { - deleteQuery(queryId); - } - // if the query is running, or closed and in the process of finishing up it's last next call, but it isn't making progress, poke it - else if ((status.getQueryState() == CREATED || (status.getQueryState() == CLOSED && status.getConcurrentNextCount() > 0) - && status.isProgressIdle(currentTimeMillis, expirationProperties.getProgressTimeoutMillis()))) { - defibrillateQuery(queryId, status.getQueryKey().getQueryPool()); + // if the query is not running + if (!status.isRunning()) { + + // if the query has been inactive too long (i.e. no interaction from the user or software) + if (status.isInactive(currentTimeMillis, monitorProperties.getInactiveQueryTimeToLiveMillis())) { + deleteQuery(queryId); + } + // delete the results queue if it exists + else { + // TODO: add in a check to see if the queue exists first + queryQueueManager.deleteQueue(queryId); + } } - // if the query is running, but the user hasn't interacted with it in a while, cancel it - else if (status.getQueryState() == CREATED && status.isUserIdle(currentTimeMillis, expirationProperties.getIdleTimeoutMillis())) { - cancelQuery(queryId); + // if the query is running + else { + // if the query isn't making progress + if (status.isProgressIdle(currentTimeMillis, expirationProperties.getProgressTimeoutMillis())) { + defibrillateQuery(queryId, status.getQueryKey().getQueryPool()); + } + // if the user hasn't interacted with the query + else if (status.isUserIdle(currentTimeMillis, expirationProperties.getIdleTimeoutMillis())) { + cancelQuery(queryId); + } } } } @@ -99,6 +110,8 @@ private void defibrillateQuery(String queryId, String queryPool) { private void deleteQuery(String queryId) { try { + // deletes everything for a query + // the result queue, the query status, the tasks, the task states queryStorageCache.deleteQuery(queryId); } catch (IOException e) { log.error("Encountered error while trying to evict inactive query: " + queryId, e); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java index b969a364..b283fe79 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java @@ -5,6 +5,7 @@ import datawave.microservice.query.config.QueryProperties; import datawave.microservice.query.monitor.cache.MonitorStatusCache; import datawave.microservice.query.monitor.config.MonitorProperties; +import datawave.microservice.query.storage.QueryQueueManager; import datawave.microservice.query.storage.QueryStorageCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,6 +27,7 @@ public class QueryMonitor { private final QueryExpirationProperties expirationProperties; private final MonitorStatusCache monitorStatusCache; private final QueryStorageCache queryStorageCache; + private final QueryQueueManager queryQueueManager; private final QueryManagementService queryManagementService; private final ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -33,11 +35,12 @@ public class QueryMonitor { private Future taskFuture; public QueryMonitor(MonitorProperties monitorProperties, QueryProperties queryProperties, MonitorStatusCache monitorStatusCache, - QueryStorageCache queryStorageCache, QueryManagementService queryManagementService) { + QueryStorageCache queryStorageCache, QueryQueueManager queryQueueManager, QueryManagementService queryManagementService) { this.monitorProperties = monitorProperties; this.expirationProperties = queryProperties.getExpiration(); this.monitorStatusCache = monitorStatusCache; this.queryStorageCache = queryStorageCache; + this.queryQueueManager = queryQueueManager; this.queryManagementService = queryManagementService; } @@ -71,6 +74,7 @@ public void monitorTaskScheduler() { expirationProperties, monitorStatusCache, queryStorageCache, + queryQueueManager, queryManagementService)); // @formatter:on } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java index 570b7855..1a19d91f 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java @@ -1,6 +1,7 @@ package datawave.microservice.query.status; import datawave.microservice.query.config.QueryProperties; +import datawave.microservice.query.storage.QueryQueueManager; import datawave.microservice.query.storage.QueryStatus; import datawave.microservice.query.storage.QueryStorageCache; import datawave.microservice.query.storage.QueryStorageLock; @@ -20,12 +21,12 @@ public QueryStatusUpdateHelper(QueryProperties queryProperties, QueryStorageCach this.queryStorageCache = queryStorageCache; } - public void claimConcurrentNext(QueryStatus queryStatus) throws QueryException { + public void claimNextCall(QueryStatus queryStatus) throws QueryException { // we can only call next on a created query if (queryStatus.getQueryState() == CREATED) { // increment the concurrent next count - if (queryStatus.getConcurrentNextCount() < queryProperties.getNextCall().getConcurrency()) { - queryStatus.setConcurrentNextCount(queryStatus.getConcurrentNextCount() + 1); + if (queryStatus.getActiveNextCalls() < queryProperties.getNextCall().getConcurrency()) { + queryStatus.setActiveNextCalls(queryStatus.getActiveNextCalls() + 1); // update the last used datetime for the query queryStatus.setLastUsedMillis(System.currentTimeMillis()); @@ -38,16 +39,21 @@ public void claimConcurrentNext(QueryStatus queryStatus) throws QueryException { } } - public void releaseConcurrentNext(QueryStatus queryStatus) throws QueryException { + public void releaseNextCall(QueryStatus queryStatus, QueryQueueManager queryQueueManager) throws QueryException { // decrement the concurrent next count - if (queryStatus.getConcurrentNextCount() > 0) { - queryStatus.setConcurrentNextCount(queryStatus.getConcurrentNextCount() - 1); + if (queryStatus.getActiveNextCalls() > 0) { + queryStatus.setActiveNextCalls(queryStatus.getActiveNextCalls() - 1); // update the last used datetime for the query queryStatus.setLastUsedMillis(System.currentTimeMillis()); + + // TODO: We should add a 'queueExists' call to determine whether this needs to be run + // if by the end of the call the query is no longer running, delete the results queue + if (!queryStatus.isRunning()) { + queryQueueManager.deleteQueue(queryStatus.getQueryKey().getQueryId()); + } } else { - throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, - "Concurrent next count can't be decremented: " + queryStatus.getConcurrentNextCount()); + throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Concurrent next count can't be decremented: " + queryStatus.getActiveNextCalls()); } } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java index a80e4b0a..e72df617 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java @@ -75,6 +75,8 @@ import java.util.UUID; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import static datawave.microservice.query.QueryParameters.QUERY_MAX_CONCURRENT_TASKS; @@ -203,6 +205,9 @@ public void testDefineSuccess() throws ParseException, IOException { // verify that no audit message was sent assertAuditNotSent(); + // verify that the results queue wasn't created + Assert.assertFalse(queryQueueManager.queueExists(queryId)); + // verify that query tasks weren't created assertTasksNotCreated(queryId); } @@ -580,6 +585,9 @@ public void testCreateSuccess() throws ParseException, IOException { // verify that an audit message was sent and the the audit id matches the query id assertAuditSent(queryId); + // verify that the results queue was created + Assert.assertTrue(queryQueueManager.queueExists(queryId)); + // verify that query tasks were created assertTasksCreated(queryId); } @@ -1002,8 +1010,13 @@ public void testNextSuccess() throws Exception { // @formatter:on // verify that the next event was published - Assert.assertEquals(1, queryRequestEvents.size()); + Assert.assertEquals(2, queryRequestEvents.size()); // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); assertQueryRequestEvent( "executor-unassigned:**", QueryRequest.Method.NEXT, @@ -1028,6 +1041,16 @@ public void testNextSuccess_multiplePages() throws Exception { fieldValues.add("LOKI", "ALLIGATOR"); fieldValues.add("LOKI", "CLASSIC"); + // verify that the create event was published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:off + for (int page = 1; page <= 2; page++) { // TODO: We have to generate the results in between next calls because the test queue manager does not handle requeueing of unused messages :( // @formatter:off @@ -1166,8 +1189,13 @@ public void testNextSuccess_cancelPartialResults() throws Exception { // @formatter:on // verify that the next events were published - Assert.assertEquals(3, queryRequestEvents.size()); + Assert.assertEquals(4, queryRequestEvents.size()); // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); assertQueryRequestEvent( "executor-unassigned:**", QueryRequest.Method.NEXT, @@ -1202,6 +1230,16 @@ public void testNextSuccess_maxResults() throws Exception { fieldValues.add("LOKI", "ALLIGATOR"); fieldValues.add("LOKI", "CLASSIC"); + // verify that the create event was published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + for (int page = 1; page <= 3; page++) { // TODO: We have to generate the results in between next calls because the test queue manager does not handle requeueing of unused messages :( // @formatter:off @@ -1274,8 +1312,13 @@ public void testNextSuccess_maxResults() throws Exception { Assert.assertEquals(404, response.getStatusCodeValue()); - assertQueryException("No results found for query. " + queryId, "Exception with no cause caught", "404-4", - Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:off + assertQueryException( + "No results found for query. " + queryId, + "Exception with no cause caught", + "404-4", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on } @DirtiesContext @@ -1299,12 +1342,22 @@ public void testNextSuccess_noResults() throws Exception { Assert.assertEquals(404, response.getStatusCodeValue()); - assertQueryException("No results found for query. " + queryId, "Exception with no cause caught", "404-4", - Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:off + assertQueryException( + "No results found for query. " + queryId, + "Exception with no cause caught", + "404-4", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on // verify that the next event was published - Assert.assertEquals(1, queryRequestEvents.size()); + Assert.assertEquals(2, queryRequestEvents.size()); // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); assertQueryRequestEvent( "executor-unassigned:**", QueryRequest.Method.NEXT, @@ -1327,8 +1380,13 @@ public void testNextFailure_queryNotFound() throws Exception { Assert.assertEquals(404, response.getStatusCodeValue()); - assertQueryException("No query object matches this id. " + queryId, "Exception with no cause caught", "404-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on // verify that no events were published Assert.assertEquals(0, queryRequestEvents.size()); @@ -1358,12 +1416,22 @@ public void testNextFailure_queryNotRunning() throws Exception { Assert.assertEquals(400, response.getStatusCodeValue()); - assertQueryException("Cannot call next on a query that is not running", "Exception with no cause caught", "400-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:off + assertQueryException( + "Cannot call next on a query that is not running", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on // verify that the next events were published - Assert.assertEquals(2, queryRequestEvents.size()); + Assert.assertEquals(3, queryRequestEvents.size()); // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); assertQueryRequestEvent( "/query:**", QueryRequest.Method.CANCEL, @@ -1394,11 +1462,23 @@ public void testNextFailure_ownershipFailure() throws Exception { Assert.assertEquals(401, response.getStatusCodeValue()); - assertQueryException("Current user does not match user that defined query. altuserdn != userdn", "Exception with no cause caught", "401-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on // verify that the next events were published - Assert.assertEquals(0, queryRequestEvents.size()); + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on } @DirtiesContext @@ -1417,12 +1497,22 @@ public void testNextFailure_timeout() throws Exception { Assert.assertEquals(500, response.getStatusCodeValue()); - assertQueryException("Query timed out. " + queryId + " timed out.", "Exception with no cause caught", "500-27", - Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:off + assertQueryException( + "Query timed out. " + queryId + " timed out.", + "Exception with no cause caught", + "500-27", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on // verify that the next events were published - Assert.assertEquals(1, queryRequestEvents.size()); + Assert.assertEquals(2, queryRequestEvents.size()); // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); assertQueryRequestEvent( "executor-unassigned:**", QueryRequest.Method.NEXT, @@ -1431,51 +1521,1502 @@ public void testNextFailure_timeout() throws Exception { // @formatter:on } - private void publishEventsToQueue(String queryId, int numEvents, MultiValueMap fieldValues, String visibility) throws Exception { - for (int resultId = 0; resultId < numEvents; resultId++) { - DefaultEvent[] events = new DefaultEvent[1]; - events[0] = new DefaultEvent(); - long currentTime = System.currentTimeMillis(); - List fields = new ArrayList<>(); - for (Map.Entry> entry : fieldValues.entrySet()) { - for (String value : entry.getValue()) { - fields.add(new DefaultField(entry.getKey(), visibility, currentTime, value)); - } - } - events[0].setFields(fields); - queryQueueManager.sendMessage(queryId, new Result(Integer.toString(resultId), events)); - } + @DirtiesContext + @Test + public void testNextFailure_nextOnDefined() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + // make the next call + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Cannot call next on a query that is not running", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); } - private String createQuery(ProxiedUserDetails authUser, MultiValueMap map) { - UriComponents uri = createUri("EventQuery/create"); + @DirtiesContext + @Test + public void testNextFailure_nextOnClosed() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); - // not testing audit with this method - auditIgnoreSetup(); + // create a valid query + String queryId = createQuery(authUser, createParams()); - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + // close the query + Future> closeFuture = closeQuery(authUser, queryId); - // remove the create event - queryRequestEvents.clear(); + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); - return (String) resp.getBody().getResult(); + Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Cannot call next on a query that is not running", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assert.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on } - private Future> nextQuery(ProxiedUserDetails authUser, String queryId) { - UriComponents uri = createUri(queryId + "/next"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + @DirtiesContext + @Test + public void testNextFailure_nextOnCanceled() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); // make the next call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, BaseResponse.class)); + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Cannot call next on a query that is not running", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that the cancel event was published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on } - private Future> cancelQuery(ProxiedUserDetails authUser, String queryId) { - UriComponents uri = createUri(queryId + "/cancel"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + @DirtiesContext + @Test + public void testCancelSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); - // make the next call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCELED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the result queue is gone + Assert.assertFalse(queryQueueManager.queueExists(queryId)); + + // verify that the query tasks are still present + assertTasksCreated(queryId); + + // verify that the cancel event was published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testCancelSuccess_activeNextCall() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // call next on the query + Future> nextFuture = nextQuery(authUser, queryId); + + try { + nextFuture.get(1, TimeUnit.SECONDS); + } catch (TimeoutException e) { + // do nothing + } + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCELED, + 0, + 0, + 1, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the result queue is gone + Assert.assertFalse(queryQueueManager.queueExists(queryId)); + + // wait for the next call to return + nextFuture.get(); + + // verify that the query tasks are still present + assertTasksCreated(queryId); + + // verify that the close event was published + Assert.assertEquals(4, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testCancelFailure_queryNotFound() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(404, cancelResponse.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(cancelResponse.getBody().getExceptions())); + // @formatter:on + + } + + @DirtiesContext + @Test + public void testCancelFailure_ownershipFailure() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails altAuthUser = createAltUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // make the cancel call as an alternate user asynchronously + Future> future = cancelQuery(altAuthUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(401, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that the next events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testCancelFailure_queryNotRunning() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // try to cancel the query again + cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + cancelResponse = cancelFuture.get(); + + Assert.assertEquals(400, cancelResponse.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Cannot call cancel on a query that is not running", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(cancelResponse.getBody().getExceptions())); + // @formatter:on + + // verify that the next events were published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testAdminCancelSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails adminUser = createAltUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // cancel the query as the admin user + Future> cancelFuture = adminCancelQuery(adminUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCELED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the result queue is gone + Assert.assertFalse(queryQueueManager.queueExists(queryId)); + + // verify that the query tasks are still present + assertTasksCreated(queryId); + + // verify that the cancel event was published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testAdminCancelFailure_notAdminUser() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + UriComponents uri = createUri(queryId + "/adminCancel"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // close the query + Future> closeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(403, closeResponse.getStatusCodeValue()); + + // verify that the create event was published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testCancelAllSuccess() throws Exception { + ProxiedUserDetails adminUser = createUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + // create a bunch of queries + List queryIds = new ArrayList<>(); + long currentTimeMillis = System.currentTimeMillis(); + for (int i = 0; i < 10; i++) { + String queryId = createQuery(adminUser, createParams()); + mockServer.reset(); + + queryIds.add(queryId); + + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + // close all queries as the admin user + Future> cancelFuture = adminCancelAllQueries(adminUser); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // verify that query status was created correctly + List queryStatusList = queryStorageCache.getQueryStatus(); + + for (QueryStatus queryStatus : queryStatusList) { + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCELED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + String queryId = queryStatus.getQueryKey().getQueryId(); + + // verify that the result queue is gone + Assert.assertFalse(queryQueueManager.queueExists(queryStatus.getQueryKey().getQueryId())); + + // verify that the query tasks are still present + assertTasksCreated(queryStatus.getQueryKey().getQueryId()); + + // @formatter:off + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + // verify that there are no more events + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @DirtiesContext + @Test + public void testCloseSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CLOSED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the result queue is gone + Assert.assertFalse(queryQueueManager.queueExists(queryId)); + + // verify that the query tasks are still present + assertTasksCreated(queryId); + + // verify that the close event was published + Assert.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testCloseSuccess_activeNextCall() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // call next on the query + Future> nextFuture = nextQuery(authUser, queryId); + + try { + nextFuture.get(1, TimeUnit.SECONDS); + } catch (TimeoutException e) { + // do nothing + } + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CLOSED, + 0, + 0, + 1, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the result queue is still present + Assert.assertTrue(queryQueueManager.queueExists(queryId)); + + // send enough results to return a page + // pump enough results into the queue to trigger a complete page + int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("LOKI", "ALLIGATOR"); + fieldValues.add("LOKI", "CLASSIC"); + + // @formatter:off + publishEventsToQueue( + queryId, + pageSize, + fieldValues, + "ALL"); + // @formatter:on + + // wait for the next call to return + nextFuture.get(); + + // verify that the result queue is now gone + Assert.assertFalse(queryQueueManager.queueExists(queryId)); + + // verify that the query tasks are still present + assertTasksCreated(queryId); + + // verify that the close event was published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testCloseFailure_queryNotFound() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(404, closeResponse.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(closeResponse.getBody().getExceptions())); + // @formatter:on + + } + + @DirtiesContext + @Test + public void testCloseFailure_ownershipFailure() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails altAuthUser = createAltUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // make the close call as an alternate user asynchronously + Future> future = closeQuery(altAuthUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(401, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that the next events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testCloseFailure_queryNotRunning() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + + // try to close the query again + closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + closeResponse = closeFuture.get(); + + Assert.assertEquals(400, closeResponse.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Cannot call close on a query that is not running", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(closeResponse.getBody().getExceptions())); + // @formatter:on + + // verify that the next events were published + Assert.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testAdminCloseSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails adminUser = createUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // close the query as the admin user + Future> closeFuture = closeQuery(adminUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CLOSED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the result queue is gone + Assert.assertFalse(queryQueueManager.queueExists(queryId)); + + // verify that the query tasks are still present + assertTasksCreated(queryId); + + // verify that the close event was published + Assert.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testAdminCloseFailure_notAdminUser() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + UriComponents uri = createUri(queryId + "/adminClose"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // close the query + Future> closeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(403, closeResponse.getStatusCodeValue()); + + // verify that the create event was published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testCloseAllSuccess() throws Exception { + ProxiedUserDetails adminUser = createUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + // create a bunch of queries + long currentTimeMillis = System.currentTimeMillis(); + for (int i = 0; i < 10; i++) { + String queryId = createQuery(adminUser, createParams()); + mockServer.reset(); + + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + // close all queries as the admin user + Future> closeFuture = adminCloseAllQueries(adminUser); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + List queryStatusList = queryStorageCache.getQueryStatus(); + + for (QueryStatus queryStatus : queryStatusList) { + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CLOSED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + String queryId = queryStatus.getQueryKey().getQueryId(); + + // verify that the result queue is gone + Assert.assertFalse(queryQueueManager.queueExists(queryStatus.getQueryKey().getQueryId())); + + // verify that the query tasks are still present + assertTasksCreated(queryStatus.getQueryKey().getQueryId()); + + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + // verify that there are no more events + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @DirtiesContext + @Test + public void testResetSuccess_resetOnDefined() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = defineQuery(authUser, createParams()); + + mockServer.reset(); + auditSentSetup(); + + // reset the query + Future> resetFuture = resetQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // @formatter:off + assertGenericResponse( + true, + HttpStatus.Series.SUCCESSFUL, + response); + // @formatter:on + + String resetQueryId = (String) response.getBody().getResult(); + + // verify that a new query id was created + Assert.assertNotEquals(queryId, resetQueryId); + + // verify that an audit record was sent + assertAuditSent(resetQueryId); + + // verify that original query was canceled + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.DEFINED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that new query was created + QueryStatus resetQueryStatus = queryStorageCache.getQueryStatus(resetQueryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + resetQueryStatus); + // @formatter:on + + // make sure the queries are equal (ignoring the query id) + queryStatus.getQuery().setId(resetQueryStatus.getQuery().getId()); + Assert.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); + + // verify that events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + resetQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testResetSuccess_resetOnCreated() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + mockServer.reset(); + auditSentSetup(); + + // reset the query + Future> resetFuture = resetQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // @formatter:off + assertGenericResponse( + true, + HttpStatus.Series.SUCCESSFUL, + response); + // @formatter:on + + String resetQueryId = (String) response.getBody().getResult(); + + // verify that a new query id was created + Assert.assertNotEquals(queryId, resetQueryId); + + // verify that an audit record was sent + assertAuditSent(resetQueryId); + + // verify that original query was canceled + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCELED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that new query was created + QueryStatus resetQueryStatus = queryStorageCache.getQueryStatus(resetQueryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + resetQueryStatus); + // @formatter:on + + // make sure the queries are equal (ignoring the query id) + queryStatus.getQuery().setId(resetQueryStatus.getQuery().getId()); + Assert.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); + + // verify that events were published + Assert.assertEquals(4, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + resetQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testResetSuccess_resetOnClosed() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + + mockServer.reset(); + auditSentSetup(); + + // reset the query + Future> resetFuture = resetQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // @formatter:off + assertGenericResponse( + true, + HttpStatus.Series.SUCCESSFUL, + response); + // @formatter:on + + String resetQueryId = (String) response.getBody().getResult(); + + // verify that a new query id was created + Assert.assertNotEquals(queryId, resetQueryId); + + // verify that an audit record was sent + assertAuditSent(resetQueryId); + + // verify that original query was closed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CLOSED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that new query was created + QueryStatus resetQueryStatus = queryStorageCache.getQueryStatus(resetQueryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + resetQueryStatus); + // @formatter:on + + // make sure the queries are equal (ignoring the query id) + queryStatus.getQuery().setId(resetQueryStatus.getQuery().getId()); + Assert.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); + + // verify that events were published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + resetQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testResetSuccess_resetOnCanceled() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + mockServer.reset(); + auditSentSetup(); + + // reset the query + Future> resetFuture = resetQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // @formatter:off + assertGenericResponse( + true, + HttpStatus.Series.SUCCESSFUL, + response); + // @formatter:on + + String resetQueryId = (String) response.getBody().getResult(); + + // verify that a new query id was created + Assert.assertNotEquals(queryId, resetQueryId); + + // verify that an audit record was sent + assertAuditSent(resetQueryId); + + // verify that original query was canceled + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCELED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that new query was created + QueryStatus resetQueryStatus = queryStorageCache.getQueryStatus(resetQueryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + resetQueryStatus); + // @formatter:on + + // make sure the queries are equal (ignoring the query id) + queryStatus.getQuery().setId(resetQueryStatus.getQuery().getId()); + Assert.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); + + // verify that events were published + Assert.assertEquals(4, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + resetQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testResetFailure_queryNotFound() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + auditNotSentSetup(); + + // reset the query + UriComponents uri = createUri(queryId + "/reset"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // close the query + Future> resetFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, BaseResponse.class)); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assert.assertEquals(404, response.getStatusCodeValue()); + + // make sure no audits were sent + assertAuditNotSent(); + + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @DirtiesContext + @Test + public void testResetFailure_ownershipFailure() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails altAuthUser = createAltUserDetails(); + + // define a valid query + String queryId = createQuery(authUser, createParams()); + + mockServer.reset(); + auditNotSentSetup(); + + // reset the query + UriComponents uri = createUri(queryId + "/reset"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(altAuthUser, null, null, HttpMethod.PUT, uri); + + // close the query + Future> resetFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, BaseResponse.class)); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assert.assertEquals(401, response.getStatusCodeValue()); + + // make sure no audits were sent + assertAuditNotSent(); + + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + // remove + + // admin remove + + // update + + // duplicate + + // list + + // admin list + + // get query + + // remove all + + // list query logic + + private void publishEventsToQueue(String queryId, int numEvents, MultiValueMap fieldValues, String visibility) throws Exception { + for (int resultId = 0; resultId < numEvents; resultId++) { + DefaultEvent[] events = new DefaultEvent[1]; + events[0] = new DefaultEvent(); + long currentTime = System.currentTimeMillis(); + List fields = new ArrayList<>(); + for (Map.Entry> entry : fieldValues.entrySet()) { + for (String value : entry.getValue()) { + fields.add(new DefaultField(entry.getKey(), visibility, currentTime, value)); + } + } + events[0].setFields(fields); + queryQueueManager.sendMessage(queryId, new Result(Integer.toString(resultId), events)); + } + } + + private String createQuery(ProxiedUserDetails authUser, MultiValueMap map) { + return newQuery(authUser, map, "create"); + } + + private String defineQuery(ProxiedUserDetails authUser, MultiValueMap map) { + return newQuery(authUser, map, "define"); + } + + private String newQuery(ProxiedUserDetails authUser, MultiValueMap map, String createOrDefine) { + UriComponents uri = createUri("EventQuery/" + createOrDefine); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + return (String) resp.getBody().getResult(); + } + + private Future> nextQuery(ProxiedUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId + "/next"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, BaseResponse.class)); + } + + private Future> adminCloseQuery(ProxiedUserDetails authUser, String queryId) { + return stopQuery(authUser, queryId, "adminClose"); + } + + private Future> closeQuery(ProxiedUserDetails authUser, String queryId) { + return stopQuery(authUser, queryId, "close"); + } + + private Future> adminCancelQuery(ProxiedUserDetails authUser, String queryId) { + return stopQuery(authUser, queryId, "adminCancel"); + } + + private Future> cancelQuery(ProxiedUserDetails authUser, String queryId) { + return stopQuery(authUser, queryId, "cancel"); + } + + private Future> stopQuery(ProxiedUserDetails authUser, String queryId, String closeOrCancel) { + UriComponents uri = createUri(queryId + "/" + closeOrCancel); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + private Future> adminCloseAllQueries(ProxiedUserDetails authUser) { + UriComponents uri = createUri("/adminCloseAll"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + private Future> adminCancelAllQueries(ProxiedUserDetails authUser) { + UriComponents uri = createUri("/adminCancelAll"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + private Future> resetQuery(ProxiedUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId + "/reset"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, GenericResponse.class)); } private ProxiedUserDetails createUserDetails() { @@ -1541,12 +3082,12 @@ private void assertQueryRequestEvent(String destination, QueryRequest.Method met Assert.assertEquals(method, queryRequestEvent.getRequest().getMethod()); } - private void assertQueryStatus(QueryStatus.QUERY_STATE queryState, long numResultsReturned, long numResultsGenerated, long concurrentNextCount, + private void assertQueryStatus(QueryStatus.QUERY_STATE queryState, long numResultsReturned, long numResultsGenerated, long activeNextCalls, long lastPageNumber, long lastCallTimeMillis, QueryStatus queryStatus) { Assert.assertEquals(queryState, queryStatus.getQueryState()); Assert.assertEquals(numResultsReturned, queryStatus.getNumResultsReturned()); Assert.assertEquals(numResultsGenerated, queryStatus.getNumResultsGenerated()); - Assert.assertEquals(concurrentNextCount, queryStatus.getConcurrentNextCount()); + Assert.assertEquals(activeNextCalls, queryStatus.getActiveNextCalls()); Assert.assertEquals(lastPageNumber, queryStatus.getLastPageNumber()); Assert.assertTrue(queryStatus.getLastUsedMillis() > lastCallTimeMillis); Assert.assertTrue(queryStatus.getLastUpdatedMillis() > lastCallTimeMillis); diff --git a/query-microservices/query-service/service/src/test/resources/config/application.yml b/query-microservices/query-service/service/src/test/resources/config/application.yml index 8e595937..2183202b 100644 --- a/query-microservices/query-service/service/src/test/resources/config/application.yml +++ b/query-microservices/query-service/service/src/test/resources/config/application.yml @@ -43,8 +43,8 @@ management: logging: level: - root: WARN - datawave.microservice.query: WARN + root: FATAL + datawave.microservice.query: FATAL warehouse: accumulo: From bf2ff0e1854db4e27f133d3a6aebb55f0ca35dda Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Sat, 31 Jul 2021 13:07:05 -0400 Subject: [PATCH 081/218] Added additional query service tests. --- .../microservice/query/QueryController.java | 8 +- .../query/QueryManagementService.java | 68 +- .../microservice/query/QueryServiceTest.java | 2189 ++++++++++++++++- .../resources/MyTestQueryLogicFactory.xml | 4 + 4 files changed, 2121 insertions(+), 148 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index efb6c213..0f219726 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -142,18 +142,18 @@ public GenericResponse duplicate(@PathVariable String queryId, @RequestP @Timed(name = "dw.query.list", absolute = true) @RequestMapping(path = "list", method = {RequestMethod.GET}, produces = {"text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public QueryImplListResponse list(@RequestParam(required = false) String id, @RequestParam(required = false) String queryName, + public QueryImplListResponse list(@RequestParam(required = false) String queryId, @RequestParam(required = false) String queryName, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { - return queryManagementService.list(id, queryName, currentUser); + return queryManagementService.list(queryId, queryName, currentUser); } @Timed(name = "dw.query.adminList", absolute = true) @RolesAllowed({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "adminList", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public QueryImplListResponse adminList(@RequestParam(required = false) String id, @RequestParam(required = false) String user, + public QueryImplListResponse adminList(@RequestParam(required = false) String queryId, @RequestParam(required = false) String user, @RequestParam(required = false) String queryName, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { - return queryManagementService.adminList(id, queryName, user, currentUser); + return queryManagementService.adminList(queryId, queryName, user, currentUser); } @Timed(name = "dw.query.get", absolute = true) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 8f819a68..8a531095 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -1133,7 +1133,9 @@ public VoidResponse adminRemoveAll(ProxiedUserDetails currentUser) throws QueryE VoidResponse response = new VoidResponse(); for (QueryStatus queryStatus : queryStatuses) { - response.addMessage(queryStatus.getQueryKey().getQueryId() + " removed."); + if (remove(queryStatus)) { + response.addMessage(queryStatus.getQueryKey().getQueryId() + " removed."); + } } return response; } catch (Exception e) { @@ -1171,7 +1173,7 @@ private VoidResponse remove(String queryId, ProxiedUserDetails currentUser, bool QueryStatus queryStatus = validateRequest(queryId, currentUser, adminOverride); // remove the query if it is not running - if (queryStatus.getQueryState() != CREATED) { + if (!queryStatus.isRunning()) { if (!remove(queryStatus)) { throw new QueryException("Failed to remove " + queryId); } @@ -1246,11 +1248,14 @@ public GenericResponse update(String queryId, MultiValueMap response = new GenericResponse<>(); if (!parameters.isEmpty()) { - // TODO: Are we losing anything by recreating the parameters this way? // recreate the query parameters MultiValueMap currentParams = new LinkedMultiValueMap<>(); - currentParams.addAll(queryStatus.getQuery().getOptionalQueryParameters()); - queryStatus.getQuery().getParameters().forEach(x -> currentParams.add(x.getParameterName(), x.getParameterValue())); + currentParams.addAll(queryStatus.getQuery().toMap()); + + // remove some of the copied params + currentParams.remove(QUERY_ID); + currentParams.remove(USER_DN); + currentParams.remove(DN_LIST); boolean updated; if (queryStatus.getQueryState() == DEFINED) { @@ -1259,33 +1264,34 @@ public GenericResponse update(String queryId, MultiValueMap ignoredParams = new ArrayList<>(parameters.keySet()); + List unsafeParams = new ArrayList<>(parameters.keySet()); List safeParams = new ArrayList<>(queryProperties.getUpdatableParams()); safeParams.retainAll(parameters.keySet()); - ignoredParams.removeAll(safeParams); - - // only update the safe parameters if the query is running - updated = updateParameters(safeParams, parameters, currentParams); + unsafeParams.removeAll(safeParams); - if (updated) { - // validate the update - String queryLogicName = queryStatus.getQuery().getQueryLogicName(); - validateQuery(queryLogicName, parameters, currentUser); - - // create a new query object - String userDn = currentUser.getPrimaryUser().getDn().subjectDN(); - Query query = createQuery(queryLogicName, parameters, userDn, currentUser.getDNs(), queryId); + // only update a running query if the params are all safe + if (unsafeParams.isEmpty()) { + updated = updateParameters(safeParams, parameters, currentParams); - // save the new query object in the cache - queryStatusUpdateHelper.lockedUpdate(queryId, status -> status.setQuery(query)); - } - - if (!ignoredParams.isEmpty()) { - response.addMessage("The following parameters cannot be updated for a running query: " + String.join(",", ignoredParams)); + if (updated) { + // validate the update + String queryLogicName = currentParams.getFirst(QUERY_LOGIC_NAME); + validateQuery(queryLogicName, currentParams, currentUser); + + // create a new query object + String userDn = currentUser.getPrimaryUser().getDn().subjectDN(); + Query query = createQuery(queryLogicName, currentParams, userDn, currentUser.getDNs(), queryId); + + // save the new query object in the cache + queryStatusUpdateHelper.lockedUpdate(queryId, status -> status.setQuery(query)); + } + } else { + throw new BadRequestQueryException("Cannot update the following parameters for a running query: " + String.join(", ", unsafeParams), + HttpStatus.SC_BAD_REQUEST + "-1"); } } else { throw new BadRequestQueryException("Cannot update a query unless it is defined or running.", HttpStatus.SC_BAD_REQUEST + "-1"); @@ -1417,11 +1423,11 @@ private TaskKey duplicate(QueryStatus queryStatus, MultiValueMap updateParameters(parameters, currentParams); // define a duplicate query - return storeQuery(queryStatus.getQuery().getQueryLogicName(), currentParams, currentUser, true); + return storeQuery(currentParams.getFirst(QUERY_LOGIC_NAME), currentParams, currentUser, true); } private boolean updateParameters(MultiValueMap newParameters, MultiValueMap currentParams) throws BadRequestQueryException { - return updateParameters(newParameters.keySet(), newParameters, currentParams); + return updateParameters(new ArrayList<>(newParameters.keySet()), newParameters, currentParams); } /** @@ -1470,7 +1476,7 @@ private boolean updateParameters(Collection parameterNames, MultiValueMa * if there is an unknown error */ public QueryImplListResponse list(String queryId, String queryName, ProxiedUserDetails currentUser) throws QueryException { - return list(queryId, queryName, ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); + return list(queryId, queryName, ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getDn().subjectDN())); } /** @@ -1516,7 +1522,11 @@ private QueryImplListResponse list(String queryId, String queryName, String user if (queryId != null && !queryId.isEmpty()) { // get the query for the given id queries = new ArrayList<>(); - queries.add(queryStorageCache.getQueryState(queryId).getQueryStatus().getQuery()); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + if (queryStatus != null) { + queries.add(queryStatus.getQuery()); + } } else { // get all of the queries queries = queryStorageCache.getQueryStatus().stream().map(QueryStatus::getQuery).collect(Collectors.toList()); diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java index e72df617..74138e9c 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java @@ -16,13 +16,17 @@ import datawave.microservice.query.storage.queue.TestQueryQueueManager; import datawave.security.authorization.DatawaveUser; import datawave.security.authorization.SubjectIssuerDNPair; +import datawave.security.util.ProxiedEntityUtils; import datawave.webservice.query.Query; import datawave.webservice.query.exception.QueryExceptionType; import datawave.webservice.query.result.event.DefaultEvent; import datawave.webservice.query.result.event.DefaultField; +import datawave.webservice.query.result.logic.QueryLogicDescription; import datawave.webservice.result.BaseResponse; import datawave.webservice.result.DefaultEventQueryResponse; import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.QueryImplListResponse; +import datawave.webservice.result.QueryLogicResponse; import datawave.webservice.result.VoidResponse; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URLEncodedUtils; @@ -79,13 +83,19 @@ import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; +import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; import static datawave.microservice.query.QueryParameters.QUERY_MAX_CONCURRENT_TASKS; import static datawave.microservice.query.QueryParameters.QUERY_MAX_RESULTS_OVERRIDE; +import static datawave.microservice.query.QueryParameters.QUERY_NAME; import static datawave.microservice.query.QueryParameters.QUERY_PAGESIZE; import static datawave.security.authorization.DatawaveUser.UserType.USER; import static datawave.webservice.common.audit.AuditParameters.AUDIT_ID; +import static datawave.webservice.common.audit.AuditParameters.QUERY_AUTHORIZATIONS; import static datawave.webservice.common.audit.AuditParameters.QUERY_STRING; import static datawave.webservice.query.QueryImpl.BEGIN_DATE; +import static datawave.webservice.query.QueryImpl.END_DATE; +import static datawave.webservice.query.QueryImpl.PAGESIZE; +import static datawave.webservice.query.QueryImpl.QUERY; import static org.springframework.test.web.client.ExpectedCount.never; import static org.springframework.test.web.client.match.MockRestRequestMatchers.anything; import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; @@ -1708,6 +1718,7 @@ public void testCancelSuccess() throws Exception { // @formatter:on } + // TODO: This test has periodic failures when running all tests @DirtiesContext @Test public void testCancelSuccess_activeNextCall() throws Exception { @@ -1720,10 +1731,16 @@ public void testCancelSuccess_activeNextCall() throws Exception { // call next on the query Future> nextFuture = nextQuery(authUser, queryId); - try { - nextFuture.get(1, TimeUnit.SECONDS); - } catch (TimeoutException e) { - // do nothing + boolean nextCallActive = queryStorageCache.getQueryStatus(queryId).getActiveNextCalls() > 0; + while (!nextCallActive) { + try { + nextFuture.get(500, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + nextCallActive = queryStorageCache.getQueryStatus(queryId).getActiveNextCalls() > 0; + if (TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - currentTimeMillis) > 5) { + throw e; + } + } } // cancel the query @@ -1966,7 +1983,7 @@ public void testAdminCancelFailure_notAdminUser() throws Exception { UriComponents uri = createUri(queryId + "/adminCancel"); RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); - // close the query + // cancel the query Future> closeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); // the response should come back right away @@ -1987,7 +2004,7 @@ public void testAdminCancelFailure_notAdminUser() throws Exception { @DirtiesContext @Test - public void testCancelAllSuccess() throws Exception { + public void testAdminCancelAllSuccess() throws Exception { ProxiedUserDetails adminUser = createUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); // create a bunch of queries @@ -2057,6 +2074,68 @@ public void testCancelAllSuccess() throws Exception { Assert.assertEquals(0, queryRequestEvents.size()); } + @DirtiesContext + @Test + public void testAdminCancelAllFailure_notAdminUser() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a bunch of queries + List queryIds = new ArrayList<>(); + long currentTimeMillis = System.currentTimeMillis(); + for (int i = 0; i < 10; i++) { + String queryId = createQuery(authUser, createParams()); + mockServer.reset(); + + queryIds.add(queryId); + + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + // cancel all queries as the admin user + UriComponents uri = createUri("/adminCancelAll"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + Future> cancelFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(403, cancelResponse.getStatusCodeValue()); + + // verify that query status was created correctly + List queryStatusList = queryStorageCache.getQueryStatus(); + + // verify that none of the queries were canceled + for (QueryStatus queryStatus : queryStatusList) { + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the result queue is still present + Assert.assertTrue(queryQueueManager.queueExists(queryStatus.getQueryKey().getQueryId())); + + // verify that the query tasks are still present + assertTasksCreated(queryStatus.getQueryKey().getQueryId()); + } + + // verify that there are no more events + Assert.assertEquals(0, queryRequestEvents.size()); + } + @DirtiesContext @Test public void testCloseSuccess() throws Exception { @@ -2122,10 +2201,16 @@ public void testCloseSuccess_activeNextCall() throws Exception { // call next on the query Future> nextFuture = nextQuery(authUser, queryId); - try { - nextFuture.get(1, TimeUnit.SECONDS); - } catch (TimeoutException e) { - // do nothing + boolean nextCallActive = queryStorageCache.getQueryStatus(queryId).getActiveNextCalls() > 0; + while (!nextCallActive) { + try { + nextFuture.get(500, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + nextCallActive = queryStorageCache.getQueryStatus(queryId).getActiveNextCalls() > 0; + if (TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - currentTimeMillis) > 5) { + throw e; + } + } } // close the query @@ -2312,14 +2397,14 @@ public void testCloseFailure_queryNotRunning() throws Exception { @Test public void testAdminCloseSuccess() throws Exception { ProxiedUserDetails authUser = createUserDetails(); - ProxiedUserDetails adminUser = createUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + ProxiedUserDetails adminUser = createAltUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); // create a valid query long currentTimeMillis = System.currentTimeMillis(); String queryId = createQuery(authUser, createParams()); // close the query as the admin user - Future> closeFuture = closeQuery(adminUser, queryId); + Future> closeFuture = adminCloseQuery(adminUser, queryId); // the response should come back right away ResponseEntity closeResponse = closeFuture.get(); @@ -2395,7 +2480,7 @@ public void testAdminCloseFailure_notAdminUser() throws Exception { @DirtiesContext @Test - public void testCloseAllSuccess() throws Exception { + public void testAdminCloseAllSuccess() throws Exception { ProxiedUserDetails adminUser = createUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); // create a bunch of queries @@ -2457,6 +2542,68 @@ public void testCloseAllSuccess() throws Exception { Assert.assertEquals(0, queryRequestEvents.size()); } + @DirtiesContext + @Test + public void testAdminCloseAllFailure_notAdminUser() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a bunch of queries + List queryIds = new ArrayList<>(); + long currentTimeMillis = System.currentTimeMillis(); + for (int i = 0; i < 10; i++) { + String queryId = createQuery(authUser, createParams()); + mockServer.reset(); + + queryIds.add(queryId); + + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + // close all queries as the admin user + UriComponents uri = createUri("/adminCloseAll"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + Future> closeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(403, closeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + List queryStatusList = queryStorageCache.getQueryStatus(); + + // verify that none of the queries were canceled + for (QueryStatus queryStatus : queryStatusList) { + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the result queue is still present + Assert.assertTrue(queryQueueManager.queueExists(queryStatus.getQueryKey().getQueryId())); + + // verify that the query tasks are still present + assertTasksCreated(queryStatus.getQueryKey().getQueryId()); + } + + // verify that there are no more events + Assert.assertEquals(0, queryRequestEvents.size()); + } + @DirtiesContext @Test public void testResetSuccess_resetOnDefined() throws Exception { @@ -2909,119 +3056,1927 @@ public void testResetFailure_ownershipFailure() throws Exception { // @formatter:on } - // remove - - // admin remove - - // update - - // duplicate - - // list - - // admin list - - // get query - - // remove all - - // list query logic - - private void publishEventsToQueue(String queryId, int numEvents, MultiValueMap fieldValues, String visibility) throws Exception { - for (int resultId = 0; resultId < numEvents; resultId++) { - DefaultEvent[] events = new DefaultEvent[1]; - events[0] = new DefaultEvent(); - long currentTime = System.currentTimeMillis(); - List fields = new ArrayList<>(); - for (Map.Entry> entry : fieldValues.entrySet()) { - for (String value : entry.getValue()) { - fields.add(new DefaultField(entry.getKey(), visibility, currentTime, value)); - } - } - events[0].setFields(fields); - queryQueueManager.sendMessage(queryId, new Result(Integer.toString(resultId), events)); - } - } - - private String createQuery(ProxiedUserDetails authUser, MultiValueMap map) { - return newQuery(authUser, map, "create"); - } - - private String defineQuery(ProxiedUserDetails authUser, MultiValueMap map) { - return newQuery(authUser, map, "define"); - } - - private String newQuery(ProxiedUserDetails authUser, MultiValueMap map, String createOrDefine) { - UriComponents uri = createUri("EventQuery/" + createOrDefine); + @DirtiesContext + @Test + public void testRemoveSuccess_removeOnDefined() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); - // not testing audit with this method - auditIgnoreSetup(); + // define a valid query + String queryId = defineQuery(authUser, createParams()); - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + // remove the query + Future> removeFuture = removeQuery(authUser, queryId); - return (String) resp.getBody().getResult(); - } - - private Future> nextQuery(ProxiedUserDetails authUser, String queryId) { - UriComponents uri = createUri(queryId + "/next"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + // the response should come back right away + ResponseEntity response = removeFuture.get(); - // make the next call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, BaseResponse.class)); - } - - private Future> adminCloseQuery(ProxiedUserDetails authUser, String queryId) { - return stopQuery(authUser, queryId, "adminClose"); - } - - private Future> closeQuery(ProxiedUserDetails authUser, String queryId) { - return stopQuery(authUser, queryId, "close"); - } - - private Future> adminCancelQuery(ProxiedUserDetails authUser, String queryId) { - return stopQuery(authUser, queryId, "adminCancel"); - } - - private Future> cancelQuery(ProxiedUserDetails authUser, String queryId) { - return stopQuery(authUser, queryId, "cancel"); - } - - private Future> stopQuery(ProxiedUserDetails authUser, String queryId, String closeOrCancel) { - UriComponents uri = createUri(queryId + "/" + closeOrCancel); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + Assert.assertEquals(200, response.getStatusCodeValue()); - // make the next call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); - } - - private Future> adminCloseAllQueries(ProxiedUserDetails authUser) { - UriComponents uri = createUri("/adminCloseAll"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + // verify that original query was removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - // make the next call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); - } - - private Future> adminCancelAllQueries(ProxiedUserDetails authUser) { - UriComponents uri = createUri("/adminCancelAll"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + Assert.assertNull(queryStatus); - // make the next call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + // verify that events were published + Assert.assertEquals(0, queryRequestEvents.size()); } - private Future> resetQuery(ProxiedUserDetails authUser, String queryId) { - UriComponents uri = createUri(queryId + "/reset"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + @DirtiesContext + @Test + public void testRemoveSuccess_removeOnClosed() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); - // make the next call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, GenericResponse.class)); + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + + // remove the query + Future> removeFuture = removeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = removeFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // verify that original query was removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + Assert.assertNull(queryStatus); + + // verify that events were published + Assert.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on } - private ProxiedUserDetails createUserDetails() { - return createUserDetails(null, null); - } + @DirtiesContext + @Test + public void testRemoveSuccess_removeOnCanceled() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // remove the query + Future> removeFuture = removeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = removeFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // verify that original query was removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + Assert.assertNull(queryStatus); + + // verify that events were published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testRemoveFailure_removeOnCreated() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // remove the query + Future> removeFuture = removeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = removeFuture.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + Assert.assertEquals("Cannot remove a running query.", Iterables.getOnlyElement(response.getBody().getExceptions()).getMessage()); + + // verify that original query was not removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + Assert.assertNotNull(queryStatus); + + // verify that events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testRemoveFailure_removeOnClosedActiveNext() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // call next on the query + nextQuery(authUser, queryId); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + + // remove the query + Future> removeFuture = removeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = removeFuture.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + Assert.assertEquals("Cannot remove a running query.", Iterables.getOnlyElement(response.getBody().getExceptions()).getMessage()); + + // verify that original query was not removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + Assert.assertNotNull(queryStatus); + + // verify that events were published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testRemoveFailure_queryNotFound() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + // remove the query + UriComponents uri = createUri(queryId + "/remove"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); + + // close the query + Future> resetFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, BaseResponse.class)); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assert.assertEquals(404, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @DirtiesContext + @Test + public void testRemoveFailure_ownershipFailure() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails altAuthUser = createAltUserDetails(); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + // remove the query + UriComponents uri = createUri(queryId + "/remove"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(altAuthUser, null, null, HttpMethod.DELETE, uri); + + // close the query + Future> resetFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, BaseResponse.class)); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assert.assertEquals(401, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @DirtiesContext + @Test + public void testAdminRemoveSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails adminUser = createAltUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + // remove the query + Future> removeFuture = adminRemoveQuery(adminUser, queryId); + + // the response should come back right away + ResponseEntity response = removeFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // verify that original query was removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + Assert.assertNull(queryStatus); + + // verify that events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @DirtiesContext + @Test + public void testAdminRemoveFailure_notAdminUser() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + // remove the query + UriComponents uri = createUri(queryId + "/adminRemove"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); + + // remove the queries + Future> removeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + // the response should come back right away + ResponseEntity response = removeFuture.get(); + + Assert.assertEquals(403, response.getStatusCodeValue()); + + // verify that original query was not removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + Assert.assertNotNull(queryStatus); + + // verify that events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @DirtiesContext + @Test + public void testAdminRemoveAllSuccess() throws Exception { + ProxiedUserDetails adminUser = createUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + // define a bunch of queries + for (int i = 0; i < 10; i++) { + defineQuery(adminUser, createParams()); + } + + // remove all queries as the admin user + Future> removeFuture = adminRemoveAllQueries(adminUser); + + // the response should come back right away + ResponseEntity removeResponse = removeFuture.get(); + + Assert.assertEquals(200, removeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + List queryStatusList = queryStorageCache.getQueryStatus(); + + Assert.assertEquals(0, queryStatusList.size()); + + // verify that there are no events + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @DirtiesContext + @Test + public void testAdminRemoveAllFailure_notAdminUser() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a bunch of queries + for (int i = 0; i < 10; i++) { + defineQuery(authUser, createParams()); + } + + UriComponents uri = createUri("/adminRemoveAll"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); + + // remove the queries + Future> removeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + // the response should come back right away + ResponseEntity removeResponse = removeFuture.get(); + + Assert.assertEquals(403, removeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + List queryStatusList = queryStorageCache.getQueryStatus(); + + Assert.assertEquals(10, queryStatusList.size()); + + // verify that there are no events + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @DirtiesContext + @Test + public void testUpdateSuccess_updateOnDefined() throws Exception { + ProxiedUserDetails authUser = createUserDetails(null, Arrays.asList("ALL", "NONE")); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + String newQuery = "SOME_OTHER_FIELD:SOME_OTHER_VALUE"; + String newAuths = "ALL,NONE"; + String newBegin = "20100101 000000.000"; + String newEnd = "20600101 000000.000"; + String newLogic = "AltEventQuery"; + int newPageSize = 100; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY, newQuery); + updateParams.set(QUERY_AUTHORIZATIONS, newAuths); + updateParams.set(BEGIN_DATE, newBegin); + updateParams.set(END_DATE, newEnd); + updateParams.set(QUERY_LOGIC_NAME, newLogic); + updateParams.set(PAGESIZE, Integer.toString(newPageSize)); + + // update the query + Future> updateFuture = updateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was updated + Assert.assertEquals(newQuery, queryStatus.getQuery().getQuery()); + Assert.assertEquals(newAuths, queryStatus.getQuery().getQueryAuthorizations()); + Assert.assertEquals(newBegin, DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate())); + Assert.assertEquals(newEnd, DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate())); + Assert.assertEquals(newLogic, queryStatus.getQuery().getQueryLogicName()); + Assert.assertEquals(newPageSize, queryStatus.getQuery().getPagesize()); + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @DirtiesContext + @Test + public void testUpdateSuccess_updateOnCreated() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + int newPageSize = 100; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(PAGESIZE, Integer.toString(newPageSize)); + + // update the query + Future> updateFuture = updateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was updated + Assert.assertEquals(newPageSize, queryStatus.getQuery().getPagesize()); + + // verify that events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testUpdateFailure_unsafeParamUpdateQuery() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + String newQuery = "SOME_OTHER_FIELD:SOME_OTHER_VALUE"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY, newQuery); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was not updated + Assert.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); + + // @formatter:off + assertQueryException( + "Cannot update the following parameters for a running query: query", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testUpdateFailure_unsafeParamUpdateDate() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + String newBegin = "20100101 000000.000"; + String newEnd = "20600101 000000.000"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(BEGIN_DATE, newBegin); + updateParams.set(END_DATE, newEnd); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was not updated + Assert.assertEquals(TEST_QUERY_BEGIN, DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate())); + Assert.assertEquals(TEST_QUERY_END, DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate())); + + // @formatter:off + assertQueryException( + "Cannot update the following parameters for a running query: begin, end", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testUpdateFailure_unsafeParamUpdateLogic() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + String newLogic = "AltEventQuery"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY_LOGIC_NAME, newLogic); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was not updated + Assert.assertEquals("EventQuery", queryStatus.getQuery().getQueryLogicName()); + + // @formatter:off + assertQueryException( + "Cannot update the following parameters for a running query: logicName", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testUpdateFailure_unsafeParamUpdateAuths() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + String newAuths = "ALL,NONE"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY_AUTHORIZATIONS, newAuths); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was not updated + Assert.assertEquals(TEST_QUERY_AUTHORIZATIONS, queryStatus.getQuery().getQueryAuthorizations()); + + // @formatter:off + assertQueryException( + "Cannot update the following parameters for a running query: auths", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testUpdateFailure_nullParams() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "No parameters specified for update.", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testUpdateFailure_queryNotFound() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY_STRING, TEST_QUERY_STRING); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assert.assertEquals(404, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @DirtiesContext + @Test + public void testUpdateFailure_ownershipFailure() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails altAuthUser = createAltUserDetails(); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + String newQuery = "SOME_OTHER_FIELD:SOME_OTHER_VALUE"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY, newQuery); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(altAuthUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assert.assertEquals(401, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was not updated + Assert.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @DirtiesContext + @Test + public void testDuplicateSuccess_duplicateOnDefined() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = defineQuery(authUser, createParams()); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + mockServer.reset(); + auditSentSetup(); + + // duplicate the query + Future> duplicateFuture = duplicateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + String dupeQueryId = (String) response.getBody().getResult(); + + // make sure an audit message was sent + assertAuditSent(dupeQueryId); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.DEFINED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + dupeQueryStatus); + // @formatter:on + + // make sure the queries are identical + Assert.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); + Assert.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); + Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); + Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); + Assert.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); + Assert.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); + + // verify that no events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + dupeQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testDuplicateSuccess_duplicateOnCreated() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + mockServer.reset(); + auditSentSetup(); + + // duplicate the query + Future> duplicateFuture = duplicateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + String dupeQueryId = (String) response.getBody().getResult(); + + // make sure an audit message was sent + assertAuditSent(dupeQueryId); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + dupeQueryStatus); + // @formatter:on + + // make sure the queries are identical + Assert.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); + Assert.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); + Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); + Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); + Assert.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); + Assert.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); + + // verify that no events were published + Assert.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + dupeQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testDuplicateSuccess_duplicateOnCanceled() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // this should return immediately + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + mockServer.reset(); + auditSentSetup(); + + // duplicate the query + Future> duplicateFuture = duplicateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + String dupeQueryId = (String) response.getBody().getResult(); + + // make sure an audit message was sent + assertAuditSent(dupeQueryId); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCELED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + dupeQueryStatus); + // @formatter:on + + // make sure the queries are identical + Assert.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); + Assert.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); + Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); + Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); + Assert.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); + Assert.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); + + // verify that no events were published + Assert.assertEquals(4, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + dupeQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testDuplicateSuccess_duplicateOnClosed() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // this should return immediately + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + mockServer.reset(); + auditSentSetup(); + + // duplicate the query + Future> duplicateFuture = duplicateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + String dupeQueryId = (String) response.getBody().getResult(); + + // make sure an audit message was sent + assertAuditSent(dupeQueryId); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CLOSED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + dupeQueryStatus); + // @formatter:on + + // make sure the queries are identical + Assert.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); + Assert.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); + Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); + Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); + Assert.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); + Assert.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); + + // verify that no events were published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + dupeQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testDuplicateSuccess_update() throws Exception { + ProxiedUserDetails authUser = createUserDetails(null, Arrays.asList("ALL", "NONE")); + + // define a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = defineQuery(authUser, createParams()); + + String newQuery = "SOME_OTHER_FIELD:SOME_OTHER_VALUE"; + String newAuths = "ALL,NONE"; + String newBegin = "20100101 000000.000"; + String newEnd = "20600101 000000.000"; + String newLogic = "AltEventQuery"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY, newQuery); + updateParams.set(QUERY_AUTHORIZATIONS, newAuths); + updateParams.set(BEGIN_DATE, newBegin); + updateParams.set(END_DATE, newEnd); + updateParams.set(QUERY_LOGIC_NAME, newLogic); + + mockServer.reset(); + auditSentSetup(); + + // duplicate the query + Future> duplicateFuture = duplicateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + String dupeQueryId = (String) response.getBody().getResult(); + + // make sure an audit message was sent + assertAuditSent(dupeQueryId); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.DEFINED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + dupeQueryStatus); + // @formatter:on + + // make sure the original query is unchanged + Assert.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); + Assert.assertEquals(TEST_QUERY_AUTHORIZATIONS, queryStatus.getQuery().getQueryAuthorizations()); + Assert.assertEquals(TEST_QUERY_BEGIN, DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate())); + Assert.assertEquals(TEST_QUERY_END, DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate())); + Assert.assertEquals("EventQuery", queryStatus.getQuery().getQueryLogicName()); + + // make sure the duplicated query is updated + Assert.assertEquals(newQuery, dupeQueryStatus.getQuery().getQuery()); + Assert.assertEquals(newAuths, dupeQueryStatus.getQuery().getQueryAuthorizations()); + Assert.assertEquals(newBegin, DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); + Assert.assertEquals(newEnd, DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); + Assert.assertEquals(newLogic, dupeQueryStatus.getQuery().getQueryLogicName()); + + // verify that no events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + dupeQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @DirtiesContext + @Test + public void testDuplicateFailure_invalidUpdate() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + String newLogic = "SomeBogusLogic"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY_LOGIC_NAME, newLogic); + + mockServer.reset(); + auditNotSentSetup(); + + // duplicate the query + UriComponents uri = createUri(queryId + "/duplicate"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.POST, uri); + + // make the duplicate call asynchronously + Future> duplicateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + // make sure an audit message wasn't sent + assertAuditNotSent(); + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testDuplicateFailure_queryNotFound() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + mockServer.reset(); + auditNotSentSetup(); + + // duplicate the query + UriComponents uri = createUri(queryId + "/duplicate"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.POST, uri); + + // make the duplicate call asynchronously + Future> duplicateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assert.assertEquals(404, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // make sure an audit message wasn't sent + assertAuditNotSent(); + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @DirtiesContext + @Test + public void testDuplicateFailure_ownershipFailure() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails altAuthUser = createAltUserDetails(); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + mockServer.reset(); + auditNotSentSetup(); + + // duplicate the query + UriComponents uri = createUri(queryId + "/duplicate"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(altAuthUser, updateParams, null, HttpMethod.POST, uri); + + // make the duplicate call asynchronously + Future> duplicateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assert.assertEquals(401, response.getStatusCodeValue()); + + // make sure an audit message wasn't sent + assertAuditNotSent(); + + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was not updated + Assert.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @DirtiesContext + @Test + public void testListSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails altAuthUser = createAltUserDetails(); + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + String queryId = createQuery(authUser, createParams()); + mockServer.reset(); + + queryIds.add(queryId); + } + + // define a bunch of queries as the alternate user + List altQueryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + String queryId = defineQuery(altAuthUser, createParams()); + mockServer.reset(); + + altQueryIds.add(queryId); + } + + // list queries as the original user + Future> listFuture = listQueries(authUser, null, null); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assert.assertEquals(5, result.getNumResults()); + + List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Collections.sort(queryIds); + Collections.sort(actualQueryIds); + + Assert.assertEquals(queryIds, actualQueryIds); + } + + @DirtiesContext + @Test + public void testListSuccess_filterOnQueryId() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + String queryId = createQuery(authUser, createParams()); + mockServer.reset(); + + queryIds.add(queryId); + } + + // list queries + Future> listFuture = listQueries(authUser, queryIds.get(0), null); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assert.assertEquals(1, result.getNumResults()); + + List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + } + + @DirtiesContext + @Test + public void testListSuccess_filterOnQueryName() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + String uniqueQueryName = "Unique Query"; + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + MultiValueMap params = createParams(); + if (i == 0) { + params.set(QUERY_NAME, uniqueQueryName); + } + + String queryId = createQuery(authUser, params); + mockServer.reset(); + + queryIds.add(queryId); + } + + // list queries + Future> listFuture = listQueries(authUser, null, uniqueQueryName); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assert.assertEquals(1, result.getNumResults()); + + List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + } + + @DirtiesContext + @Test + public void testListSuccess_filterOnMultiple() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + String uniqueQueryName = "Unique Query"; + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + MultiValueMap params = createParams(); + if (i == 0) { + params.set(QUERY_NAME, uniqueQueryName); + } + + String queryId = createQuery(authUser, params); + mockServer.reset(); + + queryIds.add(queryId); + } + + // list queries with just the query ID and a bogus name + Future> listFuture = listQueries(authUser, queryIds.get(0), "bogus name"); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assert.assertEquals(0, result.getNumResults()); + + // list queries with just the query name and a bogus ID + listFuture = listQueries(authUser, UUID.randomUUID().toString(), uniqueQueryName); + + // this should return immediately + listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + result = listResponse.getBody(); + + Assert.assertEquals(0, result.getNumResults()); + + // list queries with just the query name and a bogus ID + listFuture = listQueries(authUser, queryIds.get(0), uniqueQueryName); + + // this should return immediately + listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + result = listResponse.getBody(); + + Assert.assertEquals(1, result.getNumResults()); + + List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + } + + @DirtiesContext + @Test + public void testListFailure_ownershipFailure() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails altAuthUser = createAltUserDetails(); + + String uniqueQueryName = "Unique Query"; + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + MultiValueMap params = createParams(); + if (i == 0) { + params.set(QUERY_NAME, uniqueQueryName); + } + + String queryId = createQuery(authUser, params); + mockServer.reset(); + + queryIds.add(queryId); + } + + // list queries with just the query ID and a bogus name + Future> listFuture = listQueries(altAuthUser, queryIds.get(0), "bogus name"); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assert.assertEquals(0, result.getNumResults()); + + // list queries with just the query name and a bogus ID + listFuture = listQueries(altAuthUser, UUID.randomUUID().toString(), uniqueQueryName); + + // this should return immediately + listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + result = listResponse.getBody(); + + Assert.assertEquals(0, result.getNumResults()); + + // list queries with the query name and query ID + listFuture = listQueries(altAuthUser, queryIds.get(0), uniqueQueryName); + + // this should return immediately + listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + result = listResponse.getBody(); + + Assert.assertEquals(0, result.getNumResults()); + } + + @DirtiesContext + @Test + public void testAdminListSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails adminUser = createAltUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + String user = ProxiedEntityUtils.getShortName(authUser.getPrimaryUser().getDn().subjectDN()); + + String uniqueQueryName = "Unique Query"; + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + MultiValueMap params = createParams(); + if (i == 0) { + params.set(QUERY_NAME, uniqueQueryName); + } + + String queryId = createQuery(authUser, params); + mockServer.reset(); + + queryIds.add(queryId); + } + + // list queries with just the query ID + Future> listFuture = adminListQueries(adminUser, queryIds.get(0), user, null); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assert.assertEquals(1, result.getNumResults()); + + List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + + // list queries with just the query name + listFuture = adminListQueries(adminUser, null, user, uniqueQueryName); + + // this should return immediately + listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + result = listResponse.getBody(); + + Assert.assertEquals(1, result.getNumResults()); + + actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + + // list queries with the query name and query ID + listFuture = adminListQueries(adminUser, queryIds.get(0), user, uniqueQueryName); + + // this should return immediately + listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + result = listResponse.getBody(); + + Assert.assertEquals(1, result.getNumResults()); + + actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + } + + @DirtiesContext + @Test + public void testAdminListFailure_notAdminUser() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails altAuthUser = createAltUserDetails(); + + String user = ProxiedEntityUtils.getShortName(authUser.getPrimaryUser().getDn().subjectDN()); + + String uniqueQueryName = "Unique Query"; + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + MultiValueMap params = createParams(); + if (i == 0) { + params.set(QUERY_NAME, uniqueQueryName); + } + + String queryId = createQuery(authUser, params); + mockServer.reset(); + + queryIds.add(queryId); + } + + UriComponentsBuilder uriBuilder = uriBuilder("/adminList"); + UriComponents uri = uriBuilder.build(); + + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(altAuthUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + Future> listFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + ResponseEntity listResponse = listFuture.get(); + + Assert.assertEquals(403, listResponse.getStatusCodeValue()); + } + + @DirtiesContext + @Test + public void testGetQuerySuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a query + String queryId = createQuery(authUser, createParams()); + mockServer.reset(); + + // get the query + Future> listFuture = getQuery(authUser, queryId); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assert.assertEquals(1, result.getNumResults()); + + Assert.assertEquals(queryId, result.getQuery().get(0).getId().toString()); + } + + @Test + public void testListQueryLogicSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + Future> future = listQueryLogic(authUser); + + ResponseEntity response = future.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + QueryLogicResponse qlResponse = response.getBody(); + + Assert.assertEquals(2, qlResponse.getQueryLogicList().size()); + + List qlNames = qlResponse.getQueryLogicList().stream().map(QueryLogicDescription::getName).sorted().collect(Collectors.toList()); + + Assert.assertEquals(Arrays.asList("AltEventQuery", "EventQuery"), qlNames); + } + + private void publishEventsToQueue(String queryId, int numEvents, MultiValueMap fieldValues, String visibility) throws Exception { + for (int resultId = 0; resultId < numEvents; resultId++) { + DefaultEvent[] events = new DefaultEvent[1]; + events[0] = new DefaultEvent(); + long currentTime = System.currentTimeMillis(); + List fields = new ArrayList<>(); + for (Map.Entry> entry : fieldValues.entrySet()) { + for (String value : entry.getValue()) { + fields.add(new DefaultField(entry.getKey(), visibility, currentTime, value)); + } + } + events[0].setFields(fields); + queryQueueManager.sendMessage(queryId, new Result(Integer.toString(resultId), events)); + } + } + + private String createQuery(ProxiedUserDetails authUser, MultiValueMap map) { + return newQuery(authUser, map, "create"); + } + + private String defineQuery(ProxiedUserDetails authUser, MultiValueMap map) { + return newQuery(authUser, map, "define"); + } + + private String newQuery(ProxiedUserDetails authUser, MultiValueMap map, String createOrDefine) { + UriComponents uri = createUri("EventQuery/" + createOrDefine); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + return (String) resp.getBody().getResult(); + } + + private Future> nextQuery(ProxiedUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId + "/next"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, BaseResponse.class)); + } + + private Future> adminCloseQuery(ProxiedUserDetails authUser, String queryId) { + return stopQuery(authUser, queryId, "adminClose"); + } + + private Future> closeQuery(ProxiedUserDetails authUser, String queryId) { + return stopQuery(authUser, queryId, "close"); + } + + private Future> adminCancelQuery(ProxiedUserDetails authUser, String queryId) { + return stopQuery(authUser, queryId, "adminCancel"); + } + + private Future> cancelQuery(ProxiedUserDetails authUser, String queryId) { + return stopQuery(authUser, queryId, "cancel"); + } + + private Future> stopQuery(ProxiedUserDetails authUser, String queryId, String closeOrCancel) { + UriComponents uri = createUri(queryId + "/" + closeOrCancel); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + private Future> adminCloseAllQueries(ProxiedUserDetails authUser) { + UriComponents uri = createUri("/adminCloseAll"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + private Future> adminCancelAllQueries(ProxiedUserDetails authUser) { + UriComponents uri = createUri("/adminCancelAll"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + private Future> resetQuery(ProxiedUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId + "/reset"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, GenericResponse.class)); + } + + private Future> removeQuery(ProxiedUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId + "/remove"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + private Future> adminRemoveQuery(ProxiedUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId + "/adminRemove"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + private Future> adminRemoveAllQueries(ProxiedUserDetails authUser) { + UriComponents uri = createUri("/adminRemoveAll"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + private Future> updateQuery(ProxiedUserDetails authUser, String queryId, MultiValueMap map) { + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, GenericResponse.class)); + } + + private Future> duplicateQuery(ProxiedUserDetails authUser, String queryId, MultiValueMap map) { + UriComponents uri = createUri(queryId + "/duplicate"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // make the update call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, GenericResponse.class)); + } + + private Future> listQueries(ProxiedUserDetails authUser, String queryId, String queryName) { + UriComponentsBuilder uriBuilder = uriBuilder("/list"); + if (queryId != null) { + uriBuilder.queryParam("queryId", queryId); + } + if (queryName != null) { + uriBuilder.queryParam("queryName", queryName); + } + UriComponents uri = uriBuilder.build(); + + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, QueryImplListResponse.class)); + } + + private Future> getQuery(ProxiedUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId); + + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, QueryImplListResponse.class)); + } + + private Future> listQueryLogic(ProxiedUserDetails authUser) { + UriComponents uri = createUri("/listQueryLogic"); + + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, QueryLogicResponse.class)); + } + + private Future> adminListQueries(ProxiedUserDetails authUser, String queryId, String user, String queryName) { + UriComponentsBuilder uriBuilder = uriBuilder("/adminList"); + if (queryId != null) { + uriBuilder.queryParam("queryId", queryId); + } + if (queryName != null) { + uriBuilder.queryParam("queryName", queryName); + } + if (user != null) { + uriBuilder.queryParam("user", user); + } + UriComponents uri = uriBuilder.build(); + + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, QueryImplListResponse.class)); + } + + private ProxiedUserDetails createUserDetails() { + return createUserDetails(null, null); + } private ProxiedUserDetails createUserDetails(Collection roles, Collection auths) { Collection userRoles = roles != null ? roles : Collections.singleton("AuthorizedUser"); @@ -3041,8 +4996,12 @@ private ProxiedUserDetails createAltUserDetails(Collection roles, Collec return new ProxiedUserDetails(Collections.singleton(datawaveUser), datawaveUser.getCreationTime()); } + private UriComponentsBuilder uriBuilder(String path) { + return UriComponentsBuilder.newInstance().scheme("https").host("localhost").port(webServicePort).path("/query/v1/" + path); + } + private UriComponents createUri(String path) { - return UriComponentsBuilder.newInstance().scheme("https").host("localhost").port(webServicePort).path("/query/v1/" + path).build(); + return uriBuilder(path).build(); } private MultiValueMap createParams() { diff --git a/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml b/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml index 1c644274..7fcd6122 100644 --- a/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml +++ b/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml @@ -234,6 +234,10 @@ + + + + From 0390a8dea903049f634d49aabf3ff26dde142f07 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 6 Aug 2021 08:24:43 -0400 Subject: [PATCH 082/218] Incorporated the new query metrics api into the query service. --- .../query/QueryManagementService.java | 116 +- .../query/config/QueryServiceConfig.java | 12 +- .../microservice/query/runner/NextCall.java | 50 +- .../query/web/BaseQueryResponseAdvice.java | 1 + .../QueryMetricsEnrichmentFilterAdvice.java | 139 +- .../src/main/resources/config/bootstrap.yml | 6 +- .../query/AbstractQueryServiceTest.java | 530 ++ .../query/QueryServiceCancelTest.java | 512 ++ .../query/QueryServiceCloseTest.java | 507 ++ .../query/QueryServiceCreateTest.java | 478 ++ .../query/QueryServiceDefineTest.java | 418 ++ .../query/QueryServiceDuplicateTest.java | 588 ++ .../query/QueryServiceListTest.java | 428 ++ .../query/QueryServiceNextTest.java | 753 +++ .../query/QueryServiceRemoveTest.java | 418 ++ .../query/QueryServiceResetTest.java | 488 ++ .../microservice/query/QueryServiceTest.java | 5179 ----------------- .../query/QueryServiceUpdateTest.java | 455 ++ .../src/test/resources/log4j2-test.xml | 4 +- 19 files changed, 5771 insertions(+), 5311 deletions(-) create mode 100644 query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java create mode 100644 query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java create mode 100644 query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java create mode 100644 query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java create mode 100644 query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java create mode 100644 query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java create mode 100644 query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java create mode 100644 query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java create mode 100644 query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java create mode 100644 query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java delete mode 100644 query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java create mode 100644 query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 8a531095..39cd96cb 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -19,12 +19,13 @@ import datawave.microservice.query.storage.QueryStorageCache; import datawave.microservice.query.storage.TaskKey; import datawave.microservice.query.util.QueryUtil; +import datawave.microservice.querymetric.BaseQueryMetric; +import datawave.microservice.querymetric.QueryMetricClient; import datawave.security.util.ProxiedEntityUtils; import datawave.webservice.common.audit.AuditParameters; import datawave.webservice.common.audit.Auditor; import datawave.webservice.query.Query; import datawave.webservice.query.QueryImpl; -import datawave.webservice.query.cache.QueryMetricFactory; import datawave.webservice.query.cache.ResultsPage; import datawave.webservice.query.exception.BadRequestQueryException; import datawave.webservice.query.exception.DatawaveErrorCode; @@ -34,7 +35,6 @@ import datawave.webservice.query.exception.QueryException; import datawave.webservice.query.exception.TimeoutQueryException; import datawave.webservice.query.exception.UnauthorizedQueryException; -import datawave.webservice.query.metric.BaseQueryMetric; import datawave.webservice.query.result.event.ResponseObjectFactory; import datawave.webservice.query.result.logic.QueryLogicDescription; import datawave.webservice.query.util.QueryUncaughtExceptionHandler; @@ -96,13 +96,15 @@ public class QueryManagementService implements QueryRequestHandler { private final ApplicationEventPublisher eventPublisher; private final BusProperties busProperties; - // Note: QueryParameters need to be request scoped + // Note: QueryParameters needs to be request scoped private final QueryParameters queryParameters; // Note: SecurityMarking needs to be request scoped private final SecurityMarking securityMarking; + // Note: BaseQueryMetric needs to be request scoped + private final BaseQueryMetric baseQueryMetric; private final QueryLogicFactory queryLogicFactory; - private final QueryMetricFactory queryMetricFactory; + private final QueryMetricClient queryMetricClient; private final ResponseObjectFactory responseObjectFactory; private final QueryStorageCache queryStorageCache; private final QueryQueueManager queryQueueManager; @@ -113,17 +115,19 @@ public class QueryManagementService implements QueryRequestHandler { private final MultiValueMap nextCallMap = new LinkedMultiValueMap<>(); public QueryManagementService(QueryProperties queryProperties, ApplicationContext appCtx, ApplicationEventPublisher eventPublisher, - BusProperties busProperties, QueryParameters queryParameters, SecurityMarking securityMarking, QueryLogicFactory queryLogicFactory, - QueryMetricFactory queryMetricFactory, ResponseObjectFactory responseObjectFactory, QueryStorageCache queryStorageCache, - QueryQueueManager queryQueueManager, AuditClient auditClient, ThreadPoolTaskExecutor nextCallExecutor) { + BusProperties busProperties, QueryParameters queryParameters, SecurityMarking securityMarking, BaseQueryMetric baseQueryMetric, + QueryLogicFactory queryLogicFactory, QueryMetricClient queryMetricClient, ResponseObjectFactory responseObjectFactory, + QueryStorageCache queryStorageCache, QueryQueueManager queryQueueManager, AuditClient auditClient, + ThreadPoolTaskExecutor nextCallExecutor) { this.queryProperties = queryProperties; this.appCtx = appCtx; this.eventPublisher = eventPublisher; this.busProperties = busProperties; this.queryParameters = queryParameters; this.securityMarking = securityMarking; + this.baseQueryMetric = baseQueryMetric; this.queryLogicFactory = queryLogicFactory; - this.queryMetricFactory = queryMetricFactory; + this.queryMetricClient = queryMetricClient; this.responseObjectFactory = responseObjectFactory; this.queryStorageCache = queryStorageCache; this.queryQueueManager = queryQueueManager; @@ -396,9 +400,11 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p // @formatter:on } - // TODO: JWO: Figure out how to make query tracing work with our new architecture. Datawave issue #1155 - - // TODO: JWO: Figure out how to make query metrics work with our new architecture. Datawave issue #1156 + // update the query metric + baseQueryMetric.setQueryId(taskKey.getQueryId()); + baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.DEFINED); + baseQueryMetric.populate(query); + baseQueryMetric.setProxyServers(currentUser.getProxyServers()); return taskKey; } catch (Exception e) { @@ -449,14 +455,26 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p */ public BaseQueryResponse createAndNext(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + String queryId = null; try { - String queryId = create(queryLogicName, parameters, currentUser).getResult(); + queryId = create(queryLogicName, parameters, currentUser).getResult(); return next(queryId, currentUser.getPrimaryUser().getRoles()); - } catch (QueryException e) { - throw e; } catch (Exception e) { - log.error("Unknown error calling create and next.", e); - throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error calling create and next."); + // TODO: QueryException results in VoidResponse. This differs from the original RestAPI where an exception was returned in a + // DefaultEventQueryResponse. + QueryException qe; + if (!(e instanceof QueryException)) { + log.error("Unknown error calling create and next. " + queryId, e); + qe = new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Unknown error calling create and next. " + queryId); + } else { + qe = (QueryException) e; + } + + if (queryId != null && !(qe instanceof NoResultsQueryException)) { + baseQueryMetric.setError(qe); + } + + throw qe; } } @@ -506,11 +524,22 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th } else { throw new BadRequestQueryException("Cannot call next on a query that is not running", HttpStatus.SC_BAD_REQUEST + "-1"); } - } catch (QueryException e) { - throw e; } catch (Exception e) { - log.error("Unknown error getting next page for query " + queryId, e); - throw new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Unknown error getting next page for query " + queryId); + // TODO: QueryException results in VoidResponse. This differs from the original RestAPI where an exception was returned in a + // DefaultEventQueryResponse. + QueryException qe; + if (!(e instanceof QueryException)) { + log.error("Unknown error getting next page for query " + queryId, e); + qe = new QueryException(DatawaveErrorCode.QUERY_NEXT_ERROR, e, "Unknown error getting next page for query " + queryId); + } else { + qe = (QueryException) e; + } + + if (!(qe instanceof NoResultsQueryException)) { + baseQueryMetric.setError(qe); + } + + throw qe; } } @@ -553,12 +582,15 @@ private BaseQueryResponse next(String queryId, Collection userRoles) thr String queryLogicName = queryStatus.getQuery().getQueryLogicName(); QueryLogic queryLogic = queryLogicFactory.getQueryLogic(queryStatus.getQuery().getQueryLogicName(), userRoles); + // update query metrics + baseQueryMetric.setQueryId(queryId); + baseQueryMetric.setQueryLogic(queryLogicName); + // @formatter:off final NextCall nextCall = new NextCall.Builder() .setQueryProperties(queryProperties) .setQueryQueueManager(queryQueueManager) .setQueryStorageCache(queryStorageCache) - .setQueryMetricFactory(queryMetricFactory) .setQueryId(queryId) .setQueryLogic(queryLogic) .build(); @@ -572,6 +604,9 @@ private BaseQueryResponse next(String queryId, Collection userRoles) thr // wait for the results to be ready ResultsPage resultsPage = nextCall.getFuture().get(); + // update the query metric + nextCall.updateQueryMetric(baseQueryMetric); + // format the response if (!resultsPage.getResults().isEmpty()) { BaseQueryResponse response = queryLogic.getTransformer(queryStatus.getQuery()).createResponse(resultsPage); @@ -592,9 +627,11 @@ private BaseQueryResponse next(String queryId, Collection userRoles) thr } else { if (nextCall.isCanceled()) { throw new QueryCanceledQueryException(DatawaveErrorCode.QUERY_CANCELED, MessageFormat.format("{0} canceled;", queryId)); - } else if (nextCall.getMetric().getLifecycle() == BaseQueryMetric.Lifecycle.NEXTTIMEOUT) { + } else if (baseQueryMetric.getLifecycle() == BaseQueryMetric.Lifecycle.NEXTTIMEOUT) { throw new TimeoutQueryException(DatawaveErrorCode.QUERY_TIMEOUT, MessageFormat.format("{0} timed out.", queryId)); } else { + // if there are no results, and we didn't timeout, close the query + close(queryId); throw new NoResultsQueryException(DatawaveErrorCode.NO_QUERY_RESULTS_FOUND, MessageFormat.format("{0}", queryId)); } } @@ -804,8 +841,6 @@ public void cancel(String queryId, boolean publishEvent) throws InterruptedExcep // delete the results queue queryQueueManager.deleteQueue(queryId); - // TODO: Delete the tasks as well? - QueryRequest cancelRequest = QueryRequest.cancel(queryId); // publish a cancel event to all of the query services @@ -813,6 +848,19 @@ public void cancel(String queryId, boolean publishEvent) throws InterruptedExcep // publish a cancel event to the executor pool publishExecutorEvent(cancelRequest, queryStatus.getQueryKey().getQueryPool()); + + // update query metrics + baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.CANCELLED); + try { + // @formatter:off + queryMetricClient.submit( + new QueryMetricClient.Request.Builder() + .withMetric(baseQueryMetric) + .build()); + // @formatter:on + } catch (Exception e) { + log.error("Error updateing query metric", e); + } } } @@ -989,15 +1037,21 @@ public void close(String queryId) throws InterruptedException, QueryException { queryQueueManager.deleteQueue(queryId); } - // TODO: If we are running a next call, don't delete tasks, task states, or delete query queues - // TODO: If we aren't running a next call, do what we would do for a cancel - - // TODO: update tasks, task states, and delete query queues - // TODO: Tasks should be - // TODO: Figure out when to delete the queue for a close. After the last page is returned? - // publish a close event to the executor pool publishExecutorEvent(QueryRequest.close(queryId), queryStatus.getQueryKey().getQueryPool()); + + // update query metrics + baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.CLOSED); + try { + // @formatter:off + queryMetricClient.submit( + new QueryMetricClient.Request.Builder() + .withMetric(baseQueryMetric) + .build()); + // @formatter:on + } catch (Exception e) { + log.error("Error updating query metric", e); + } } /** diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java index 7aa13b47..88f85a6d 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java @@ -4,8 +4,9 @@ import datawave.marking.SecurityMarking; import datawave.microservice.query.DefaultQueryParameters; import datawave.microservice.query.QueryParameters; -import datawave.webservice.query.cache.QueryMetricFactory; -import datawave.webservice.query.cache.QueryMetricFactoryImpl; +import datawave.microservice.querymetric.BaseQueryMetric; +import datawave.microservice.querymetric.QueryMetricFactory; +import datawave.microservice.querymetric.QueryMetricFactoryImpl; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.cloud.context.config.annotation.RefreshScope; @@ -35,6 +36,13 @@ public SecurityMarking securityMarking() { return securityMarking; } + @Bean + @ConditionalOnMissingBean + @RequestScope + public BaseQueryMetric baseQueryMetric() { + return queryMetricFactory().createMetric(); + } + @Bean @ConditionalOnMissingBean(QueryProperties.class) @ConfigurationProperties("datawave.query") diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 93e7dd83..e8a286f9 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -10,11 +10,10 @@ import datawave.microservice.query.storage.QueryStorageCache; import datawave.microservice.query.storage.Result; import datawave.microservice.query.storage.TaskStates; -import datawave.webservice.query.cache.QueryMetricFactory; +import datawave.microservice.querymetric.BaseQueryMetric; +import datawave.microservice.querymetric.QueryMetric; import datawave.webservice.query.cache.ResultsPage; import datawave.webservice.query.data.ObjectSizeOf; -import datawave.webservice.query.metric.BaseQueryMetric; -import datawave.webservice.query.metric.QueryMetric; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.messaging.Message; @@ -53,6 +52,7 @@ public class NextCall implements Callable> { private final List results = new LinkedList<>(); private long pageSizeBytes; private long startTimeMillis; + private long stopTimeMillis; private ResultsPage.Status status = ResultsPage.Status.COMPLETE; private long lastQueryStatusUpdateTime = 0L; @@ -62,7 +62,7 @@ public class NextCall implements Callable> { private boolean tasksFinished = false; private long prevResultCount = 0L; - private final BaseQueryMetric metric; + private BaseQueryMetric.Lifecycle lifecycle; private NextCall(Builder builder) { this.nextCallProperties = builder.nextCallProperties; @@ -98,8 +98,6 @@ private NextCall(Builder builder) { log.info("Maximum results set to " + this.maxResults + " instead of default " + builder.queryLogic.getMaxResults() + ", user " + status.getQuery().getUserDN() + " has a DN configured with a different limit"); } - - this.metric = builder.queryMetricFactory.createMetric(); } @Override @@ -141,9 +139,20 @@ public ResultsPage call() { // stop the result listener resultListener.stop(); + // update some values for metrics + stopTimeMillis = System.currentTimeMillis(); + if (lifecycle == null && !results.isEmpty()) { + lifecycle = BaseQueryMetric.Lifecycle.RESULTS; + } + return new ResultsPage<>(results, status); } + public void updateQueryMetric(BaseQueryMetric baseQueryMetric) { + baseQueryMetric.addPageTime(results.size(), stopTimeMillis - startTimeMillis, startTimeMillis, stopTimeMillis); + baseQueryMetric.setLifecycle(lifecycle); + } + private boolean isFinished(String queryId) { boolean finished = false; long callTimeMillis = System.currentTimeMillis() - startTimeMillis; @@ -167,6 +176,8 @@ private boolean isFinished(String queryId) { if (!finished && (canceled || queryStatus.getQueryState() == QueryStatus.QUERY_STATE.CANCELED)) { log.info("Query [{}]: query cancelled, aborting next call", queryId); + // no query metric lifecycle update - assumption is that the cancel call handled that + // set to partial for now - if there are no results, it will switch to NONE later status = ResultsPage.Status.PARTIAL; finished = true; @@ -206,8 +217,7 @@ private boolean isFinished(String queryId) { if (maxResultsOverride >= 0 && numResults >= maxResultsOverride) { log.info("Query [{}]: max results override has been reached, aborting next call", queryId); - // TODO: Figure out query metrics - metric.setLifecycle(QueryMetric.Lifecycle.MAXRESULTS); + lifecycle = QueryMetric.Lifecycle.MAXRESULTS; status = ResultsPage.Status.PARTIAL; @@ -216,8 +226,7 @@ private boolean isFinished(String queryId) { } else if (maxResults >= 0 && numResults >= maxResults) { log.info("Query [{}]: logic max results has been reached, aborting next call", queryId); - // TODO: Figure out query metrics - metric.setLifecycle(QueryMetric.Lifecycle.MAXRESULTS); + lifecycle = QueryMetric.Lifecycle.MAXRESULTS; status = ResultsPage.Status.PARTIAL; @@ -225,14 +234,12 @@ private boolean isFinished(String queryId) { } } - // TODO: Do I need to pull query metrics to get the next/seek count? - // This used to come from the query logic transform iterator + // TODO: We need to stop relying on query metrics for these values // 7) have we reached the "max work" limit? (i.e. next count + seek count) - if (!finished && logicMaxWork > 0 && (metric.getNextCount() + metric.getSeekCount()) >= logicMaxWork) { + if (!finished && logicMaxWork > 0 && (queryStatus.getNextCount() + queryStatus.getSeekCount()) >= logicMaxWork) { log.info("Query [{}]: logic max work has been reached, aborting next call", queryId); - // TODO: Figure out query metrics - metric.setLifecycle(BaseQueryMetric.Lifecycle.MAXWORK); + lifecycle = BaseQueryMetric.Lifecycle.MAXWORK; status = ResultsPage.Status.PARTIAL; @@ -254,8 +261,7 @@ private boolean isFinished(String queryId) { log.info("Query [{}]: max call time reached, returning existing results: {} of {} results in {}ms", queryId, results.size(), maxResultsPerPage, callTimeMillis); - // TODO: Figure out query metrics - metric.setLifecycle(BaseQueryMetric.Lifecycle.NEXTTIMEOUT); + lifecycle = BaseQueryMetric.Lifecycle.NEXTTIMEOUT; status = ResultsPage.Status.PARTIAL; @@ -336,16 +342,11 @@ public void setFuture(Future> future) { this.future = future; } - public BaseQueryMetric getMetric() { - return metric; - } - public static class Builder { private NextCallProperties nextCallProperties; private QueryExpirationProperties expirationProperties; private QueryQueueManager queryQueueManager; private QueryStorageCache queryStorageCache; - private QueryMetricFactory queryMetricFactory; private String queryId; private QueryLogic queryLogic; @@ -375,11 +376,6 @@ public Builder setQueryStorageCache(QueryStorageCache queryStorageCache) { return this; } - public Builder setQueryMetricFactory(QueryMetricFactory queryMetricFactory) { - this.queryMetricFactory = queryMetricFactory; - return this; - } - public Builder setQueryId(String queryId) { this.queryId = queryId; return this; diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java index 4adbca10..5c5ee97d 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java @@ -29,6 +29,7 @@ public boolean supports(@NonNull MethodParameter returnType, @NonNull Class conv public BaseQueryResponse beforeBodyWrite(BaseQueryResponse baseQueryResponse, @NonNull MethodParameter returnType, @NonNull MediaType selectedContentType, @NonNull Class selectedConverterType, @NonNull ServerHttpRequest request, ServerHttpResponse response) { response.getHeaders().add(Constants.PAGE_NUMBER, String.valueOf(baseQueryResponse.getPageNumber())); + // TODO: This isn't the most accurate way of determining whether this is the last page response.getHeaders().add(Constants.IS_LAST_PAGE, String.valueOf(!baseQueryResponse.getHasResults())); response.getHeaders().add(Constants.PARTIAL_RESULTS, String.valueOf(baseQueryResponse.isPartialResults())); return baseQueryResponse; diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java index 73f3bbfe..6ed1a8e8 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java @@ -1,12 +1,11 @@ package datawave.microservice.query.web.filter; import datawave.microservice.query.logic.QueryLogicFactory; -import datawave.microservice.query.storage.QueryState; +import datawave.microservice.query.storage.QueryStatus; import datawave.microservice.query.storage.QueryStorageCache; -import datawave.microservice.query.web.QueryMetrics; import datawave.microservice.query.web.annotation.EnrichQueryMetrics; -import datawave.webservice.query.metric.BaseQueryMetric; -import datawave.webservice.query.metric.QueryMetric; +import datawave.microservice.querymetric.BaseQueryMetric; +import datawave.microservice.querymetric.QueryMetricClient; import datawave.webservice.result.BaseQueryResponse; import datawave.webservice.result.GenericResponse; import org.apache.log4j.Logger; @@ -32,12 +31,17 @@ public class QueryMetricsEnrichmentFilterAdvice extends BaseMethodStatsFilter im private final QueryStorageCache queryStorageCache; - private final QueryMetrics queryMetrics; + private final QueryMetricClient queryMetricClient; - public QueryMetricsEnrichmentFilterAdvice(QueryLogicFactory queryLogicFactory, QueryStorageCache queryStorageCache, QueryMetrics queryMetrics) { + // Note: BaseQueryMetric needs to be request scoped + private final BaseQueryMetric baseQueryMetric; + + public QueryMetricsEnrichmentFilterAdvice(QueryLogicFactory queryLogicFactory, QueryStorageCache queryStorageCache, QueryMetricClient queryMetricClient, + BaseQueryMetric baseQueryMetric) { this.queryLogicFactory = queryLogicFactory; this.queryStorageCache = queryStorageCache; - this.queryMetrics = queryMetrics; + this.queryMetricClient = queryMetricClient; + this.baseQueryMetric = baseQueryMetric; } @Override @@ -70,8 +74,6 @@ public boolean supports(MethodParameter returnType, @NonNull Class> selectedConverterType, @NonNull ServerHttpRequest request, @NonNull ServerHttpResponse response) { - System.out.println("QueryMetricsEnrichmentFilterAdvice writing metrics"); - if (body instanceof GenericResponse) { @SuppressWarnings({"unchecked", "rawtypes"}) GenericResponse genericResponse = (GenericResponse) body; @@ -87,66 +89,69 @@ public Object beforeBodyWrite(Object body, @NonNull MethodParameter returnType, @Override public void postProcess(ResponseMethodStats responseStats) { String queryId = QueryMetricsEnrichmentContext.getQueryId(); - if (queryId != null) { - if (queryStorageCache != null) { - // TODO: JWO: Need to figure out where the query metrics object is supposed to come from. I assumed - // that it would be contained in the QueryState similar to how it is stored in the RunningQuery, - // but that does not appear to be the case. - QueryState queryState = null; - - boolean isMetricsEnabled = false; - // try { - // isMetricsEnabled = queryLogicFactory.getQueryLogic(queryState.getQueryLogic()).getCollectQueryMetrics(); - // } catch (Exception e) { - // log.warn("Unable to determine if query logic '" + queryState.getQueryLogic() + "' supports metrics"); - // } - - if (queryState != null && isMetricsEnabled) { - try { - // TODO: JWO: This probably shouldn't be instantiated here, and we also shouldn't hard code the basequerymetric implementation. - // just using this as a placeholder until we start to bring things together. - BaseQueryMetric metric = new QueryMetric(); - switch (QueryMetricsEnrichmentContext.getMethodType()) { - case CREATE: - metric.setCreateCallTime(responseStats.getCallTime()); - metric.setLoginTime(responseStats.getLoginTime()); - break; - case CREATE_AND_NEXT: - metric.setCreateCallTime(responseStats.getCallTime()); - metric.setLoginTime(responseStats.getLoginTime()); - List pageTimes = metric.getPageTimes(); - if (pageTimes != null && !pageTimes.isEmpty()) { - BaseQueryMetric.PageMetric pm = pageTimes.get(pageTimes.size() - 1); - pm.setCallTime(responseStats.getCallTime()); - pm.setLoginTime(responseStats.getLoginTime()); - pm.setSerializationTime(responseStats.getSerializationTime()); - pm.setBytesWritten(responseStats.getBytesWritten()); - } - break; - case NEXT: - pageTimes = metric.getPageTimes(); - if (pageTimes != null && !pageTimes.isEmpty()) { - BaseQueryMetric.PageMetric pm = pageTimes.get(pageTimes.size() - 1); - pm.setCallTime(responseStats.getCallTime()); - pm.setLoginTime(responseStats.getLoginTime()); - pm.setSerializationTime(responseStats.getSerializationTime()); - pm.setBytesWritten(responseStats.getBytesWritten()); - } - break; - } - - if (queryMetrics != null) - queryMetrics.updateMetric(metric); - else - log.error("QueryMetricsBean JNDI lookup returned null"); - } catch (Exception e) { - log.error("Unable to record metrics for " + QueryMetricsEnrichmentContext.getMethodType() + " method: " + e.getLocalizedMessage(), e); + EnrichQueryMetrics.MethodType methodType = QueryMetricsEnrichmentContext.getMethodType(); + + if (queryId != null && methodType != null) { + // determine which query logic is being used + String queryLogic = null; + if (baseQueryMetric.getQueryLogic() != null) { + queryLogic = baseQueryMetric.getQueryLogic(); + } else { + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + if (queryStatus != null) { + queryLogic = queryStatus.getQuery().getQueryLogicName(); + } + } + + // determine whether metrics are enabled + boolean isMetricsEnabled = false; + try { + isMetricsEnabled = queryLogicFactory.getQueryLogic(queryLogic).getCollectQueryMetrics(); + } catch (Exception e) { + log.warn("Unable to determine if query logic '" + queryLogic + "' supports metrics"); + } + + if (isMetricsEnabled) { + try { + switch (QueryMetricsEnrichmentContext.getMethodType()) { + case CREATE: + baseQueryMetric.setCreateCallTime(responseStats.getCallTime()); + baseQueryMetric.setLoginTime(responseStats.getLoginTime()); + break; + case CREATE_AND_NEXT: + baseQueryMetric.setCreateCallTime(responseStats.getCallTime()); + baseQueryMetric.setLoginTime(responseStats.getLoginTime()); + List pageTimes = baseQueryMetric.getPageTimes(); + if (pageTimes != null && !pageTimes.isEmpty()) { + BaseQueryMetric.PageMetric pm = pageTimes.get(pageTimes.size() - 1); + pm.setCallTime(responseStats.getCallTime()); + pm.setLoginTime(responseStats.getLoginTime()); + pm.setSerializationTime(responseStats.getSerializationTime()); + pm.setBytesWritten(responseStats.getBytesWritten()); + } + break; + case NEXT: + pageTimes = baseQueryMetric.getPageTimes(); + if (pageTimes != null && !pageTimes.isEmpty()) { + BaseQueryMetric.PageMetric pm = pageTimes.get(pageTimes.size() - 1); + pm.setCallTime(responseStats.getCallTime()); + pm.setLoginTime(responseStats.getLoginTime()); + pm.setSerializationTime(responseStats.getSerializationTime()); + pm.setBytesWritten(responseStats.getBytesWritten()); + } + break; } - } else { - log.error("RunningQuery instance not in the cache!, queryId: " + queryId); + + // @formatter:off + queryMetricClient.submit( + new QueryMetricClient.Request.Builder() + .withMetric(baseQueryMetric) + .build()); + // @formatter:on + } catch (Exception e) { + log.error("Unable to record metrics for query '" + QueryMetricsEnrichmentContext.getQueryId() + "' and method '" + + QueryMetricsEnrichmentContext.getMethodType() + "': " + e.getLocalizedMessage(), e); } - } else { - log.error("Query cache not injected! No metrics will be recorded for serialization times."); } } } diff --git a/query-microservices/query-service/service/src/main/resources/config/bootstrap.yml b/query-microservices/query-service/service/src/main/resources/config/bootstrap.yml index 2f8ddb10..d2e4730f 100644 --- a/query-microservices/query-service/service/src/main/resources/config/bootstrap.yml +++ b/query-microservices/query-service/service/src/main/resources/config/bootstrap.yml @@ -19,7 +19,7 @@ spring: # For the dev profile, check localhost for the config server by default spring: - profiles: 'dev' + config.activate.on-profile: 'dev' cloud: config: uri: '${CONFIG_SERVER_URL:http://localhost:8888/configserver}' @@ -27,7 +27,7 @@ spring: --- spring: - profiles: 'consul' + config.activate.on-profile: 'consul' cloud: config: # Use Consul to locate the configuration server and bootstrap app config. @@ -44,7 +44,7 @@ spring: # For the "No Messaging" profile, we need to disable the AMQP bus, our custom RabbitMQ discovery, and the RabbitMQ health indicator. spring: - profiles: 'nomessaging' + config.activate.on-profile: 'nomessaging' cloud: bus: enabled: false diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java new file mode 100644 index 00000000..bf05fae4 --- /dev/null +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java @@ -0,0 +1,530 @@ +package datawave.microservice.query; + +import datawave.marking.ColumnVisibilitySecurityMarking; +import datawave.microservice.audit.AuditClient; +import datawave.microservice.authorization.jwt.JWTRestTemplate; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.query.config.QueryProperties; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.microservice.query.storage.QueryStorageCache; +import datawave.microservice.query.storage.Result; +import datawave.microservice.query.storage.TaskKey; +import datawave.microservice.query.storage.TaskStates; +import datawave.microservice.query.storage.queue.TestQueryQueueManager; +import datawave.security.authorization.DatawaveUser; +import datawave.security.authorization.SubjectIssuerDNPair; +import datawave.webservice.query.Query; +import datawave.webservice.query.exception.QueryExceptionType; +import datawave.webservice.query.result.event.DefaultEvent; +import datawave.webservice.query.result.event.DefaultField; +import datawave.webservice.result.BaseResponse; +import datawave.webservice.result.DefaultEventQueryResponse; +import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.QueryImplListResponse; +import datawave.webservice.result.QueryLogicResponse; +import datawave.webservice.result.VoidResponse; +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URLEncodedUtils; +import org.junit.Assert; +import org.springframework.beans.DirectFieldAccessor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.boot.web.server.LocalServerPort; +import org.springframework.cloud.bus.event.RemoteQueryRequestEvent; +import org.springframework.context.ApplicationEvent; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.context.annotation.Profile; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.http.client.ClientHttpResponse; +import org.springframework.test.web.client.MockRestServiceServer; +import org.springframework.test.web.client.RequestMatcher; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.client.DefaultResponseErrorHandler; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponents; +import org.springframework.web.util.UriComponentsBuilder; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.stream.Collectors; + +import static datawave.microservice.query.QueryParameters.QUERY_MAX_CONCURRENT_TASKS; +import static datawave.microservice.query.QueryParameters.QUERY_MAX_RESULTS_OVERRIDE; +import static datawave.microservice.query.QueryParameters.QUERY_PAGESIZE; +import static datawave.security.authorization.DatawaveUser.UserType.USER; +import static datawave.webservice.common.audit.AuditParameters.AUDIT_ID; +import static org.springframework.test.web.client.ExpectedCount.never; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.anything; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; + +public abstract class AbstractQueryServiceTest { + protected static final String EXPECTED_AUDIT_URI = "http://localhost:11111/audit/v1/audit"; + protected static final String TEST_QUERY_STRING = "FIELD:SOME_VALUE"; + protected static final String TEST_QUERY_NAME = "The Greatest Query in the World - Tribute"; + protected static final String TEST_QUERY_AUTHORIZATIONS = "ALL"; + protected static final String TEST_QUERY_BEGIN = "20000101 000000.000"; + protected static final String TEST_QUERY_END = "20500101 000000.000"; + protected static final String TEST_VISIBILITY_MARKING = "ALL"; + + @LocalServerPort + protected int webServicePort; + + @Autowired + protected RestTemplateBuilder restTemplateBuilder; + + protected JWTRestTemplate jwtRestTemplate; + + protected SubjectIssuerDNPair DN; + protected String userDN = "userDn"; + + protected SubjectIssuerDNPair altDN; + protected String altUserDN = "altUserDN"; + + @Autowired + protected QueryStorageCache queryStorageCache; + + @Autowired + protected TestQueryQueueManager queryQueueManager; + + @Autowired + protected AuditClient auditClient; + + @Autowired + protected QueryProperties queryProperties; + + @Autowired + protected LinkedList queryRequestEvents; + + protected List auditIds; + protected MockRestServiceServer mockServer; + + public void setup() { + auditIds = new ArrayList<>(); + + jwtRestTemplate = restTemplateBuilder.build(JWTRestTemplate.class); + jwtRestTemplate.setErrorHandler(new NoOpResponseErrorHandler()); + DN = SubjectIssuerDNPair.of(userDN, "issuerDn"); + altDN = SubjectIssuerDNPair.of(altUserDN, "issuerDN"); + + RestTemplate auditorRestTemplate = (RestTemplate) new DirectFieldAccessor(auditClient).getPropertyValue("jwtRestTemplate"); + mockServer = MockRestServiceServer.createServer(auditorRestTemplate); + + queryRequestEvents.clear(); + } + + public void teardown() throws Exception { + queryStorageCache.clear(); + queryRequestEvents.clear(); + queryQueueManager.clear(); + } + + protected void publishEventsToQueue(String queryId, int numEvents, MultiValueMap fieldValues, String visibility) throws Exception { + for (int resultId = 0; resultId < numEvents; resultId++) { + DefaultEvent[] events = new DefaultEvent[1]; + events[0] = new DefaultEvent(); + long currentTime = System.currentTimeMillis(); + List fields = new ArrayList<>(); + for (Map.Entry> entry : fieldValues.entrySet()) { + for (String value : entry.getValue()) { + fields.add(new DefaultField(entry.getKey(), visibility, currentTime, value)); + } + } + events[0].setFields(fields); + queryQueueManager.sendMessage(queryId, new Result(Integer.toString(resultId), events)); + } + } + + protected String createQuery(ProxiedUserDetails authUser, MultiValueMap map) { + return newQuery(authUser, map, "create"); + } + + protected String defineQuery(ProxiedUserDetails authUser, MultiValueMap map) { + return newQuery(authUser, map, "define"); + } + + protected String newQuery(ProxiedUserDetails authUser, MultiValueMap map, String createOrDefine) { + UriComponents uri = createUri("EventQuery/" + createOrDefine); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + + return (String) resp.getBody().getResult(); + } + + protected Future> nextQuery(ProxiedUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId + "/next"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, BaseResponse.class)); + } + + protected Future> adminCloseQuery(ProxiedUserDetails authUser, String queryId) { + return stopQuery(authUser, queryId, "adminClose"); + } + + protected Future> closeQuery(ProxiedUserDetails authUser, String queryId) { + return stopQuery(authUser, queryId, "close"); + } + + protected Future> adminCancelQuery(ProxiedUserDetails authUser, String queryId) { + return stopQuery(authUser, queryId, "adminCancel"); + } + + protected Future> cancelQuery(ProxiedUserDetails authUser, String queryId) { + return stopQuery(authUser, queryId, "cancel"); + } + + protected Future> stopQuery(ProxiedUserDetails authUser, String queryId, String closeOrCancel) { + UriComponents uri = createUri(queryId + "/" + closeOrCancel); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + protected Future> adminCloseAllQueries(ProxiedUserDetails authUser) { + UriComponents uri = createUri("/adminCloseAll"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + protected Future> adminCancelAllQueries(ProxiedUserDetails authUser) { + UriComponents uri = createUri("/adminCancelAll"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + protected Future> resetQuery(ProxiedUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId + "/reset"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, GenericResponse.class)); + } + + protected Future> removeQuery(ProxiedUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId + "/remove"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + protected Future> adminRemoveQuery(ProxiedUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId + "/adminRemove"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + protected Future> adminRemoveAllQueries(ProxiedUserDetails authUser) { + UriComponents uri = createUri("/adminRemoveAll"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + } + + protected Future> updateQuery(ProxiedUserDetails authUser, String queryId, MultiValueMap map) { + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, GenericResponse.class)); + } + + protected Future> duplicateQuery(ProxiedUserDetails authUser, String queryId, MultiValueMap map) { + UriComponents uri = createUri(queryId + "/duplicate"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // make the update call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, GenericResponse.class)); + } + + protected Future> listQueries(ProxiedUserDetails authUser, String queryId, String queryName) { + UriComponentsBuilder uriBuilder = uriBuilder("/list"); + if (queryId != null) { + uriBuilder.queryParam("queryId", queryId); + } + if (queryName != null) { + uriBuilder.queryParam("queryName", queryName); + } + UriComponents uri = uriBuilder.build(); + + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, QueryImplListResponse.class)); + } + + protected Future> getQuery(ProxiedUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId); + + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, QueryImplListResponse.class)); + } + + protected Future> listQueryLogic(ProxiedUserDetails authUser) { + UriComponents uri = createUri("/listQueryLogic"); + + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, QueryLogicResponse.class)); + } + + protected Future> adminListQueries(ProxiedUserDetails authUser, String queryId, String user, String queryName) { + UriComponentsBuilder uriBuilder = uriBuilder("/adminList"); + if (queryId != null) { + uriBuilder.queryParam("queryId", queryId); + } + if (queryName != null) { + uriBuilder.queryParam("queryName", queryName); + } + if (user != null) { + uriBuilder.queryParam("user", user); + } + UriComponents uri = uriBuilder.build(); + + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, QueryImplListResponse.class)); + } + + protected ProxiedUserDetails createUserDetails() { + return createUserDetails(null, null); + } + + protected ProxiedUserDetails createUserDetails(Collection roles, Collection auths) { + Collection userRoles = roles != null ? roles : Collections.singleton("AuthorizedUser"); + Collection userAuths = auths != null ? auths : Collections.singleton("ALL"); + DatawaveUser datawaveUser = new DatawaveUser(DN, USER, userAuths, userRoles, null, System.currentTimeMillis()); + return new ProxiedUserDetails(Collections.singleton(datawaveUser), datawaveUser.getCreationTime()); + } + + protected ProxiedUserDetails createAltUserDetails() { + return createAltUserDetails(null, null); + } + + protected ProxiedUserDetails createAltUserDetails(Collection roles, Collection auths) { + Collection userRoles = roles != null ? roles : Collections.singleton("AuthorizedUser"); + Collection userAuths = auths != null ? auths : Collections.singleton("ALL"); + DatawaveUser datawaveUser = new DatawaveUser(altDN, USER, userAuths, userRoles, null, System.currentTimeMillis()); + return new ProxiedUserDetails(Collections.singleton(datawaveUser), datawaveUser.getCreationTime()); + } + + protected UriComponentsBuilder uriBuilder(String path) { + return UriComponentsBuilder.newInstance().scheme("https").host("localhost").port(webServicePort).path("/query/v1/" + path); + } + + protected UriComponents createUri(String path) { + return uriBuilder(path).build(); + } + + protected MultiValueMap createParams() { + MultiValueMap map = new LinkedMultiValueMap<>(); + map.set(DefaultQueryParameters.QUERY_STRING, TEST_QUERY_STRING); + map.set(DefaultQueryParameters.QUERY_NAME, TEST_QUERY_NAME); + map.set(DefaultQueryParameters.QUERY_AUTHORIZATIONS, TEST_QUERY_AUTHORIZATIONS); + map.set(DefaultQueryParameters.QUERY_BEGIN, TEST_QUERY_BEGIN); + map.set(DefaultQueryParameters.QUERY_END, TEST_QUERY_END); + map.set(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, TEST_VISIBILITY_MARKING); + map.set(QUERY_MAX_CONCURRENT_TASKS, Integer.toString(1)); + map.set(QUERY_MAX_RESULTS_OVERRIDE, Long.toString(369)); + map.set(QUERY_PAGESIZE, Long.toString(123)); + return map; + } + + protected void assertDefaultEvent(List fields, List values, DefaultEvent event) { + Assert.assertEquals(fields, event.getFields().stream().map(DefaultField::getName).collect(Collectors.toList())); + Assert.assertEquals(values, event.getFields().stream().map(DefaultField::getValueString).collect(Collectors.toList())); + } + + protected void assertQueryResponse(String queryId, String logicName, long pageNumber, boolean partialResults, long operationTimeInMS, int numFields, + List fieldNames, int numEvents, DefaultEventQueryResponse queryResponse) { + Assert.assertEquals(queryId, queryResponse.getQueryId()); + Assert.assertEquals(logicName, queryResponse.getLogicName()); + Assert.assertEquals(pageNumber, queryResponse.getPageNumber()); + Assert.assertEquals(partialResults, queryResponse.isPartialResults()); + Assert.assertEquals(operationTimeInMS, queryResponse.getOperationTimeMS()); + Assert.assertEquals(numFields, queryResponse.getFields().size()); + Assert.assertEquals(fieldNames, queryResponse.getFields()); + Assert.assertEquals(numEvents, queryResponse.getEvents().size()); + } + + protected void assertQueryRequestEvent(String destination, QueryRequest.Method method, String queryId, RemoteQueryRequestEvent queryRequestEvent) { + Assert.assertEquals(destination, queryRequestEvent.getDestinationService()); + Assert.assertEquals(queryId, queryRequestEvent.getRequest().getQueryId()); + Assert.assertEquals(method, queryRequestEvent.getRequest().getMethod()); + } + + protected void assertQueryStatus(QueryStatus.QUERY_STATE queryState, long numResultsReturned, long numResultsGenerated, long activeNextCalls, + long lastPageNumber, long lastCallTimeMillis, QueryStatus queryStatus) { + Assert.assertEquals(queryState, queryStatus.getQueryState()); + Assert.assertEquals(numResultsReturned, queryStatus.getNumResultsReturned()); + Assert.assertEquals(numResultsGenerated, queryStatus.getNumResultsGenerated()); + Assert.assertEquals(activeNextCalls, queryStatus.getActiveNextCalls()); + Assert.assertEquals(lastPageNumber, queryStatus.getLastPageNumber()); + Assert.assertTrue(queryStatus.getLastUsedMillis() > lastCallTimeMillis); + Assert.assertTrue(queryStatus.getLastUpdatedMillis() > lastCallTimeMillis); + } + + protected void assertQuery(String queryString, String queryName, String authorizations, String begin, String end, String visibility, Query query) + throws ParseException { + SimpleDateFormat sdf = new SimpleDateFormat(DefaultQueryParameters.formatPattern); + Assert.assertEquals(queryString, query.getQuery()); + Assert.assertEquals(queryName, query.getQueryName()); + Assert.assertEquals(authorizations, query.getQueryAuthorizations()); + Assert.assertEquals(sdf.parse(begin), query.getBeginDate()); + Assert.assertEquals(sdf.parse(end), query.getEndDate()); + Assert.assertEquals(visibility, query.getColumnVisibility()); + } + + protected void assertTasksCreated(String queryId) throws IOException { + // verify that the query task states were created + TaskStates taskStates = queryStorageCache.getTaskStates(queryId); + Assert.assertNotNull(taskStates); + + // verify that a query task was created + List taskKeys = queryStorageCache.getTasks(queryId); + Assert.assertFalse(taskKeys.isEmpty()); + } + + protected void assertTasksNotCreated(String queryId) throws IOException { + // verify that the query task states were not created + TaskStates taskStates = queryStorageCache.getTaskStates(queryId); + Assert.assertNull(taskStates); + + // verify that a query task was not created + List taskKeys = queryStorageCache.getTasks(queryId); + Assert.assertTrue(taskKeys.isEmpty()); + } + + public RequestMatcher auditIdGrabber() { + return request -> { + List params = URLEncodedUtils.parse(request.getBody().toString(), Charset.defaultCharset()); + params.stream().filter(p -> p.getName().equals(AUDIT_ID)).forEach(p -> auditIds.add(p.getValue())); + }; + } + + protected void auditIgnoreSetup() { + mockServer.expect(anything()).andRespond(withSuccess()); + } + + protected void auditSentSetup() { + mockServer.expect(requestTo(EXPECTED_AUDIT_URI)).andExpect(auditIdGrabber()).andRespond(withSuccess()); + } + + protected void auditNotSentSetup() { + mockServer.expect(never(), requestTo(EXPECTED_AUDIT_URI)).andExpect(auditIdGrabber()).andRespond(withSuccess()); + } + + protected void assertAuditSent(String queryId) { + mockServer.verify(); + Assert.assertEquals(1, auditIds.size()); + Assert.assertEquals(queryId, auditIds.get(0)); + } + + protected void assertAuditNotSent() { + mockServer.verify(); + Assert.assertEquals(0, auditIds.size()); + } + + protected void assertQueryException(String message, String cause, String code, QueryExceptionType queryException) { + Assert.assertEquals(message, queryException.getMessage()); + Assert.assertEquals(cause, queryException.getCause()); + Assert.assertEquals(code, queryException.getCode()); + } + + protected BaseResponse assertBaseResponse(boolean hasResults, HttpStatus.Series series, ResponseEntity response) { + Assert.assertEquals(series, response.getStatusCode().series()); + Assert.assertNotNull(response); + BaseResponse baseResponse = response.getBody(); + Assert.assertNotNull(baseResponse); + Assert.assertEquals(hasResults, baseResponse.getHasResults()); + return baseResponse; + } + + @SuppressWarnings("unchecked") + protected GenericResponse assertGenericResponse(boolean hasResults, HttpStatus.Series series, ResponseEntity response) { + Assert.assertEquals(series, response.getStatusCode().series()); + Assert.assertNotNull(response); + GenericResponse genericResponse = (GenericResponse) response.getBody(); + Assert.assertNotNull(genericResponse); + Assert.assertEquals(hasResults, genericResponse.getHasResults()); + return genericResponse; + } + + protected static class NoOpResponseErrorHandler extends DefaultResponseErrorHandler { + @Override + public void handleError(ClientHttpResponse response) throws IOException { + // do nothing + } + } + + @Configuration + @Profile("QueryServiceTest") + @ComponentScan(basePackages = "datawave.microservice") + public static class QueryServiceTestConfiguration { + @Bean + public LinkedList queryRequestEvents() { + return new LinkedList<>(); + } + + @Bean + @Primary + public ApplicationEventPublisher eventPublisher(ApplicationEventPublisher eventPublisher) { + return new ApplicationEventPublisher() { + @Override + public void publishEvent(ApplicationEvent event) { + saveEvent(event); + } + + @Override + public void publishEvent(Object event) { + saveEvent(event); + } + + private void saveEvent(Object event) { + if (event instanceof RemoteQueryRequestEvent) { + queryRequestEvents().push(((RemoteQueryRequestEvent) event)); + } + } + }; + } + } +} diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java new file mode 100644 index 00000000..56d80ffe --- /dev/null +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java @@ -0,0 +1,512 @@ +package datawave.microservice.query; + +import com.google.common.collect.Iterables; +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.result.BaseResponse; +import datawave.webservice.result.VoidResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.web.util.UriComponents; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +public class QueryServiceCancelTest extends AbstractQueryServiceTest { + @Before + public void setup() { + super.setup(); + } + + @After + public void teardown() throws Exception { + super.teardown(); + } + + @Test + public void testCancelSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCELED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the result queue is gone + Assert.assertFalse(queryQueueManager.queueExists(queryId)); + + // verify that the query tasks are still present + assertTasksCreated(queryId); + + // verify that the cancel event was published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testCancelSuccess_activeNextCall() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // call next on the query + Future> nextFuture = nextQuery(authUser, queryId); + + boolean nextCallActive = queryStorageCache.getQueryStatus(queryId).getActiveNextCalls() > 0; + while (!nextCallActive) { + try { + nextFuture.get(500, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + nextCallActive = queryStorageCache.getQueryStatus(queryId).getActiveNextCalls() > 0; + if (TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - currentTimeMillis) > 5) { + throw e; + } + } + } + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCELED, + 0, + 0, + 1, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the result queue is gone + Assert.assertFalse(queryQueueManager.queueExists(queryId)); + + // wait for the next call to return + nextFuture.get(); + + // verify that the query tasks are still present + assertTasksCreated(queryId); + + // verify that the close event was published + Assert.assertEquals(4, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testCancelFailure_queryNotFound() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(404, cancelResponse.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(cancelResponse.getBody().getExceptions())); + // @formatter:on + + } + + @Test + public void testCancelFailure_ownershipFailure() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails altAuthUser = createAltUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // make the cancel call as an alternate user asynchronously + Future> future = cancelQuery(altAuthUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(401, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that the next events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testCancelFailure_queryNotRunning() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // try to cancel the query again + cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + cancelResponse = cancelFuture.get(); + + Assert.assertEquals(400, cancelResponse.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Cannot call cancel on a query that is not running", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(cancelResponse.getBody().getExceptions())); + // @formatter:on + + // verify that the next events were published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testAdminCancelSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails adminUser = createAltUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // cancel the query as the admin user + Future> cancelFuture = adminCancelQuery(adminUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCELED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the result queue is gone + Assert.assertFalse(queryQueueManager.queueExists(queryId)); + + // verify that the query tasks are still present + assertTasksCreated(queryId); + + // verify that the cancel event was published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testAdminCancelFailure_notAdminUser() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + UriComponents uri = createUri(queryId + "/adminCancel"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // cancel the query + Future> closeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(403, closeResponse.getStatusCodeValue()); + + // verify that the create event was published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testAdminCancelAllSuccess() throws Exception { + ProxiedUserDetails adminUser = createUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + // create a bunch of queries + List queryIds = new ArrayList<>(); + long currentTimeMillis = System.currentTimeMillis(); + for (int i = 0; i < 10; i++) { + String queryId = createQuery(adminUser, createParams()); + mockServer.reset(); + + queryIds.add(queryId); + + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + // close all queries as the admin user + Future> cancelFuture = adminCancelAllQueries(adminUser); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // verify that query status was created correctly + List queryStatusList = queryStorageCache.getQueryStatus(); + + for (QueryStatus queryStatus : queryStatusList) { + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCELED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + String queryId = queryStatus.getQueryKey().getQueryId(); + + // verify that the result queue is gone + Assert.assertFalse(queryQueueManager.queueExists(queryStatus.getQueryKey().getQueryId())); + + // verify that the query tasks are still present + assertTasksCreated(queryStatus.getQueryKey().getQueryId()); + + // @formatter:off + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + // verify that there are no more events + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testAdminCancelAllFailure_notAdminUser() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a bunch of queries + List queryIds = new ArrayList<>(); + long currentTimeMillis = System.currentTimeMillis(); + for (int i = 0; i < 10; i++) { + String queryId = createQuery(authUser, createParams()); + mockServer.reset(); + + queryIds.add(queryId); + + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + // cancel all queries as the admin user + UriComponents uri = createUri("/adminCancelAll"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + Future> cancelFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(403, cancelResponse.getStatusCodeValue()); + + // verify that query status was created correctly + List queryStatusList = queryStorageCache.getQueryStatus(); + + // verify that none of the queries were canceled + for (QueryStatus queryStatus : queryStatusList) { + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the result queue is still present + Assert.assertTrue(queryQueueManager.queueExists(queryStatus.getQueryKey().getQueryId())); + + // verify that the query tasks are still present + assertTasksCreated(queryStatus.getQueryKey().getQueryId()); + } + + // verify that there are no more events + Assert.assertEquals(0, queryRequestEvents.size()); + } +} diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java new file mode 100644 index 00000000..88e92730 --- /dev/null +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java @@ -0,0 +1,507 @@ +package datawave.microservice.query; + +import com.google.common.collect.Iterables; +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.result.BaseResponse; +import datawave.webservice.result.VoidResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +public class QueryServiceCloseTest extends AbstractQueryServiceTest { + @Before + public void setup() { + super.setup(); + } + + @After + public void teardown() throws Exception { + super.teardown(); + } + + @Test + public void testCloseSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CLOSED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the result queue is gone + Assert.assertFalse(queryQueueManager.queueExists(queryId)); + + // verify that the query tasks are still present + assertTasksCreated(queryId); + + // verify that the close event was published + Assert.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testCloseSuccess_activeNextCall() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // call next on the query + Future> nextFuture = nextQuery(authUser, queryId); + + boolean nextCallActive = queryStorageCache.getQueryStatus(queryId).getActiveNextCalls() > 0; + while (!nextCallActive) { + try { + nextFuture.get(500, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + nextCallActive = queryStorageCache.getQueryStatus(queryId).getActiveNextCalls() > 0; + if (TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - currentTimeMillis) > 5) { + throw e; + } + } + } + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CLOSED, + 0, + 0, + 1, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the result queue is still present + Assert.assertTrue(queryQueueManager.queueExists(queryId)); + + // send enough results to return a page + // pump enough results into the queue to trigger a complete page + int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("LOKI", "ALLIGATOR"); + fieldValues.add("LOKI", "CLASSIC"); + + // @formatter:off + publishEventsToQueue( + queryId, + pageSize, + fieldValues, + "ALL"); + // @formatter:on + + // wait for the next call to return + nextFuture.get(); + + // verify that the result queue is now gone + Assert.assertFalse(queryQueueManager.queueExists(queryId)); + + // verify that the query tasks are still present + assertTasksCreated(queryId); + + // verify that the close event was published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testCloseFailure_queryNotFound() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(404, closeResponse.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(closeResponse.getBody().getExceptions())); + // @formatter:on + + } + + @Test + public void testCloseFailure_ownershipFailure() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails altAuthUser = createAltUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // make the close call as an alternate user asynchronously + Future> future = closeQuery(altAuthUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(401, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that the next events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testCloseFailure_queryNotRunning() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + + // try to close the query again + closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + closeResponse = closeFuture.get(); + + Assert.assertEquals(400, closeResponse.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Cannot call close on a query that is not running", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(closeResponse.getBody().getExceptions())); + // @formatter:on + + // verify that the next events were published + Assert.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testAdminCloseSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails adminUser = createAltUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // close the query as the admin user + Future> closeFuture = adminCloseQuery(adminUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CLOSED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the result queue is gone + Assert.assertFalse(queryQueueManager.queueExists(queryId)); + + // verify that the query tasks are still present + assertTasksCreated(queryId); + + // verify that the close event was published + Assert.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testAdminCloseFailure_notAdminUser() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + UriComponents uri = createUri(queryId + "/adminClose"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // close the query + Future> closeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(403, closeResponse.getStatusCodeValue()); + + // verify that the create event was published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testAdminCloseAllSuccess() throws Exception { + ProxiedUserDetails adminUser = createUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + // create a bunch of queries + long currentTimeMillis = System.currentTimeMillis(); + for (int i = 0; i < 10; i++) { + String queryId = createQuery(adminUser, createParams()); + mockServer.reset(); + + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + // close all queries as the admin user + Future> closeFuture = adminCloseAllQueries(adminUser); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + List queryStatusList = queryStorageCache.getQueryStatus(); + + for (QueryStatus queryStatus : queryStatusList) { + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CLOSED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + String queryId = queryStatus.getQueryKey().getQueryId(); + + // verify that the result queue is gone + Assert.assertFalse(queryQueueManager.queueExists(queryStatus.getQueryKey().getQueryId())); + + // verify that the query tasks are still present + assertTasksCreated(queryStatus.getQueryKey().getQueryId()); + + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + // verify that there are no more events + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testAdminCloseAllFailure_notAdminUser() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a bunch of queries + List queryIds = new ArrayList<>(); + long currentTimeMillis = System.currentTimeMillis(); + for (int i = 0; i < 10; i++) { + String queryId = createQuery(authUser, createParams()); + mockServer.reset(); + + queryIds.add(queryId); + + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + // close all queries as the admin user + UriComponents uri = createUri("/adminCloseAll"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // make the next call asynchronously + Future> closeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(403, closeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + List queryStatusList = queryStorageCache.getQueryStatus(); + + // verify that none of the queries were canceled + for (QueryStatus queryStatus : queryStatusList) { + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that the result queue is still present + Assert.assertTrue(queryQueueManager.queueExists(queryStatus.getQueryKey().getQueryId())); + + // verify that the query tasks are still present + assertTasksCreated(queryStatus.getQueryKey().getQueryId()); + } + + // verify that there are no more events + Assert.assertEquals(0, queryRequestEvents.size()); + } +} diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java new file mode 100644 index 00000000..51d5ac0f --- /dev/null +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java @@ -0,0 +1,478 @@ +package datawave.microservice.query; + +import datawave.marking.ColumnVisibilitySecurityMarking; +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.query.Query; +import datawave.webservice.query.exception.QueryExceptionType; +import datawave.webservice.result.BaseResponse; +import datawave.webservice.result.GenericResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; + +import java.io.IOException; +import java.text.ParseException; +import java.util.Arrays; +import java.util.Collections; + +import static datawave.microservice.query.QueryParameters.QUERY_MAX_CONCURRENT_TASKS; +import static datawave.microservice.query.QueryParameters.QUERY_MAX_RESULTS_OVERRIDE; +import static datawave.microservice.query.QueryParameters.QUERY_PAGESIZE; +import static datawave.webservice.common.audit.AuditParameters.QUERY_STRING; +import static datawave.webservice.query.QueryImpl.BEGIN_DATE; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +public class QueryServiceCreateTest extends AbstractQueryServiceTest { + @Before + public void setup() { + super.setup(); + } + + @After + public void teardown() throws Exception { + super.teardown(); + } + + @Test + public void testCreateSuccess() throws ParseException, IOException { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditSentSetup(); + + long currentTimeMillis = System.currentTimeMillis(); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + true, + HttpStatus.Series.SUCCESSFUL, + resp); + // @formatter:on + + // verify that a query id was returned + String queryId = genericResponse.getResult(); + Assert.assertNotNull(queryId); + + // verify that the create event was published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that they query was created correctly + Query query = queryStatus.getQuery(); + + // @formatter:off + assertQuery( + TEST_QUERY_STRING, + TEST_QUERY_NAME, + TEST_QUERY_AUTHORIZATIONS, + TEST_QUERY_BEGIN, + TEST_QUERY_END, + TEST_VISIBILITY_MARKING, + query); + // @formatter:on + + // verify that an audit message was sent and the the audit id matches the query id + assertAuditSent(queryId); + + // verify that the results queue was created + Assert.assertTrue(queryQueueManager.queueExists(queryId)); + + // verify that query tasks were created + assertTasksCreated(queryId); + } + + @Test + public void testCreateFailure_paramValidation() { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + // remove the query param to induce a parameter validation failure + map.remove(QUERY_STRING); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assert.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Missing one or more required QueryParameters", + "java.lang.IllegalArgumentException: Missing one or more required QueryParameters", + "400-1", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + @Test + public void testCreateFailure_authValidation() { + ProxiedUserDetails authUser = createUserDetails(Collections.singleton("AuthorizedUser"), Collections.emptyList()); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assert.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "User requested authorizations that they don't have. Missing: [ALL], Requested: [ALL], User: []", + "java.lang.IllegalArgumentException: User requested authorizations that they don't have. Missing: [ALL], Requested: [ALL], User: []", + "400-1", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + @Test + public void testCreateFailure_queryLogicValidation() { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + // remove the beginDate param to induce a query logic validation failure + map.remove(BEGIN_DATE); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assert.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Required parameter begin not found", + "java.lang.IllegalArgumentException: Required parameter begin not found", + "400-1", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + @Test + public void testCreateFailure_maxPageSize() { + ProxiedUserDetails authUser = createUserDetails(Arrays.asList("AuthorizedUser", queryProperties.getPrivilegedRole()), null); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + // set an invalid page size override + map.set(QUERY_PAGESIZE, Integer.toString(Integer.MAX_VALUE)); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assert.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Page size is larger than configured max. Max = 10,000.", + "Exception with no cause caught", + "400-6", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + @Test + public void testCreateFailure_maxResultsOverride() { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + // set an invalid max results override + map.set(QUERY_MAX_RESULTS_OVERRIDE, Long.toString(Long.MAX_VALUE)); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assert.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Invalid max results override value. Max = 369.", + "Exception with no cause caught", + "400-43", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + @Test + public void testCreateFailure_maxConcurrentTasksOverride() { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + // add an invalid max results override + map.set(QUERY_MAX_CONCURRENT_TASKS, Integer.toString(Integer.MAX_VALUE)); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assert.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Invalid max concurrent tasks override value. Max = 10.", + "Exception with no cause caught", + "400-44", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + @Test + public void testCreateFailure_roleValidation() { + // create a user without the required role + ProxiedUserDetails authUser = createUserDetails(Collections.emptyList(), Collections.singletonList("ALL")); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assert.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "User does not have the required roles.", + "datawave.webservice.query.exception.UnauthorizedQueryException: User does not have the required roles.", + "400-5", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } + + @Test + public void testCreateFailure_markingValidation() { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/create"); + MultiValueMap map = createParams(); + + // remove the column visibility param to induce a security marking validation failure + map.remove(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assert.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Required parameter columnVisibility not found", + "java.lang.IllegalArgumentException: Required parameter columnVisibility not found", + "400-4", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + + // verify that no audit message was sent + assertAuditNotSent(); + } +} diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java new file mode 100644 index 00000000..bc75e46e --- /dev/null +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java @@ -0,0 +1,418 @@ +package datawave.microservice.query; + +import datawave.marking.ColumnVisibilitySecurityMarking; +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.query.Query; +import datawave.webservice.query.exception.QueryExceptionType; +import datawave.webservice.result.BaseResponse; +import datawave.webservice.result.GenericResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; + +import java.io.IOException; +import java.text.ParseException; +import java.util.Arrays; +import java.util.Collections; + +import static datawave.microservice.query.QueryParameters.QUERY_MAX_CONCURRENT_TASKS; +import static datawave.microservice.query.QueryParameters.QUERY_MAX_RESULTS_OVERRIDE; +import static datawave.microservice.query.QueryParameters.QUERY_PAGESIZE; +import static datawave.webservice.common.audit.AuditParameters.QUERY_STRING; +import static datawave.webservice.query.QueryImpl.BEGIN_DATE; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +public class QueryServiceDefineTest extends AbstractQueryServiceTest { + @Before + public void setup() { + super.setup(); + } + + @After + public void teardown() throws Exception { + super.teardown(); + } + + @Test + public void testDefineSuccess() throws ParseException, IOException { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditNotSentSetup(); + + long currentTimeMillis = System.currentTimeMillis(); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + false, + HttpStatus.Series.SUCCESSFUL, + resp); + // @formatter:on + + // verify that a query id was returned + String queryId = genericResponse.getResult(); + Assert.assertNotNull(queryId); + + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.DEFINED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that they query was created correctly + Query query = queryStatus.getQuery(); + // @formatter:off + assertQuery( + TEST_QUERY_STRING, + TEST_QUERY_NAME, + TEST_QUERY_AUTHORIZATIONS, + TEST_QUERY_BEGIN, + TEST_QUERY_END, + TEST_VISIBILITY_MARKING, + query); + // @formatter:on + + // verify that no audit message was sent + assertAuditNotSent(); + + // verify that the results queue wasn't created + Assert.assertFalse(queryQueueManager.queueExists(queryId)); + + // verify that query tasks weren't created + assertTasksNotCreated(queryId); + } + + @Test + public void testDefineFailure_paramValidation() { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + // remove the query param to induce a parameter validation failure + map.remove(QUERY_STRING); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assert.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Missing one or more required QueryParameters", + "java.lang.IllegalArgumentException: Missing one or more required QueryParameters", + "400-1", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @Test + public void testDefineFailure_authValidation() { + ProxiedUserDetails authUser = createUserDetails(Collections.singleton("AuthorizedUser"), Collections.emptyList()); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assert.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "User requested authorizations that they don't have. Missing: [ALL], Requested: [ALL], User: []", + "java.lang.IllegalArgumentException: User requested authorizations that they don't have. Missing: [ALL], Requested: [ALL], User: []", + "400-1", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @Test + public void testDefineFailure_queryLogicValidation() { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + // remove the beginDate param to induce a query logic validation failure + map.remove(BEGIN_DATE); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assert.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Required parameter begin not found", + "java.lang.IllegalArgumentException: Required parameter begin not found", + "400-1", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @Test + public void testDefineFailure_maxPageSize() { + ProxiedUserDetails authUser = createUserDetails(Arrays.asList("AuthorizedUser", queryProperties.getPrivilegedRole()), null); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + // set an invalid page size override + map.set(QUERY_PAGESIZE, Integer.toString(Integer.MAX_VALUE)); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assert.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Page size is larger than configured max. Max = 10,000.", + "Exception with no cause caught", + "400-6", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @Test + public void testDefineFailure_maxResultsOverride() { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + // set an invalid max results override + map.set(QUERY_MAX_RESULTS_OVERRIDE, Long.toString(Long.MAX_VALUE)); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assert.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Invalid max results override value. Max = 369.", + "Exception with no cause caught", + "400-43", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @Test + public void testDefineFailure_maxConcurrentTasksOverride() { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + // add an invalid max results override + map.set(QUERY_MAX_CONCURRENT_TASKS, Integer.toString(Integer.MAX_VALUE)); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assert.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Invalid max concurrent tasks override value. Max = 10.", + "Exception with no cause caught", + "400-44", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @Test + public void testDefineFailure_roleValidation() { + // create a user without the required role + ProxiedUserDetails authUser = createUserDetails(Collections.emptyList(), Collections.singletonList("ALL")); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assert.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "User does not have the required roles.", + "datawave.webservice.query.exception.UnauthorizedQueryException: User does not have the required roles.", + "400-5", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } + + @Test + public void testDefineFailure_markingValidation() { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/define"); + MultiValueMap map = createParams(); + + // remove the column visibility param to induce a security marking validation failure + map.remove(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); + + // @formatter:off + BaseResponse baseResponse = assertBaseResponse( + false, + HttpStatus.Series.CLIENT_ERROR, + resp); + // @formatter:on + + // verify that there is no result + Assert.assertFalse(baseResponse.getHasResults()); + + // verify that an exception was returned + Assert.assertEquals(1, baseResponse.getExceptions().size()); + + QueryExceptionType queryException = baseResponse.getExceptions().get(0); + // @formatter:off + assertQueryException( + "Required parameter columnVisibility not found", + "java.lang.IllegalArgumentException: Required parameter columnVisibility not found", + "400-4", + queryException); + // @formatter:on + + // verify that there are no query statuses + Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + } +} diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java new file mode 100644 index 00000000..2facee7c --- /dev/null +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java @@ -0,0 +1,588 @@ +package datawave.microservice.query; + +import com.google.common.collect.Iterables; +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.VoidResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; + +import java.util.Arrays; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; +import static datawave.webservice.common.audit.AuditParameters.QUERY_AUTHORIZATIONS; +import static datawave.webservice.query.QueryImpl.BEGIN_DATE; +import static datawave.webservice.query.QueryImpl.END_DATE; +import static datawave.webservice.query.QueryImpl.QUERY; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +public class QueryServiceDuplicateTest extends AbstractQueryServiceTest { + @Before + public void setup() { + super.setup(); + } + + @After + public void teardown() throws Exception { + super.teardown(); + } + + @Test + public void testDuplicateSuccess_duplicateOnDefined() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = defineQuery(authUser, createParams()); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + mockServer.reset(); + auditSentSetup(); + + // duplicate the query + Future> duplicateFuture = duplicateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + String dupeQueryId = (String) response.getBody().getResult(); + + // make sure an audit message was sent + assertAuditSent(dupeQueryId); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.DEFINED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + dupeQueryStatus); + // @formatter:on + + // make sure the queries are identical + Assert.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); + Assert.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); + Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); + Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); + Assert.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); + Assert.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); + + // verify that no events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + dupeQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testDuplicateSuccess_duplicateOnCreated() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + mockServer.reset(); + auditSentSetup(); + + // duplicate the query + Future> duplicateFuture = duplicateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + String dupeQueryId = (String) response.getBody().getResult(); + + // make sure an audit message was sent + assertAuditSent(dupeQueryId); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + dupeQueryStatus); + // @formatter:on + + // make sure the queries are identical + Assert.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); + Assert.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); + Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); + Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); + Assert.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); + Assert.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); + + // verify that no events were published + Assert.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + dupeQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testDuplicateSuccess_duplicateOnCanceled() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // this should return immediately + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + mockServer.reset(); + auditSentSetup(); + + // duplicate the query + Future> duplicateFuture = duplicateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + String dupeQueryId = (String) response.getBody().getResult(); + + // make sure an audit message was sent + assertAuditSent(dupeQueryId); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCELED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + dupeQueryStatus); + // @formatter:on + + // make sure the queries are identical + Assert.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); + Assert.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); + Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); + Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); + Assert.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); + Assert.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); + + // verify that no events were published + Assert.assertEquals(4, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + dupeQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testDuplicateSuccess_duplicateOnClosed() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // this should return immediately + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + mockServer.reset(); + auditSentSetup(); + + // duplicate the query + Future> duplicateFuture = duplicateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + String dupeQueryId = (String) response.getBody().getResult(); + + // make sure an audit message was sent + assertAuditSent(dupeQueryId); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CLOSED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + dupeQueryStatus); + // @formatter:on + + // make sure the queries are identical + Assert.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); + Assert.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); + Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); + Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), + DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); + Assert.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); + Assert.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); + + // verify that no events were published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + dupeQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testDuplicateSuccess_update() throws Exception { + ProxiedUserDetails authUser = createUserDetails(null, Arrays.asList("ALL", "NONE")); + + // define a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = defineQuery(authUser, createParams()); + + String newQuery = "SOME_OTHER_FIELD:SOME_OTHER_VALUE"; + String newAuths = "ALL,NONE"; + String newBegin = "20100101 000000.000"; + String newEnd = "20600101 000000.000"; + String newLogic = "AltEventQuery"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY, newQuery); + updateParams.set(QUERY_AUTHORIZATIONS, newAuths); + updateParams.set(BEGIN_DATE, newBegin); + updateParams.set(END_DATE, newEnd); + updateParams.set(QUERY_LOGIC_NAME, newLogic); + + mockServer.reset(); + auditSentSetup(); + + // duplicate the query + Future> duplicateFuture = duplicateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + String dupeQueryId = (String) response.getBody().getResult(); + + // make sure an audit message was sent + assertAuditSent(dupeQueryId); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.DEFINED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + dupeQueryStatus); + // @formatter:on + + // make sure the original query is unchanged + Assert.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); + Assert.assertEquals(TEST_QUERY_AUTHORIZATIONS, queryStatus.getQuery().getQueryAuthorizations()); + Assert.assertEquals(TEST_QUERY_BEGIN, DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate())); + Assert.assertEquals(TEST_QUERY_END, DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate())); + Assert.assertEquals("EventQuery", queryStatus.getQuery().getQueryLogicName()); + + // make sure the duplicated query is updated + Assert.assertEquals(newQuery, dupeQueryStatus.getQuery().getQuery()); + Assert.assertEquals(newAuths, dupeQueryStatus.getQuery().getQueryAuthorizations()); + Assert.assertEquals(newBegin, DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); + Assert.assertEquals(newEnd, DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); + Assert.assertEquals(newLogic, dupeQueryStatus.getQuery().getQueryLogicName()); + + // verify that no events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + dupeQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testDuplicateFailure_invalidUpdate() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + String newLogic = "SomeBogusLogic"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY_LOGIC_NAME, newLogic); + + mockServer.reset(); + auditNotSentSetup(); + + // duplicate the query + UriComponents uri = createUri(queryId + "/duplicate"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.POST, uri); + + // make the duplicate call asynchronously + Future> duplicateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + // make sure an audit message wasn't sent + assertAuditNotSent(); + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testDuplicateFailure_queryNotFound() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + mockServer.reset(); + auditNotSentSetup(); + + // duplicate the query + UriComponents uri = createUri(queryId + "/duplicate"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.POST, uri); + + // make the duplicate call asynchronously + Future> duplicateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assert.assertEquals(404, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // make sure an audit message wasn't sent + assertAuditNotSent(); + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testDuplicateFailure_ownershipFailure() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails altAuthUser = createAltUserDetails(); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + mockServer.reset(); + auditNotSentSetup(); + + // duplicate the query + UriComponents uri = createUri(queryId + "/duplicate"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(altAuthUser, updateParams, null, HttpMethod.POST, uri); + + // make the duplicate call asynchronously + Future> duplicateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = duplicateFuture.get(); + + Assert.assertEquals(401, response.getStatusCodeValue()); + + // make sure an audit message wasn't sent + assertAuditNotSent(); + + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was not updated + Assert.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } +} diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java new file mode 100644 index 00000000..31dca9f2 --- /dev/null +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java @@ -0,0 +1,428 @@ +package datawave.microservice.query; + +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.security.util.ProxiedEntityUtils; +import datawave.webservice.query.Query; +import datawave.webservice.query.result.logic.QueryLogicDescription; +import datawave.webservice.result.QueryImplListResponse; +import datawave.webservice.result.QueryLogicResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; +import org.springframework.web.util.UriComponentsBuilder; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.stream.Collectors; + +import static datawave.microservice.query.QueryParameters.QUERY_NAME; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +public class QueryServiceListTest extends AbstractQueryServiceTest { + @Before + public void setup() { + super.setup(); + } + + @After + public void teardown() throws Exception { + super.teardown(); + } + + @Test + public void testListSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails altAuthUser = createAltUserDetails(); + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + String queryId = createQuery(authUser, createParams()); + mockServer.reset(); + + queryIds.add(queryId); + } + + // define a bunch of queries as the alternate user + List altQueryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + String queryId = defineQuery(altAuthUser, createParams()); + mockServer.reset(); + + altQueryIds.add(queryId); + } + + // list queries as the original user + Future> listFuture = listQueries(authUser, null, null); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assert.assertEquals(5, result.getNumResults()); + + List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Collections.sort(queryIds); + Collections.sort(actualQueryIds); + + Assert.assertEquals(queryIds, actualQueryIds); + } + + @Test + public void testListSuccess_filterOnQueryId() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + String queryId = createQuery(authUser, createParams()); + mockServer.reset(); + + queryIds.add(queryId); + } + + // list queries + Future> listFuture = listQueries(authUser, queryIds.get(0), null); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assert.assertEquals(1, result.getNumResults()); + + List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + } + + @Test + public void testListSuccess_filterOnQueryName() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + String uniqueQueryName = "Unique Query"; + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + MultiValueMap params = createParams(); + if (i == 0) { + params.set(QUERY_NAME, uniqueQueryName); + } + + String queryId = createQuery(authUser, params); + mockServer.reset(); + + queryIds.add(queryId); + } + + // list queries + Future> listFuture = listQueries(authUser, null, uniqueQueryName); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assert.assertEquals(1, result.getNumResults()); + + List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + } + + @Test + public void testListSuccess_filterOnMultiple() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + String uniqueQueryName = "Unique Query"; + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + MultiValueMap params = createParams(); + if (i == 0) { + params.set(QUERY_NAME, uniqueQueryName); + } + + String queryId = createQuery(authUser, params); + mockServer.reset(); + + queryIds.add(queryId); + } + + // list queries with just the query ID and a bogus name + Future> listFuture = listQueries(authUser, queryIds.get(0), "bogus name"); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assert.assertEquals(0, result.getNumResults()); + + // list queries with just the query name and a bogus ID + listFuture = listQueries(authUser, UUID.randomUUID().toString(), uniqueQueryName); + + // this should return immediately + listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + result = listResponse.getBody(); + + Assert.assertEquals(0, result.getNumResults()); + + // list queries with just the query name and a bogus ID + listFuture = listQueries(authUser, queryIds.get(0), uniqueQueryName); + + // this should return immediately + listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + result = listResponse.getBody(); + + Assert.assertEquals(1, result.getNumResults()); + + List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + } + + @Test + public void testListFailure_ownershipFailure() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails altAuthUser = createAltUserDetails(); + + String uniqueQueryName = "Unique Query"; + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + MultiValueMap params = createParams(); + if (i == 0) { + params.set(QUERY_NAME, uniqueQueryName); + } + + String queryId = createQuery(authUser, params); + mockServer.reset(); + + queryIds.add(queryId); + } + + // list queries with just the query ID and a bogus name + Future> listFuture = listQueries(altAuthUser, queryIds.get(0), "bogus name"); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assert.assertEquals(0, result.getNumResults()); + + // list queries with just the query name and a bogus ID + listFuture = listQueries(altAuthUser, UUID.randomUUID().toString(), uniqueQueryName); + + // this should return immediately + listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + result = listResponse.getBody(); + + Assert.assertEquals(0, result.getNumResults()); + + // list queries with the query name and query ID + listFuture = listQueries(altAuthUser, queryIds.get(0), uniqueQueryName); + + // this should return immediately + listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + result = listResponse.getBody(); + + Assert.assertEquals(0, result.getNumResults()); + } + + @Test + public void testAdminListSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails adminUser = createAltUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + String user = ProxiedEntityUtils.getShortName(authUser.getPrimaryUser().getDn().subjectDN()); + + String uniqueQueryName = "Unique Query"; + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + MultiValueMap params = createParams(); + if (i == 0) { + params.set(QUERY_NAME, uniqueQueryName); + } + + String queryId = createQuery(authUser, params); + mockServer.reset(); + + queryIds.add(queryId); + } + + // list queries with just the query ID + Future> listFuture = adminListQueries(adminUser, queryIds.get(0), user, null); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assert.assertEquals(1, result.getNumResults()); + + List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + + // list queries with just the query name + listFuture = adminListQueries(adminUser, null, user, uniqueQueryName); + + // this should return immediately + listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + result = listResponse.getBody(); + + Assert.assertEquals(1, result.getNumResults()); + + actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + + // list queries with the query name and query ID + listFuture = adminListQueries(adminUser, queryIds.get(0), user, uniqueQueryName); + + // this should return immediately + listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + result = listResponse.getBody(); + + Assert.assertEquals(1, result.getNumResults()); + + actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); + + Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + } + + @Test + public void testAdminListFailure_notAdminUser() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails altAuthUser = createAltUserDetails(); + + String user = ProxiedEntityUtils.getShortName(authUser.getPrimaryUser().getDn().subjectDN()); + + String uniqueQueryName = "Unique Query"; + + // define a bunch of queries as the original user + List queryIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + MultiValueMap params = createParams(); + if (i == 0) { + params.set(QUERY_NAME, uniqueQueryName); + } + + String queryId = createQuery(authUser, params); + mockServer.reset(); + + queryIds.add(queryId); + } + + UriComponentsBuilder uriBuilder = uriBuilder("/adminList"); + UriComponents uri = uriBuilder.build(); + + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(altAuthUser, null, null, HttpMethod.GET, uri); + + // make the next call asynchronously + Future> listFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + ResponseEntity listResponse = listFuture.get(); + + Assert.assertEquals(403, listResponse.getStatusCodeValue()); + } + + @Test + public void testGetQuerySuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a query + String queryId = createQuery(authUser, createParams()); + mockServer.reset(); + + // get the query + Future> listFuture = getQuery(authUser, queryId); + + // this should return immediately + ResponseEntity listResponse = listFuture.get(); + + Assert.assertEquals(200, listResponse.getStatusCodeValue()); + + QueryImplListResponse result = listResponse.getBody(); + + Assert.assertEquals(1, result.getNumResults()); + + Assert.assertEquals(queryId, result.getQuery().get(0).getId().toString()); + } + + @Test + public void testListQueryLogicSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + Future> future = listQueryLogic(authUser); + + ResponseEntity response = future.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + QueryLogicResponse qlResponse = response.getBody(); + + Assert.assertEquals(2, qlResponse.getQueryLogicList().size()); + + List qlNames = qlResponse.getQueryLogicList().stream().map(QueryLogicDescription::getName).sorted().collect(Collectors.toList()); + + Assert.assertEquals(Arrays.asList("AltEventQuery", "EventQuery"), qlNames); + } +} diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java new file mode 100644 index 00000000..c826d7a2 --- /dev/null +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java @@ -0,0 +1,753 @@ +package datawave.microservice.query; + +import com.google.common.collect.Iterables; +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.TaskStates; +import datawave.webservice.query.result.event.DefaultEvent; +import datawave.webservice.result.BaseResponse; +import datawave.webservice.result.DefaultEventQueryResponse; +import datawave.webservice.result.VoidResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Objects; +import java.util.UUID; +import java.util.concurrent.Future; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +public class QueryServiceNextTest extends AbstractQueryServiceTest { + @Before + public void setup() { + super.setup(); + } + + @After + public void teardown() throws Exception { + super.teardown(); + } + + @Test + public void testNextSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // pump enough results into the queue to trigger a complete page + int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("LOKI", "ALLIGATOR"); + fieldValues.add("LOKI", "CLASSIC"); + + // @formatter:off + publishEventsToQueue( + queryId, + (int) (1.5 * pageSize), + fieldValues, + "ALL"); + // @formatter:on + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // verify some headers + Assert.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + + DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); + + // verify the query response + // @formatter:off + assertQueryResponse( + queryId, + "EventQuery", + 1, + false, + Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-OperationTimeInMS")))), + 1, + Collections.singletonList("LOKI"), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Arrays.asList("LOKI", "LOKI"), + Arrays.asList("ALLIGATOR", "CLASSIC"), + event); + // @formatter:on + + // verify that the next event was published + Assert.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testNextSuccess_multiplePages() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // pump enough results into the queue to trigger two complete pages + int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("LOKI", "ALLIGATOR"); + fieldValues.add("LOKI", "CLASSIC"); + + // verify that the create event was published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:off + + for (int page = 1; page <= 2; page++) { + // TODO: We have to generate the results in between next calls because the test queue manager does not handle requeueing of unused messages :( + // @formatter:off + publishEventsToQueue( + queryId, + pageSize, + fieldValues, + "ALL"); + // @formatter:on + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // verify some headers + Assert.assertEquals(Integer.toString(page), Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + + DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); + + // verify the query response + // @formatter:off + assertQueryResponse( + queryId, + "EventQuery", + page, + false, + Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-OperationTimeInMS")))), + 1, + Collections.singletonList("LOKI"), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Arrays.asList("LOKI", "LOKI"), + Arrays.asList("ALLIGATOR", "CLASSIC"), + event); + // @formatter:on + + // verify that the next event was published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + } + + @Test + public void testNextSuccess_cancelPartialResults() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // pump enough results into the queue to trigger a complete page + int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("LOKI", "ALLIGATOR"); + fieldValues.add("LOKI", "CLASSIC"); + + int numEvents = (int) (0.5 * pageSize); + + // @formatter:off + publishEventsToQueue( + queryId, + numEvents, + fieldValues, + "ALL"); + // @formatter:on + + // make the next call asynchronously + Future> nextFuture = nextQuery(authUser, queryId); + + // make sure all events were consumed before canceling + while (queryQueueManager.getQueueSize(queryId) != 0) { + Thread.sleep(100); + } + + // cancel the query so that it returns partial results + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // the response should come back right away + ResponseEntity nextResponse = nextFuture.get(); + + Assert.assertEquals(200, nextResponse.getStatusCodeValue()); + + // verify some headers + Assert.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-query-page-number")))); + Assert.assertEquals("true", Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-Partial-Results")))); + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-query-last-page")))); + + DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) nextResponse.getBody(); + + // verify the query response + // @formatter:off + assertQueryResponse( + queryId, + "EventQuery", + 1, + true, + Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-OperationTimeInMS")))), + 1, + Collections.singletonList("LOKI"), + numEvents, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Arrays.asList("LOKI", "LOKI"), + Arrays.asList("ALLIGATOR", "CLASSIC"), + event); + // @formatter:on + + // verify that the next events were published + Assert.assertEquals(4, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testNextSuccess_maxResults() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // pump enough results into the queue to trigger two complete pages + int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("LOKI", "ALLIGATOR"); + fieldValues.add("LOKI", "CLASSIC"); + + // verify that the create event was published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + + for (int page = 1; page <= 4; page++) { + // TODO: We have to generate the results in between next calls because the test queue manager does not handle requeueing of unused messages :( + // @formatter:off + publishEventsToQueue( + queryId, + pageSize, + fieldValues, + "ALL"); + // @formatter:on + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + if (page != 4) { + Assert.assertEquals(200, response.getStatusCodeValue()); + } else { + Assert.assertEquals(204, response.getStatusCodeValue()); + } + + if (page != 4) { + // verify some headers + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assert.assertEquals(Integer.toString(page), Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + + DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); + + // verify the query response + // @formatter:off + assertQueryResponse( + queryId, + "EventQuery", + page, + false, + Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-OperationTimeInMS")))), + 1, + Collections.singletonList("LOKI"), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Arrays.asList("LOKI", "LOKI"), + Arrays.asList("ALLIGATOR", "CLASSIC"), + event); + // @formatter:on + + // verify that the next event was published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } else { + Assert.assertNull(response.getBody()); + + // verify that the next and close events were published + Assert.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + } + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Cannot call next on a query that is not running", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + } + + @Test + public void testNextSuccess_noResults() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // remove the task states to make it appear that the executor has finished + TaskStates taskStates = queryStorageCache.getTaskStates(queryId); + taskStates.getTaskStates().remove(TaskStates.TASK_STATE.READY); + queryStorageCache.updateTaskStates(taskStates); + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(204, response.getStatusCodeValue()); + Assert.assertNull(response.getBody()); + + // verify that the next event was published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testNextFailure_queryNotFound() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(404, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testNextFailure_queryNotRunning() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // cancel the query so that it returns partial results + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Cannot call next on a query that is not running", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that the next events were published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testNextFailure_ownershipFailure() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails altAuthUser = createAltUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // make the next call as an alternate user asynchronously + Future> future = nextQuery(altAuthUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(401, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that the next events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testNextFailure_timeout() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back after the configured timeout (5 seconds) + ResponseEntity response = future.get(); + + Assert.assertEquals(500, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Query timed out. " + queryId + " timed out.", + "Exception with no cause caught", + "500-27", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that the next events were published + Assert.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testNextFailure_nextOnDefined() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + // make the next call + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Cannot call next on a query that is not running", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testNextFailure_nextOnClosed() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Cannot call next on a query that is not running", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assert.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testNextFailure_nextOnCanceled() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // make the next call asynchronously + Future> future = nextQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Cannot call next on a query that is not running", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that the cancel event was published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } +} diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java new file mode 100644 index 00000000..fac5f558 --- /dev/null +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java @@ -0,0 +1,418 @@ +package datawave.microservice.query; + +import com.google.common.collect.Iterables; +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.result.BaseResponse; +import datawave.webservice.result.VoidResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.web.util.UriComponents; + +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +public class QueryServiceRemoveTest extends AbstractQueryServiceTest { + @Before + public void setup() { + super.setup(); + } + + @After + public void teardown() throws Exception { + super.teardown(); + } + + @Test + public void testRemoveSuccess_removeOnDefined() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + // remove the query + Future> removeFuture = removeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = removeFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // verify that original query was removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + Assert.assertNull(queryStatus); + + // verify that events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testRemoveSuccess_removeOnClosed() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + + // remove the query + Future> removeFuture = removeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = removeFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // verify that original query was removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + Assert.assertNull(queryStatus); + + // verify that events were published + Assert.assertEquals(2, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testRemoveSuccess_removeOnCanceled() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + // remove the query + Future> removeFuture = removeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = removeFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // verify that original query was removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + Assert.assertNull(queryStatus); + + // verify that events were published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testRemoveFailure_removeOnCreated() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // remove the query + Future> removeFuture = removeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = removeFuture.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + Assert.assertEquals("Cannot remove a running query.", Iterables.getOnlyElement(response.getBody().getExceptions()).getMessage()); + + // verify that original query was not removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + Assert.assertNotNull(queryStatus); + + // verify that events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testRemoveFailure_removeOnClosedActiveNext() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // call next on the query + nextQuery(authUser, queryId); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + + // remove the query + Future> removeFuture = removeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = removeFuture.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + Assert.assertEquals("Cannot remove a running query.", Iterables.getOnlyElement(response.getBody().getExceptions()).getMessage()); + + // verify that original query was not removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + Assert.assertNotNull(queryStatus); + + // verify that events were published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testRemoveFailure_queryNotFound() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + // remove the query + UriComponents uri = createUri(queryId + "/remove"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); + + // close the query + Future> resetFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, BaseResponse.class)); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assert.assertEquals(404, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testRemoveFailure_ownershipFailure() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails altAuthUser = createAltUserDetails(); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + // remove the query + UriComponents uri = createUri(queryId + "/remove"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(altAuthUser, null, null, HttpMethod.DELETE, uri); + + // close the query + Future> resetFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, BaseResponse.class)); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assert.assertEquals(401, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testAdminRemoveSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails adminUser = createAltUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + // remove the query + Future> removeFuture = adminRemoveQuery(adminUser, queryId); + + // the response should come back right away + ResponseEntity response = removeFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // verify that original query was removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + Assert.assertNull(queryStatus); + + // verify that events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testAdminRemoveFailure_notAdminUser() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + // remove the query + UriComponents uri = createUri(queryId + "/adminRemove"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); + + // remove the queries + Future> removeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + // the response should come back right away + ResponseEntity response = removeFuture.get(); + + Assert.assertEquals(403, response.getStatusCodeValue()); + + // verify that original query was not removed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + Assert.assertNotNull(queryStatus); + + // verify that events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testAdminRemoveAllSuccess() throws Exception { + ProxiedUserDetails adminUser = createUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); + + // define a bunch of queries + for (int i = 0; i < 10; i++) { + defineQuery(adminUser, createParams()); + } + + // remove all queries as the admin user + Future> removeFuture = adminRemoveAllQueries(adminUser); + + // the response should come back right away + ResponseEntity removeResponse = removeFuture.get(); + + Assert.assertEquals(200, removeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + List queryStatusList = queryStorageCache.getQueryStatus(); + + Assert.assertEquals(0, queryStatusList.size()); + + // verify that there are no events + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testAdminRemoveAllFailure_notAdminUser() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a bunch of queries + for (int i = 0; i < 10; i++) { + defineQuery(authUser, createParams()); + } + + UriComponents uri = createUri("/adminRemoveAll"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); + + // remove the queries + Future> removeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + + // the response should come back right away + ResponseEntity removeResponse = removeFuture.get(); + + Assert.assertEquals(403, removeResponse.getStatusCodeValue()); + + // verify that query status was created correctly + List queryStatusList = queryStorageCache.getQueryStatus(); + + Assert.assertEquals(10, queryStatusList.size()); + + // verify that there are no events + Assert.assertEquals(0, queryRequestEvents.size()); + } +} diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java new file mode 100644 index 00000000..30c40407 --- /dev/null +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java @@ -0,0 +1,488 @@ +package datawave.microservice.query; + +import com.google.common.collect.Iterables; +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.result.BaseResponse; +import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.VoidResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.web.util.UriComponents; + +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +public class QueryServiceResetTest extends AbstractQueryServiceTest { + @Before + public void setup() { + super.setup(); + } + + @After + public void teardown() throws Exception { + super.teardown(); + } + + @Test + public void testResetSuccess_resetOnDefined() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = defineQuery(authUser, createParams()); + + mockServer.reset(); + auditSentSetup(); + + // reset the query + Future> resetFuture = resetQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // @formatter:off + assertGenericResponse( + true, + HttpStatus.Series.SUCCESSFUL, + response); + // @formatter:on + + String resetQueryId = (String) response.getBody().getResult(); + + // verify that a new query id was created + Assert.assertNotEquals(queryId, resetQueryId); + + // verify that an audit record was sent + assertAuditSent(resetQueryId); + + // verify that original query was canceled + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.DEFINED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that new query was created + QueryStatus resetQueryStatus = queryStorageCache.getQueryStatus(resetQueryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + resetQueryStatus); + // @formatter:on + + // make sure the queries are equal (ignoring the query id) + queryStatus.getQuery().setId(resetQueryStatus.getQuery().getId()); + Assert.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); + + // verify that events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + resetQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testResetSuccess_resetOnCreated() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + mockServer.reset(); + auditSentSetup(); + + // reset the query + Future> resetFuture = resetQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // @formatter:off + assertGenericResponse( + true, + HttpStatus.Series.SUCCESSFUL, + response); + // @formatter:on + + String resetQueryId = (String) response.getBody().getResult(); + + // verify that a new query id was created + Assert.assertNotEquals(queryId, resetQueryId); + + // verify that an audit record was sent + assertAuditSent(resetQueryId); + + // verify that original query was canceled + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCELED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that new query was created + QueryStatus resetQueryStatus = queryStorageCache.getQueryStatus(resetQueryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + resetQueryStatus); + // @formatter:on + + // make sure the queries are equal (ignoring the query id) + queryStatus.getQuery().setId(resetQueryStatus.getQuery().getId()); + Assert.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); + + // verify that events were published + Assert.assertEquals(4, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + resetQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testResetSuccess_resetOnClosed() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // close the query + Future> closeFuture = closeQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity closeResponse = closeFuture.get(); + + Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + + mockServer.reset(); + auditSentSetup(); + + // reset the query + Future> resetFuture = resetQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // @formatter:off + assertGenericResponse( + true, + HttpStatus.Series.SUCCESSFUL, + response); + // @formatter:on + + String resetQueryId = (String) response.getBody().getResult(); + + // verify that a new query id was created + Assert.assertNotEquals(queryId, resetQueryId); + + // verify that an audit record was sent + assertAuditSent(resetQueryId); + + // verify that original query was closed + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CLOSED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that new query was created + QueryStatus resetQueryStatus = queryStorageCache.getQueryStatus(resetQueryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + resetQueryStatus); + // @formatter:on + + // make sure the queries are equal (ignoring the query id) + queryStatus.getQuery().setId(resetQueryStatus.getQuery().getId()); + Assert.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); + + // verify that events were published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + resetQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testResetSuccess_resetOnCanceled() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // define a valid query + long currentTimeMillis = System.currentTimeMillis(); + String queryId = createQuery(authUser, createParams()); + + // cancel the query + Future> cancelFuture = cancelQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity cancelResponse = cancelFuture.get(); + + Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + + mockServer.reset(); + auditSentSetup(); + + // reset the query + Future> resetFuture = resetQuery(authUser, queryId); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // @formatter:off + assertGenericResponse( + true, + HttpStatus.Series.SUCCESSFUL, + response); + // @formatter:on + + String resetQueryId = (String) response.getBody().getResult(); + + // verify that a new query id was created + Assert.assertNotEquals(queryId, resetQueryId); + + // verify that an audit record was sent + assertAuditSent(resetQueryId); + + // verify that original query was canceled + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CANCELED, + 0, + 0, + 0, + 0, + currentTimeMillis, + queryStatus); + // @formatter:on + + // verify that new query was created + QueryStatus resetQueryStatus = queryStorageCache.getQueryStatus(resetQueryId); + + // @formatter:off + assertQueryStatus( + QueryStatus.QUERY_STATE.CREATED, + 0, + 0, + 0, + 0, + currentTimeMillis, + resetQueryStatus); + // @formatter:on + + // make sure the queries are equal (ignoring the query id) + queryStatus.getQuery().setId(resetQueryStatus.getQuery().getId()); + Assert.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); + + // verify that events were published + Assert.assertEquals(4, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "/query:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CANCEL, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + resetQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testResetFailure_queryNotFound() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + auditNotSentSetup(); + + // reset the query + UriComponents uri = createUri(queryId + "/reset"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); + + // close the query + Future> resetFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, BaseResponse.class)); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assert.assertEquals(404, response.getStatusCodeValue()); + + // make sure no audits were sent + assertAuditNotSent(); + + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testResetFailure_ownershipFailure() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails altAuthUser = createAltUserDetails(); + + // define a valid query + String queryId = createQuery(authUser, createParams()); + + mockServer.reset(); + auditNotSentSetup(); + + // reset the query + UriComponents uri = createUri(queryId + "/reset"); + RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(altAuthUser, null, null, HttpMethod.PUT, uri); + + // close the query + Future> resetFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, BaseResponse.class)); + + // the response should come back right away + ResponseEntity response = resetFuture.get(); + + Assert.assertEquals(401, response.getStatusCodeValue()); + + // make sure no audits were sent + assertAuditNotSent(); + + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } +} diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java deleted file mode 100644 index 74138e9c..00000000 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceTest.java +++ /dev/null @@ -1,5179 +0,0 @@ -package datawave.microservice.query; - -import com.google.common.collect.Iterables; -import datawave.marking.ColumnVisibilitySecurityMarking; -import datawave.microservice.audit.AuditClient; -import datawave.microservice.authorization.jwt.JWTRestTemplate; -import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; -import datawave.microservice.authorization.user.ProxiedUserDetails; -import datawave.microservice.query.config.QueryProperties; -import datawave.microservice.query.remote.QueryRequest; -import datawave.microservice.query.storage.QueryStatus; -import datawave.microservice.query.storage.QueryStorageCache; -import datawave.microservice.query.storage.Result; -import datawave.microservice.query.storage.TaskKey; -import datawave.microservice.query.storage.TaskStates; -import datawave.microservice.query.storage.queue.TestQueryQueueManager; -import datawave.security.authorization.DatawaveUser; -import datawave.security.authorization.SubjectIssuerDNPair; -import datawave.security.util.ProxiedEntityUtils; -import datawave.webservice.query.Query; -import datawave.webservice.query.exception.QueryExceptionType; -import datawave.webservice.query.result.event.DefaultEvent; -import datawave.webservice.query.result.event.DefaultField; -import datawave.webservice.query.result.logic.QueryLogicDescription; -import datawave.webservice.result.BaseResponse; -import datawave.webservice.result.DefaultEventQueryResponse; -import datawave.webservice.result.GenericResponse; -import datawave.webservice.result.QueryImplListResponse; -import datawave.webservice.result.QueryLogicResponse; -import datawave.webservice.result.VoidResponse; -import org.apache.http.NameValuePair; -import org.apache.http.client.utils.URLEncodedUtils; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.DirectFieldAccessor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.web.client.RestTemplateBuilder; -import org.springframework.boot.web.server.LocalServerPort; -import org.springframework.cloud.bus.event.RemoteQueryRequestEvent; -import org.springframework.context.ApplicationEvent; -import org.springframework.context.ApplicationEventPublisher; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.context.annotation.Profile; -import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; -import org.springframework.http.RequestEntity; -import org.springframework.http.ResponseEntity; -import org.springframework.http.client.ClientHttpResponse; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; -import org.springframework.test.web.client.MockRestServiceServer; -import org.springframework.test.web.client.RequestMatcher; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; -import org.springframework.web.client.DefaultResponseErrorHandler; -import org.springframework.web.client.RestTemplate; -import org.springframework.web.util.UriComponents; -import org.springframework.web.util.UriComponentsBuilder; - -import java.io.IOException; -import java.nio.charset.Charset; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.UUID; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; - -import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; -import static datawave.microservice.query.QueryParameters.QUERY_MAX_CONCURRENT_TASKS; -import static datawave.microservice.query.QueryParameters.QUERY_MAX_RESULTS_OVERRIDE; -import static datawave.microservice.query.QueryParameters.QUERY_NAME; -import static datawave.microservice.query.QueryParameters.QUERY_PAGESIZE; -import static datawave.security.authorization.DatawaveUser.UserType.USER; -import static datawave.webservice.common.audit.AuditParameters.AUDIT_ID; -import static datawave.webservice.common.audit.AuditParameters.QUERY_AUTHORIZATIONS; -import static datawave.webservice.common.audit.AuditParameters.QUERY_STRING; -import static datawave.webservice.query.QueryImpl.BEGIN_DATE; -import static datawave.webservice.query.QueryImpl.END_DATE; -import static datawave.webservice.query.QueryImpl.PAGESIZE; -import static datawave.webservice.query.QueryImpl.QUERY; -import static org.springframework.test.web.client.ExpectedCount.never; -import static org.springframework.test.web.client.match.MockRestRequestMatchers.anything; -import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; -import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; - -@RunWith(SpringRunner.class) -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) -public class QueryServiceTest { - private static final String EXPECTED_AUDIT_URI = "http://localhost:11111/audit/v1/audit"; - private static final String TEST_QUERY_STRING = "FIELD:SOME_VALUE"; - private static final String TEST_QUERY_NAME = "The Greatest Query in the World - Tribute"; - private static final String TEST_QUERY_AUTHORIZATIONS = "ALL"; - private static final String TEST_QUERY_BEGIN = "20000101 000000.000"; - private static final String TEST_QUERY_END = "20500101 000000.000"; - private static final String TEST_VISIBILITY_MARKING = "ALL"; - - @LocalServerPort - private int webServicePort; - - @Autowired - private RestTemplateBuilder restTemplateBuilder; - - private JWTRestTemplate jwtRestTemplate; - - private SubjectIssuerDNPair DN; - private String userDN = "userDn"; - - private SubjectIssuerDNPair altDN; - private String altUserDN = "altUserDN"; - - @Autowired - private QueryStorageCache queryStorageCache; - - @Autowired - private TestQueryQueueManager queryQueueManager; - - @Autowired - private AuditClient auditClient; - - @Autowired - private QueryProperties queryProperties; - - @Autowired - private LinkedList queryRequestEvents; - - private List auditIds; - private MockRestServiceServer mockServer; - - @Before - public void setup() { - auditIds = new ArrayList<>(); - - jwtRestTemplate = restTemplateBuilder.build(JWTRestTemplate.class); - jwtRestTemplate.setErrorHandler(new NoOpResponseErrorHandler()); - DN = SubjectIssuerDNPair.of(userDN, "issuerDn"); - altDN = SubjectIssuerDNPair.of(altUserDN, "issuerDN"); - - RestTemplate auditorRestTemplate = (RestTemplate) new DirectFieldAccessor(auditClient).getPropertyValue("jwtRestTemplate"); - mockServer = MockRestServiceServer.createServer(auditorRestTemplate); - - queryRequestEvents.clear(); - } - - @DirtiesContext - @Test - public void testDefineSuccess() throws ParseException, IOException { - ProxiedUserDetails authUser = createUserDetails(); - UriComponents uri = createUri("EventQuery/define"); - MultiValueMap map = createParams(); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - // setup a mock audit service - auditNotSentSetup(); - - long currentTimeMillis = System.currentTimeMillis(); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); - // @formatter:off - GenericResponse genericResponse = assertGenericResponse( - false, - HttpStatus.Series.SUCCESSFUL, - resp); - // @formatter:on - - // verify that a query id was returned - String queryId = genericResponse.getResult(); - Assert.assertNotNull(queryId); - - // verify that query status was created correctly - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.DEFINED, - 0, - 0, - 0, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - // verify that they query was created correctly - Query query = queryStatus.getQuery(); - // @formatter:off - assertQuery( - TEST_QUERY_STRING, - TEST_QUERY_NAME, - TEST_QUERY_AUTHORIZATIONS, - TEST_QUERY_BEGIN, - TEST_QUERY_END, - TEST_VISIBILITY_MARKING, - query); - // @formatter:on - - // verify that no audit message was sent - assertAuditNotSent(); - - // verify that the results queue wasn't created - Assert.assertFalse(queryQueueManager.queueExists(queryId)); - - // verify that query tasks weren't created - assertTasksNotCreated(queryId); - } - - @Test - public void testDefineFailure_paramValidation() { - ProxiedUserDetails authUser = createUserDetails(); - UriComponents uri = createUri("EventQuery/define"); - MultiValueMap map = createParams(); - - // remove the query param to induce a parameter validation failure - map.remove(QUERY_STRING); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); - - // @formatter:off - BaseResponse baseResponse = assertBaseResponse( - false, - HttpStatus.Series.CLIENT_ERROR, - resp); - // @formatter:on - - // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); - - // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); - - QueryExceptionType queryException = baseResponse.getExceptions().get(0); - // @formatter:off - assertQueryException( - "Missing one or more required QueryParameters", - "java.lang.IllegalArgumentException: Missing one or more required QueryParameters", - "400-1", - queryException); - // @formatter:on - - // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); - } - - @Test - public void testDefineFailure_authValidation() { - ProxiedUserDetails authUser = createUserDetails(Collections.singleton("AuthorizedUser"), Collections.emptyList()); - UriComponents uri = createUri("EventQuery/define"); - MultiValueMap map = createParams(); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); - - // @formatter:off - BaseResponse baseResponse = assertBaseResponse( - false, - HttpStatus.Series.CLIENT_ERROR, - resp); - // @formatter:on - - // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); - - // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); - - QueryExceptionType queryException = baseResponse.getExceptions().get(0); - // @formatter:off - assertQueryException( - "User requested authorizations that they don't have. Missing: [ALL], Requested: [ALL], User: []", - "java.lang.IllegalArgumentException: User requested authorizations that they don't have. Missing: [ALL], Requested: [ALL], User: []", - "400-1", - queryException); - // @formatter:on - - // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); - } - - @Test - public void testDefineFailure_queryLogicValidation() { - ProxiedUserDetails authUser = createUserDetails(); - UriComponents uri = createUri("EventQuery/define"); - MultiValueMap map = createParams(); - - // remove the beginDate param to induce a query logic validation failure - map.remove(BEGIN_DATE); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); - - // @formatter:off - BaseResponse baseResponse = assertBaseResponse( - false, - HttpStatus.Series.CLIENT_ERROR, - resp); - // @formatter:on - - // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); - - // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); - - QueryExceptionType queryException = baseResponse.getExceptions().get(0); - // @formatter:off - assertQueryException( - "Required parameter begin not found", - "java.lang.IllegalArgumentException: Required parameter begin not found", - "400-1", - queryException); - // @formatter:on - - // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); - } - - @Test - public void testDefineFailure_maxPageSize() { - ProxiedUserDetails authUser = createUserDetails(Arrays.asList("AuthorizedUser", queryProperties.getPrivilegedRole()), null); - UriComponents uri = createUri("EventQuery/define"); - MultiValueMap map = createParams(); - - // set an invalid page size override - map.set(QUERY_PAGESIZE, Integer.toString(Integer.MAX_VALUE)); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); - - // @formatter:off - BaseResponse baseResponse = assertBaseResponse( - false, - HttpStatus.Series.CLIENT_ERROR, - resp); - // @formatter:on - - // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); - - // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); - - QueryExceptionType queryException = baseResponse.getExceptions().get(0); - // @formatter:off - assertQueryException( - "Page size is larger than configured max. Max = 10,000.", - "Exception with no cause caught", - "400-6", - queryException); - // @formatter:on - - // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); - } - - @Test - public void testDefineFailure_maxResultsOverride() { - ProxiedUserDetails authUser = createUserDetails(); - UriComponents uri = createUri("EventQuery/define"); - MultiValueMap map = createParams(); - - // set an invalid max results override - map.set(QUERY_MAX_RESULTS_OVERRIDE, Long.toString(Long.MAX_VALUE)); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); - - // @formatter:off - BaseResponse baseResponse = assertBaseResponse( - false, - HttpStatus.Series.CLIENT_ERROR, - resp); - // @formatter:on - - // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); - - // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); - - QueryExceptionType queryException = baseResponse.getExceptions().get(0); - // @formatter:off - assertQueryException( - "Invalid max results override value. Max = 369.", - "Exception with no cause caught", - "400-43", - queryException); - // @formatter:on - - // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); - } - - @Test - public void testDefineFailure_maxConcurrentTasksOverride() { - ProxiedUserDetails authUser = createUserDetails(); - UriComponents uri = createUri("EventQuery/define"); - MultiValueMap map = createParams(); - - // add an invalid max results override - map.set(QUERY_MAX_CONCURRENT_TASKS, Integer.toString(Integer.MAX_VALUE)); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); - - // @formatter:off - BaseResponse baseResponse = assertBaseResponse( - false, - HttpStatus.Series.CLIENT_ERROR, - resp); - // @formatter:on - - // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); - - // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); - - QueryExceptionType queryException = baseResponse.getExceptions().get(0); - // @formatter:off - assertQueryException( - "Invalid max concurrent tasks override value. Max = 10.", - "Exception with no cause caught", - "400-44", - queryException); - // @formatter:on - - // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); - } - - @Test - public void testDefineFailure_roleValidation() { - // create a user without the required role - ProxiedUserDetails authUser = createUserDetails(Collections.emptyList(), Collections.singletonList("ALL")); - UriComponents uri = createUri("EventQuery/define"); - MultiValueMap map = createParams(); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); - - // @formatter:off - BaseResponse baseResponse = assertBaseResponse( - false, - HttpStatus.Series.CLIENT_ERROR, - resp); - // @formatter:on - - // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); - - // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); - - QueryExceptionType queryException = baseResponse.getExceptions().get(0); - // @formatter:off - assertQueryException( - "User does not have the required roles.", - "datawave.webservice.query.exception.UnauthorizedQueryException: User does not have the required roles.", - "400-5", - queryException); - // @formatter:on - - // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); - } - - @Test - public void testDefineFailure_markingValidation() { - ProxiedUserDetails authUser = createUserDetails(); - UriComponents uri = createUri("EventQuery/define"); - MultiValueMap map = createParams(); - - // remove the column visibility param to induce a security marking validation failure - map.remove(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); - - // @formatter:off - BaseResponse baseResponse = assertBaseResponse( - false, - HttpStatus.Series.CLIENT_ERROR, - resp); - // @formatter:on - - // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); - - // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); - - QueryExceptionType queryException = baseResponse.getExceptions().get(0); - // @formatter:off - assertQueryException( - "Required parameter columnVisibility not found", - "java.lang.IllegalArgumentException: Required parameter columnVisibility not found", - "400-4", - queryException); - // @formatter:on - - // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); - } - - @DirtiesContext - @Test - public void testCreateSuccess() throws ParseException, IOException { - ProxiedUserDetails authUser = createUserDetails(); - UriComponents uri = createUri("EventQuery/create"); - MultiValueMap map = createParams(); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - // setup a mock audit service - auditSentSetup(); - - long currentTimeMillis = System.currentTimeMillis(); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); - // @formatter:off - GenericResponse genericResponse = assertGenericResponse( - true, - HttpStatus.Series.SUCCESSFUL, - resp); - // @formatter:on - - // verify that a query id was returned - String queryId = genericResponse.getResult(); - Assert.assertNotNull(queryId); - - // verify that the create event was published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - - // verify that query status was created correctly - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, - 0, - 0, - 0, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - // verify that they query was created correctly - Query query = queryStatus.getQuery(); - - // @formatter:off - assertQuery( - TEST_QUERY_STRING, - TEST_QUERY_NAME, - TEST_QUERY_AUTHORIZATIONS, - TEST_QUERY_BEGIN, - TEST_QUERY_END, - TEST_VISIBILITY_MARKING, - query); - // @formatter:on - - // verify that an audit message was sent and the the audit id matches the query id - assertAuditSent(queryId); - - // verify that the results queue was created - Assert.assertTrue(queryQueueManager.queueExists(queryId)); - - // verify that query tasks were created - assertTasksCreated(queryId); - } - - @Test - public void testCreateFailure_paramValidation() { - ProxiedUserDetails authUser = createUserDetails(); - UriComponents uri = createUri("EventQuery/create"); - MultiValueMap map = createParams(); - - // remove the query param to induce a parameter validation failure - map.remove(QUERY_STRING); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - // setup a mock audit service - auditNotSentSetup(); - - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); - - // @formatter:off - BaseResponse baseResponse = assertBaseResponse( - false, - HttpStatus.Series.CLIENT_ERROR, - resp); - // @formatter:on - - // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); - - // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); - - QueryExceptionType queryException = baseResponse.getExceptions().get(0); - // @formatter:off - assertQueryException( - "Missing one or more required QueryParameters", - "java.lang.IllegalArgumentException: Missing one or more required QueryParameters", - "400-1", - queryException); - // @formatter:on - - // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); - - // verify that no audit message was sent - assertAuditNotSent(); - } - - @Test - public void testCreateFailure_authValidation() { - ProxiedUserDetails authUser = createUserDetails(Collections.singleton("AuthorizedUser"), Collections.emptyList()); - UriComponents uri = createUri("EventQuery/create"); - MultiValueMap map = createParams(); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - // setup a mock audit service - auditNotSentSetup(); - - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); - - // @formatter:off - BaseResponse baseResponse = assertBaseResponse( - false, - HttpStatus.Series.CLIENT_ERROR, - resp); - // @formatter:on - - // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); - - // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); - - QueryExceptionType queryException = baseResponse.getExceptions().get(0); - // @formatter:off - assertQueryException( - "User requested authorizations that they don't have. Missing: [ALL], Requested: [ALL], User: []", - "java.lang.IllegalArgumentException: User requested authorizations that they don't have. Missing: [ALL], Requested: [ALL], User: []", - "400-1", - queryException); - // @formatter:on - - // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); - - // verify that no audit message was sent - assertAuditNotSent(); - } - - @Test - public void testCreateFailure_queryLogicValidation() { - ProxiedUserDetails authUser = createUserDetails(); - UriComponents uri = createUri("EventQuery/create"); - MultiValueMap map = createParams(); - - // remove the beginDate param to induce a query logic validation failure - map.remove(BEGIN_DATE); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - // setup a mock audit service - auditNotSentSetup(); - - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); - - // @formatter:off - BaseResponse baseResponse = assertBaseResponse( - false, - HttpStatus.Series.CLIENT_ERROR, - resp); - // @formatter:on - - // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); - - // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); - - QueryExceptionType queryException = baseResponse.getExceptions().get(0); - // @formatter:off - assertQueryException( - "Required parameter begin not found", - "java.lang.IllegalArgumentException: Required parameter begin not found", - "400-1", - queryException); - // @formatter:on - - // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); - - // verify that no audit message was sent - assertAuditNotSent(); - } - - @Test - public void testCreateFailure_maxPageSize() { - ProxiedUserDetails authUser = createUserDetails(Arrays.asList("AuthorizedUser", queryProperties.getPrivilegedRole()), null); - UriComponents uri = createUri("EventQuery/create"); - MultiValueMap map = createParams(); - - // set an invalid page size override - map.set(QUERY_PAGESIZE, Integer.toString(Integer.MAX_VALUE)); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - // setup a mock audit service - auditNotSentSetup(); - - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); - - // @formatter:off - BaseResponse baseResponse = assertBaseResponse( - false, - HttpStatus.Series.CLIENT_ERROR, - resp); - // @formatter:on - - // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); - - // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); - - QueryExceptionType queryException = baseResponse.getExceptions().get(0); - // @formatter:off - assertQueryException( - "Page size is larger than configured max. Max = 10,000.", - "Exception with no cause caught", - "400-6", - queryException); - // @formatter:on - - // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); - - // verify that no audit message was sent - assertAuditNotSent(); - } - - @Test - public void testCreateFailure_maxResultsOverride() { - ProxiedUserDetails authUser = createUserDetails(); - UriComponents uri = createUri("EventQuery/create"); - MultiValueMap map = createParams(); - - // set an invalid max results override - map.set(QUERY_MAX_RESULTS_OVERRIDE, Long.toString(Long.MAX_VALUE)); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - // setup a mock audit service - auditNotSentSetup(); - - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); - - // @formatter:off - BaseResponse baseResponse = assertBaseResponse( - false, - HttpStatus.Series.CLIENT_ERROR, - resp); - // @formatter:on - - // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); - - // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); - - QueryExceptionType queryException = baseResponse.getExceptions().get(0); - // @formatter:off - assertQueryException( - "Invalid max results override value. Max = 369.", - "Exception with no cause caught", - "400-43", - queryException); - // @formatter:on - - // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); - - // verify that no audit message was sent - assertAuditNotSent(); - } - - @Test - public void testCreateFailure_maxConcurrentTasksOverride() { - ProxiedUserDetails authUser = createUserDetails(); - UriComponents uri = createUri("EventQuery/create"); - MultiValueMap map = createParams(); - - // add an invalid max results override - map.set(QUERY_MAX_CONCURRENT_TASKS, Integer.toString(Integer.MAX_VALUE)); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - // setup a mock audit service - auditNotSentSetup(); - - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); - - // @formatter:off - BaseResponse baseResponse = assertBaseResponse( - false, - HttpStatus.Series.CLIENT_ERROR, - resp); - // @formatter:on - - // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); - - // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); - - QueryExceptionType queryException = baseResponse.getExceptions().get(0); - // @formatter:off - assertQueryException( - "Invalid max concurrent tasks override value. Max = 10.", - "Exception with no cause caught", - "400-44", - queryException); - // @formatter:on - - // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); - - // verify that no audit message was sent - assertAuditNotSent(); - } - - @Test - public void testCreateFailure_roleValidation() { - // create a user without the required role - ProxiedUserDetails authUser = createUserDetails(Collections.emptyList(), Collections.singletonList("ALL")); - UriComponents uri = createUri("EventQuery/create"); - MultiValueMap map = createParams(); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - // setup a mock audit service - auditNotSentSetup(); - - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); - - // @formatter:off - BaseResponse baseResponse = assertBaseResponse( - false, - HttpStatus.Series.CLIENT_ERROR, - resp); - // @formatter:on - - // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); - - // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); - - QueryExceptionType queryException = baseResponse.getExceptions().get(0); - // @formatter:off - assertQueryException( - "User does not have the required roles.", - "datawave.webservice.query.exception.UnauthorizedQueryException: User does not have the required roles.", - "400-5", - queryException); - // @formatter:on - - // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); - - // verify that no audit message was sent - assertAuditNotSent(); - } - - @Test - public void testCreateFailure_markingValidation() { - ProxiedUserDetails authUser = createUserDetails(); - UriComponents uri = createUri("EventQuery/create"); - MultiValueMap map = createParams(); - - // remove the column visibility param to induce a security marking validation failure - map.remove(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - // setup a mock audit service - auditNotSentSetup(); - - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseResponse.class); - - // @formatter:off - BaseResponse baseResponse = assertBaseResponse( - false, - HttpStatus.Series.CLIENT_ERROR, - resp); - // @formatter:on - - // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); - - // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); - - QueryExceptionType queryException = baseResponse.getExceptions().get(0); - // @formatter:off - assertQueryException( - "Required parameter columnVisibility not found", - "java.lang.IllegalArgumentException: Required parameter columnVisibility not found", - "400-4", - queryException); - // @formatter:on - - // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); - - // verify that no audit message was sent - assertAuditNotSent(); - } - - @DirtiesContext - @Test - public void testNextSuccess() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - // pump enough results into the queue to trigger a complete page - int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); - - // test field value pairings - MultiValueMap fieldValues = new LinkedMultiValueMap<>(); - fieldValues.add("LOKI", "ALLIGATOR"); - fieldValues.add("LOKI", "CLASSIC"); - - // @formatter:off - publishEventsToQueue( - queryId, - (int)(1.5*pageSize), - fieldValues, - "ALL"); - // @formatter:on - - // make the next call asynchronously - Future> future = nextQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity response = future.get(); - - Assert.assertEquals(200, response.getStatusCodeValue()); - - // verify some headers - Assert.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); - - DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); - - // verify the query response - // @formatter:off - assertQueryResponse( - queryId, - "EventQuery", - 1, - false, - Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-OperationTimeInMS")))), - 1, - Collections.singletonList("LOKI"), - pageSize, - Objects.requireNonNull(queryResponse)); - // @formatter:on - - // validate one of the events - DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); - // @formatter:off - assertDefaultEvent( - Arrays.asList("LOKI", "LOKI"), - Arrays.asList("ALLIGATOR", "CLASSIC"), - event); - // @formatter:on - - // verify that the next event was published - Assert.assertEquals(2, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.NEXT, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testNextSuccess_multiplePages() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - // pump enough results into the queue to trigger two complete pages - int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); - - // test field value pairings - MultiValueMap fieldValues = new LinkedMultiValueMap<>(); - fieldValues.add("LOKI", "ALLIGATOR"); - fieldValues.add("LOKI", "CLASSIC"); - - // verify that the create event was published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:off - - for (int page = 1; page <= 2; page++) { - // TODO: We have to generate the results in between next calls because the test queue manager does not handle requeueing of unused messages :( - // @formatter:off - publishEventsToQueue( - queryId, - pageSize, - fieldValues, - "ALL"); - // @formatter:on - - // make the next call asynchronously - Future> future = nextQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity response = future.get(); - - Assert.assertEquals(200, response.getStatusCodeValue()); - - // verify some headers - Assert.assertEquals(Integer.toString(page), Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); - - DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); - - // verify the query response - // @formatter:off - assertQueryResponse( - queryId, - "EventQuery", - page, - false, - Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-OperationTimeInMS")))), - 1, - Collections.singletonList("LOKI"), - pageSize, - Objects.requireNonNull(queryResponse)); - // @formatter:on - - // validate one of the events - DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); - // @formatter:off - assertDefaultEvent( - Arrays.asList("LOKI", "LOKI"), - Arrays.asList("ALLIGATOR", "CLASSIC"), - event); - // @formatter:on - - // verify that the next event was published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.NEXT, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - } - - @DirtiesContext - @Test - public void testNextSuccess_cancelPartialResults() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - // pump enough results into the queue to trigger a complete page - int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); - - // test field value pairings - MultiValueMap fieldValues = new LinkedMultiValueMap<>(); - fieldValues.add("LOKI", "ALLIGATOR"); - fieldValues.add("LOKI", "CLASSIC"); - - int numEvents = (int) (0.5 * pageSize); - - // @formatter:off - publishEventsToQueue( - queryId, - numEvents, - fieldValues, - "ALL"); - // @formatter:on - - // make the next call asynchronously - Future> nextFuture = nextQuery(authUser, queryId); - - // make sure all events were consumed before canceling - while (queryQueueManager.getQueueSize(queryId) != 0) { - Thread.sleep(100); - } - - // cancel the query so that it returns partial results - Future> cancelFuture = cancelQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity cancelResponse = cancelFuture.get(); - - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); - - // the response should come back right away - ResponseEntity nextResponse = nextFuture.get(); - - Assert.assertEquals(200, nextResponse.getStatusCodeValue()); - - // verify some headers - Assert.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-query-page-number")))); - Assert.assertEquals("true", Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-Partial-Results")))); - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-query-last-page")))); - - DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) nextResponse.getBody(); - - // verify the query response - // @formatter:off - assertQueryResponse( - queryId, - "EventQuery", - 1, - true, - Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-OperationTimeInMS")))), - 1, - Collections.singletonList("LOKI"), - numEvents, - Objects.requireNonNull(queryResponse)); - // @formatter:on - - // validate one of the events - DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); - // @formatter:off - assertDefaultEvent( - Arrays.asList("LOKI", "LOKI"), - Arrays.asList("ALLIGATOR", "CLASSIC"), - event); - // @formatter:on - - // verify that the next events were published - Assert.assertEquals(4, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.NEXT, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "/query:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testNextSuccess_maxResults() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - // pump enough results into the queue to trigger two complete pages - int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); - - // test field value pairings - MultiValueMap fieldValues = new LinkedMultiValueMap<>(); - fieldValues.add("LOKI", "ALLIGATOR"); - fieldValues.add("LOKI", "CLASSIC"); - - // verify that the create event was published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - - for (int page = 1; page <= 3; page++) { - // TODO: We have to generate the results in between next calls because the test queue manager does not handle requeueing of unused messages :( - // @formatter:off - publishEventsToQueue( - queryId, - pageSize, - fieldValues, - "ALL"); - // @formatter:on - - // make the next call asynchronously - Future> future = nextQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity response = future.get(); - - Assert.assertEquals(200, response.getStatusCodeValue()); - - // verify some headers - Assert.assertEquals(Integer.toString(page), Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); - - if (page != 4) { - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); - } else { - Assert.assertEquals("true", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); - } - - DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); - - // verify the query response - // @formatter:off - assertQueryResponse( - queryId, - "EventQuery", - page, - false, - Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-OperationTimeInMS")))), - 1, - Collections.singletonList("LOKI"), - pageSize, - Objects.requireNonNull(queryResponse)); - // @formatter:on - - // validate one of the events - DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); - // @formatter:off - assertDefaultEvent( - Arrays.asList("LOKI", "LOKI"), - Arrays.asList("ALLIGATOR", "CLASSIC"), - event); - // @formatter:on - - // verify that the next event was published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.NEXT, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - // make the next call asynchronously - Future> future = nextQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity response = future.get(); - - Assert.assertEquals(404, response.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "No results found for query. " + queryId, - "Exception with no cause caught", - "404-4", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - } - - @DirtiesContext - @Test - public void testNextSuccess_noResults() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - // remove the task states to make it appear that the executor has finished - TaskStates taskStates = queryStorageCache.getTaskStates(queryId); - taskStates.getTaskStates().remove(TaskStates.TASK_STATE.READY); - queryStorageCache.updateTaskStates(taskStates); - - // make the next call asynchronously - Future> future = nextQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity response = future.get(); - - Assert.assertEquals(404, response.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "No results found for query. " + queryId, - "Exception with no cause caught", - "404-4", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // verify that the next event was published - Assert.assertEquals(2, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.NEXT, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @Test - public void testNextFailure_queryNotFound() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - String queryId = UUID.randomUUID().toString(); - - // make the next call asynchronously - Future> future = nextQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity response = future.get(); - - Assert.assertEquals(404, response.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "No query object matches this id. " + queryId, - "Exception with no cause caught", - "404-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); - } - - @DirtiesContext - @Test - public void testNextFailure_queryNotRunning() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - // cancel the query so that it returns partial results - Future> cancelFuture = cancelQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity cancelResponse = cancelFuture.get(); - - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); - - // make the next call asynchronously - Future> future = nextQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity response = future.get(); - - Assert.assertEquals(400, response.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "Cannot call next on a query that is not running", - "Exception with no cause caught", - "400-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // verify that the next events were published - Assert.assertEquals(3, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "/query:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testNextFailure_ownershipFailure() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - ProxiedUserDetails altAuthUser = createAltUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - // make the next call as an alternate user asynchronously - Future> future = nextQuery(altAuthUser, queryId); - - // the response should come back right away - ResponseEntity response = future.get(); - - Assert.assertEquals(401, response.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "Current user does not match user that defined query. altuserdn != userdn", - "Exception with no cause caught", - "401-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // verify that the next events were published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testNextFailure_timeout() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - // make the next call asynchronously - Future> future = nextQuery(authUser, queryId); - - // the response should come back after the configured timeout (5 seconds) - ResponseEntity response = future.get(); - - Assert.assertEquals(500, response.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "Query timed out. " + queryId + " timed out.", - "Exception with no cause caught", - "500-27", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // verify that the next events were published - Assert.assertEquals(2, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.NEXT, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testNextFailure_nextOnDefined() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // define a valid query - String queryId = defineQuery(authUser, createParams()); - - // make the next call - Future> future = nextQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity response = future.get(); - - Assert.assertEquals(400, response.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "Cannot call next on a query that is not running", - "Exception with no cause caught", - "400-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); - } - - @DirtiesContext - @Test - public void testNextFailure_nextOnClosed() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - // close the query - Future> closeFuture = closeQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity closeResponse = closeFuture.get(); - - Assert.assertEquals(200, closeResponse.getStatusCodeValue()); - - // make the next call asynchronously - Future> future = nextQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity response = future.get(); - - Assert.assertEquals(400, response.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "Cannot call next on a query that is not running", - "Exception with no cause caught", - "400-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // verify that no events were published - Assert.assertEquals(2, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CLOSE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testNextFailure_nextOnCanceled() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - // cancel the query - Future> cancelFuture = cancelQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity cancelResponse = cancelFuture.get(); - - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); - - // make the next call asynchronously - Future> future = nextQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity response = future.get(); - - Assert.assertEquals(400, response.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "Cannot call next on a query that is not running", - "Exception with no cause caught", - "400-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // verify that the cancel event was published - Assert.assertEquals(3, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "/query:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testCancelSuccess() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - long currentTimeMillis = System.currentTimeMillis(); - String queryId = createQuery(authUser, createParams()); - - // cancel the query - Future> cancelFuture = cancelQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity cancelResponse = cancelFuture.get(); - - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); - - // verify that query status was created correctly - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CANCELED, - 0, - 0, - 0, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - // verify that the result queue is gone - Assert.assertFalse(queryQueueManager.queueExists(queryId)); - - // verify that the query tasks are still present - assertTasksCreated(queryId); - - // verify that the cancel event was published - Assert.assertEquals(3, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "/query:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - // TODO: This test has periodic failures when running all tests - @DirtiesContext - @Test - public void testCancelSuccess_activeNextCall() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - long currentTimeMillis = System.currentTimeMillis(); - String queryId = createQuery(authUser, createParams()); - - // call next on the query - Future> nextFuture = nextQuery(authUser, queryId); - - boolean nextCallActive = queryStorageCache.getQueryStatus(queryId).getActiveNextCalls() > 0; - while (!nextCallActive) { - try { - nextFuture.get(500, TimeUnit.MILLISECONDS); - } catch (TimeoutException e) { - nextCallActive = queryStorageCache.getQueryStatus(queryId).getActiveNextCalls() > 0; - if (TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - currentTimeMillis) > 5) { - throw e; - } - } - } - - // cancel the query - Future> cancelFuture = cancelQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity cancelResponse = cancelFuture.get(); - - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); - - // verify that query status was created correctly - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CANCELED, - 0, - 0, - 1, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - // verify that the result queue is gone - Assert.assertFalse(queryQueueManager.queueExists(queryId)); - - // wait for the next call to return - nextFuture.get(); - - // verify that the query tasks are still present - assertTasksCreated(queryId); - - // verify that the close event was published - Assert.assertEquals(4, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.NEXT, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "/query:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @Test - public void testCancelFailure_queryNotFound() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - String queryId = UUID.randomUUID().toString(); - - // cancel the query - Future> cancelFuture = cancelQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity cancelResponse = cancelFuture.get(); - - Assert.assertEquals(404, cancelResponse.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "No query object matches this id. " + queryId, - "Exception with no cause caught", - "404-1", - Iterables.getOnlyElement(cancelResponse.getBody().getExceptions())); - // @formatter:on - - } - - @DirtiesContext - @Test - public void testCancelFailure_ownershipFailure() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - ProxiedUserDetails altAuthUser = createAltUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - // make the cancel call as an alternate user asynchronously - Future> future = cancelQuery(altAuthUser, queryId); - - // the response should come back right away - ResponseEntity response = future.get(); - - Assert.assertEquals(401, response.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "Current user does not match user that defined query. altuserdn != userdn", - "Exception with no cause caught", - "401-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // verify that the next events were published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testCancelFailure_queryNotRunning() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - // cancel the query - Future> cancelFuture = cancelQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity cancelResponse = cancelFuture.get(); - - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); - - // try to cancel the query again - cancelFuture = cancelQuery(authUser, queryId); - - // the response should come back right away - cancelResponse = cancelFuture.get(); - - Assert.assertEquals(400, cancelResponse.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "Cannot call cancel on a query that is not running", - "Exception with no cause caught", - "400-1", - Iterables.getOnlyElement(cancelResponse.getBody().getExceptions())); - // @formatter:on - - // verify that the next events were published - Assert.assertEquals(3, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "/query:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testAdminCancelSuccess() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - ProxiedUserDetails adminUser = createAltUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); - - // create a valid query - long currentTimeMillis = System.currentTimeMillis(); - String queryId = createQuery(authUser, createParams()); - - // cancel the query as the admin user - Future> cancelFuture = adminCancelQuery(adminUser, queryId); - - // the response should come back right away - ResponseEntity cancelResponse = cancelFuture.get(); - - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); - - // verify that query status was created correctly - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CANCELED, - 0, - 0, - 0, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - // verify that the result queue is gone - Assert.assertFalse(queryQueueManager.queueExists(queryId)); - - // verify that the query tasks are still present - assertTasksCreated(queryId); - - // verify that the cancel event was published - Assert.assertEquals(3, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "/query:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testAdminCancelFailure_notAdminUser() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - UriComponents uri = createUri(queryId + "/adminCancel"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); - - // cancel the query - Future> closeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); - - // the response should come back right away - ResponseEntity closeResponse = closeFuture.get(); - - Assert.assertEquals(403, closeResponse.getStatusCodeValue()); - - // verify that the create event was published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testAdminCancelAllSuccess() throws Exception { - ProxiedUserDetails adminUser = createUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); - - // create a bunch of queries - List queryIds = new ArrayList<>(); - long currentTimeMillis = System.currentTimeMillis(); - for (int i = 0; i < 10; i++) { - String queryId = createQuery(adminUser, createParams()); - mockServer.reset(); - - queryIds.add(queryId); - - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - // close all queries as the admin user - Future> cancelFuture = adminCancelAllQueries(adminUser); - - // the response should come back right away - ResponseEntity cancelResponse = cancelFuture.get(); - - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); - - // verify that query status was created correctly - List queryStatusList = queryStorageCache.getQueryStatus(); - - for (QueryStatus queryStatus : queryStatusList) { - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CANCELED, - 0, - 0, - 0, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - String queryId = queryStatus.getQueryKey().getQueryId(); - - // verify that the result queue is gone - Assert.assertFalse(queryQueueManager.queueExists(queryStatus.getQueryKey().getQueryId())); - - // verify that the query tasks are still present - assertTasksCreated(queryStatus.getQueryKey().getQueryId()); - - // @formatter:off - assertQueryRequestEvent( - "/query:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - // verify that there are no more events - Assert.assertEquals(0, queryRequestEvents.size()); - } - - @DirtiesContext - @Test - public void testAdminCancelAllFailure_notAdminUser() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a bunch of queries - List queryIds = new ArrayList<>(); - long currentTimeMillis = System.currentTimeMillis(); - for (int i = 0; i < 10; i++) { - String queryId = createQuery(authUser, createParams()); - mockServer.reset(); - - queryIds.add(queryId); - - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - // cancel all queries as the admin user - UriComponents uri = createUri("/adminCancelAll"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); - - // make the next call asynchronously - Future> cancelFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); - - // the response should come back right away - ResponseEntity cancelResponse = cancelFuture.get(); - - Assert.assertEquals(403, cancelResponse.getStatusCodeValue()); - - // verify that query status was created correctly - List queryStatusList = queryStorageCache.getQueryStatus(); - - // verify that none of the queries were canceled - for (QueryStatus queryStatus : queryStatusList) { - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, - 0, - 0, - 0, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - // verify that the result queue is still present - Assert.assertTrue(queryQueueManager.queueExists(queryStatus.getQueryKey().getQueryId())); - - // verify that the query tasks are still present - assertTasksCreated(queryStatus.getQueryKey().getQueryId()); - } - - // verify that there are no more events - Assert.assertEquals(0, queryRequestEvents.size()); - } - - @DirtiesContext - @Test - public void testCloseSuccess() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - long currentTimeMillis = System.currentTimeMillis(); - String queryId = createQuery(authUser, createParams()); - - // close the query - Future> closeFuture = closeQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity closeResponse = closeFuture.get(); - - Assert.assertEquals(200, closeResponse.getStatusCodeValue()); - - // verify that query status was created correctly - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CLOSED, - 0, - 0, - 0, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - // verify that the result queue is gone - Assert.assertFalse(queryQueueManager.queueExists(queryId)); - - // verify that the query tasks are still present - assertTasksCreated(queryId); - - // verify that the close event was published - Assert.assertEquals(2, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CLOSE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testCloseSuccess_activeNextCall() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - long currentTimeMillis = System.currentTimeMillis(); - String queryId = createQuery(authUser, createParams()); - - // call next on the query - Future> nextFuture = nextQuery(authUser, queryId); - - boolean nextCallActive = queryStorageCache.getQueryStatus(queryId).getActiveNextCalls() > 0; - while (!nextCallActive) { - try { - nextFuture.get(500, TimeUnit.MILLISECONDS); - } catch (TimeoutException e) { - nextCallActive = queryStorageCache.getQueryStatus(queryId).getActiveNextCalls() > 0; - if (TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - currentTimeMillis) > 5) { - throw e; - } - } - } - - // close the query - Future> closeFuture = closeQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity closeResponse = closeFuture.get(); - - Assert.assertEquals(200, closeResponse.getStatusCodeValue()); - - // verify that query status was created correctly - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CLOSED, - 0, - 0, - 1, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - // verify that the result queue is still present - Assert.assertTrue(queryQueueManager.queueExists(queryId)); - - // send enough results to return a page - // pump enough results into the queue to trigger a complete page - int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); - - // test field value pairings - MultiValueMap fieldValues = new LinkedMultiValueMap<>(); - fieldValues.add("LOKI", "ALLIGATOR"); - fieldValues.add("LOKI", "CLASSIC"); - - // @formatter:off - publishEventsToQueue( - queryId, - pageSize, - fieldValues, - "ALL"); - // @formatter:on - - // wait for the next call to return - nextFuture.get(); - - // verify that the result queue is now gone - Assert.assertFalse(queryQueueManager.queueExists(queryId)); - - // verify that the query tasks are still present - assertTasksCreated(queryId); - - // verify that the close event was published - Assert.assertEquals(3, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.NEXT, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CLOSE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @Test - public void testCloseFailure_queryNotFound() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - String queryId = UUID.randomUUID().toString(); - - // close the query - Future> closeFuture = closeQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity closeResponse = closeFuture.get(); - - Assert.assertEquals(404, closeResponse.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "No query object matches this id. " + queryId, - "Exception with no cause caught", - "404-1", - Iterables.getOnlyElement(closeResponse.getBody().getExceptions())); - // @formatter:on - - } - - @DirtiesContext - @Test - public void testCloseFailure_ownershipFailure() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - ProxiedUserDetails altAuthUser = createAltUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - // make the close call as an alternate user asynchronously - Future> future = closeQuery(altAuthUser, queryId); - - // the response should come back right away - ResponseEntity response = future.get(); - - Assert.assertEquals(401, response.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "Current user does not match user that defined query. altuserdn != userdn", - "Exception with no cause caught", - "401-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // verify that the next events were published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testCloseFailure_queryNotRunning() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - // close the query - Future> closeFuture = closeQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity closeResponse = closeFuture.get(); - - Assert.assertEquals(200, closeResponse.getStatusCodeValue()); - - // try to close the query again - closeFuture = closeQuery(authUser, queryId); - - // the response should come back right away - closeResponse = closeFuture.get(); - - Assert.assertEquals(400, closeResponse.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "Cannot call close on a query that is not running", - "Exception with no cause caught", - "400-1", - Iterables.getOnlyElement(closeResponse.getBody().getExceptions())); - // @formatter:on - - // verify that the next events were published - Assert.assertEquals(2, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CLOSE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testAdminCloseSuccess() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - ProxiedUserDetails adminUser = createAltUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); - - // create a valid query - long currentTimeMillis = System.currentTimeMillis(); - String queryId = createQuery(authUser, createParams()); - - // close the query as the admin user - Future> closeFuture = adminCloseQuery(adminUser, queryId); - - // the response should come back right away - ResponseEntity closeResponse = closeFuture.get(); - - Assert.assertEquals(200, closeResponse.getStatusCodeValue()); - - // verify that query status was created correctly - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CLOSED, - 0, - 0, - 0, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - // verify that the result queue is gone - Assert.assertFalse(queryQueueManager.queueExists(queryId)); - - // verify that the query tasks are still present - assertTasksCreated(queryId); - - // verify that the close event was published - Assert.assertEquals(2, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CLOSE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testAdminCloseFailure_notAdminUser() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - long currentTimeMillis = System.currentTimeMillis(); - String queryId = createQuery(authUser, createParams()); - - UriComponents uri = createUri(queryId + "/adminClose"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); - - // close the query - Future> closeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); - - // the response should come back right away - ResponseEntity closeResponse = closeFuture.get(); - - Assert.assertEquals(403, closeResponse.getStatusCodeValue()); - - // verify that the create event was published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testAdminCloseAllSuccess() throws Exception { - ProxiedUserDetails adminUser = createUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); - - // create a bunch of queries - long currentTimeMillis = System.currentTimeMillis(); - for (int i = 0; i < 10; i++) { - String queryId = createQuery(adminUser, createParams()); - mockServer.reset(); - - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - // close all queries as the admin user - Future> closeFuture = adminCloseAllQueries(adminUser); - - // the response should come back right away - ResponseEntity closeResponse = closeFuture.get(); - - Assert.assertEquals(200, closeResponse.getStatusCodeValue()); - - // verify that query status was created correctly - List queryStatusList = queryStorageCache.getQueryStatus(); - - for (QueryStatus queryStatus : queryStatusList) { - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CLOSED, - 0, - 0, - 0, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - String queryId = queryStatus.getQueryKey().getQueryId(); - - // verify that the result queue is gone - Assert.assertFalse(queryQueueManager.queueExists(queryStatus.getQueryKey().getQueryId())); - - // verify that the query tasks are still present - assertTasksCreated(queryStatus.getQueryKey().getQueryId()); - - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CLOSE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - // verify that there are no more events - Assert.assertEquals(0, queryRequestEvents.size()); - } - - @DirtiesContext - @Test - public void testAdminCloseAllFailure_notAdminUser() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a bunch of queries - List queryIds = new ArrayList<>(); - long currentTimeMillis = System.currentTimeMillis(); - for (int i = 0; i < 10; i++) { - String queryId = createQuery(authUser, createParams()); - mockServer.reset(); - - queryIds.add(queryId); - - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - // close all queries as the admin user - UriComponents uri = createUri("/adminCloseAll"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); - - // make the next call asynchronously - Future> closeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); - - // the response should come back right away - ResponseEntity closeResponse = closeFuture.get(); - - Assert.assertEquals(403, closeResponse.getStatusCodeValue()); - - // verify that query status was created correctly - List queryStatusList = queryStorageCache.getQueryStatus(); - - // verify that none of the queries were canceled - for (QueryStatus queryStatus : queryStatusList) { - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, - 0, - 0, - 0, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - // verify that the result queue is still present - Assert.assertTrue(queryQueueManager.queueExists(queryStatus.getQueryKey().getQueryId())); - - // verify that the query tasks are still present - assertTasksCreated(queryStatus.getQueryKey().getQueryId()); - } - - // verify that there are no more events - Assert.assertEquals(0, queryRequestEvents.size()); - } - - @DirtiesContext - @Test - public void testResetSuccess_resetOnDefined() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // define a valid query - long currentTimeMillis = System.currentTimeMillis(); - String queryId = defineQuery(authUser, createParams()); - - mockServer.reset(); - auditSentSetup(); - - // reset the query - Future> resetFuture = resetQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity response = resetFuture.get(); - - Assert.assertEquals(200, response.getStatusCodeValue()); - - // @formatter:off - assertGenericResponse( - true, - HttpStatus.Series.SUCCESSFUL, - response); - // @formatter:on - - String resetQueryId = (String) response.getBody().getResult(); - - // verify that a new query id was created - Assert.assertNotEquals(queryId, resetQueryId); - - // verify that an audit record was sent - assertAuditSent(resetQueryId); - - // verify that original query was canceled - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.DEFINED, - 0, - 0, - 0, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - // verify that new query was created - QueryStatus resetQueryStatus = queryStorageCache.getQueryStatus(resetQueryId); - - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, - 0, - 0, - 0, - 0, - currentTimeMillis, - resetQueryStatus); - // @formatter:on - - // make sure the queries are equal (ignoring the query id) - queryStatus.getQuery().setId(resetQueryStatus.getQuery().getId()); - Assert.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); - - // verify that events were published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - resetQueryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testResetSuccess_resetOnCreated() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // define a valid query - long currentTimeMillis = System.currentTimeMillis(); - String queryId = createQuery(authUser, createParams()); - - mockServer.reset(); - auditSentSetup(); - - // reset the query - Future> resetFuture = resetQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity response = resetFuture.get(); - - Assert.assertEquals(200, response.getStatusCodeValue()); - - // @formatter:off - assertGenericResponse( - true, - HttpStatus.Series.SUCCESSFUL, - response); - // @formatter:on - - String resetQueryId = (String) response.getBody().getResult(); - - // verify that a new query id was created - Assert.assertNotEquals(queryId, resetQueryId); - - // verify that an audit record was sent - assertAuditSent(resetQueryId); - - // verify that original query was canceled - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CANCELED, - 0, - 0, - 0, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - // verify that new query was created - QueryStatus resetQueryStatus = queryStorageCache.getQueryStatus(resetQueryId); - - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, - 0, - 0, - 0, - 0, - currentTimeMillis, - resetQueryStatus); - // @formatter:on - - // make sure the queries are equal (ignoring the query id) - queryStatus.getQuery().setId(resetQueryStatus.getQuery().getId()); - Assert.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); - - // verify that events were published - Assert.assertEquals(4, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "/query:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - resetQueryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testResetSuccess_resetOnClosed() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // define a valid query - long currentTimeMillis = System.currentTimeMillis(); - String queryId = createQuery(authUser, createParams()); - - // close the query - Future> closeFuture = closeQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity closeResponse = closeFuture.get(); - - Assert.assertEquals(200, closeResponse.getStatusCodeValue()); - - mockServer.reset(); - auditSentSetup(); - - // reset the query - Future> resetFuture = resetQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity response = resetFuture.get(); - - Assert.assertEquals(200, response.getStatusCodeValue()); - - // @formatter:off - assertGenericResponse( - true, - HttpStatus.Series.SUCCESSFUL, - response); - // @formatter:on - - String resetQueryId = (String) response.getBody().getResult(); - - // verify that a new query id was created - Assert.assertNotEquals(queryId, resetQueryId); - - // verify that an audit record was sent - assertAuditSent(resetQueryId); - - // verify that original query was closed - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CLOSED, - 0, - 0, - 0, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - // verify that new query was created - QueryStatus resetQueryStatus = queryStorageCache.getQueryStatus(resetQueryId); - - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, - 0, - 0, - 0, - 0, - currentTimeMillis, - resetQueryStatus); - // @formatter:on - - // make sure the queries are equal (ignoring the query id) - queryStatus.getQuery().setId(resetQueryStatus.getQuery().getId()); - Assert.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); - - // verify that events were published - Assert.assertEquals(3, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CLOSE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - resetQueryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testResetSuccess_resetOnCanceled() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // define a valid query - long currentTimeMillis = System.currentTimeMillis(); - String queryId = createQuery(authUser, createParams()); - - // cancel the query - Future> cancelFuture = cancelQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity cancelResponse = cancelFuture.get(); - - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); - - mockServer.reset(); - auditSentSetup(); - - // reset the query - Future> resetFuture = resetQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity response = resetFuture.get(); - - Assert.assertEquals(200, response.getStatusCodeValue()); - - // @formatter:off - assertGenericResponse( - true, - HttpStatus.Series.SUCCESSFUL, - response); - // @formatter:on - - String resetQueryId = (String) response.getBody().getResult(); - - // verify that a new query id was created - Assert.assertNotEquals(queryId, resetQueryId); - - // verify that an audit record was sent - assertAuditSent(resetQueryId); - - // verify that original query was canceled - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CANCELED, - 0, - 0, - 0, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - // verify that new query was created - QueryStatus resetQueryStatus = queryStorageCache.getQueryStatus(resetQueryId); - - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, - 0, - 0, - 0, - 0, - currentTimeMillis, - resetQueryStatus); - // @formatter:on - - // make sure the queries are equal (ignoring the query id) - queryStatus.getQuery().setId(resetQueryStatus.getQuery().getId()); - Assert.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); - - // verify that events were published - Assert.assertEquals(4, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "/query:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - resetQueryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testResetFailure_queryNotFound() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - String queryId = UUID.randomUUID().toString(); - - auditNotSentSetup(); - - // reset the query - UriComponents uri = createUri(queryId + "/reset"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); - - // close the query - Future> resetFuture = Executors.newSingleThreadExecutor() - .submit(() -> jwtRestTemplate.exchange(requestEntity, BaseResponse.class)); - - // the response should come back right away - ResponseEntity response = resetFuture.get(); - - Assert.assertEquals(404, response.getStatusCodeValue()); - - // make sure no audits were sent - assertAuditNotSent(); - - // @formatter:off - assertQueryException( - "No query object matches this id. " + queryId, - "Exception with no cause caught", - "404-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); - } - - @DirtiesContext - @Test - public void testResetFailure_ownershipFailure() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - ProxiedUserDetails altAuthUser = createAltUserDetails(); - - // define a valid query - String queryId = createQuery(authUser, createParams()); - - mockServer.reset(); - auditNotSentSetup(); - - // reset the query - UriComponents uri = createUri(queryId + "/reset"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(altAuthUser, null, null, HttpMethod.PUT, uri); - - // close the query - Future> resetFuture = Executors.newSingleThreadExecutor() - .submit(() -> jwtRestTemplate.exchange(requestEntity, BaseResponse.class)); - - // the response should come back right away - ResponseEntity response = resetFuture.get(); - - Assert.assertEquals(401, response.getStatusCodeValue()); - - // make sure no audits were sent - assertAuditNotSent(); - - // @formatter:off - assertQueryException( - "Current user does not match user that defined query. altuserdn != userdn", - "Exception with no cause caught", - "401-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // verify that no events were published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testRemoveSuccess_removeOnDefined() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // define a valid query - String queryId = defineQuery(authUser, createParams()); - - // remove the query - Future> removeFuture = removeQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity response = removeFuture.get(); - - Assert.assertEquals(200, response.getStatusCodeValue()); - - // verify that original query was removed - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - Assert.assertNull(queryStatus); - - // verify that events were published - Assert.assertEquals(0, queryRequestEvents.size()); - } - - @DirtiesContext - @Test - public void testRemoveSuccess_removeOnClosed() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - // close the query - Future> closeFuture = closeQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity closeResponse = closeFuture.get(); - - Assert.assertEquals(200, closeResponse.getStatusCodeValue()); - - // remove the query - Future> removeFuture = removeQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity response = removeFuture.get(); - - Assert.assertEquals(200, response.getStatusCodeValue()); - - // verify that original query was removed - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - Assert.assertNull(queryStatus); - - // verify that events were published - Assert.assertEquals(2, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CLOSE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testRemoveSuccess_removeOnCanceled() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - // cancel the query - Future> cancelFuture = cancelQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity cancelResponse = cancelFuture.get(); - - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); - - // remove the query - Future> removeFuture = removeQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity response = removeFuture.get(); - - Assert.assertEquals(200, response.getStatusCodeValue()); - - // verify that original query was removed - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - Assert.assertNull(queryStatus); - - // verify that events were published - Assert.assertEquals(3, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "/query:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testRemoveFailure_removeOnCreated() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - // remove the query - Future> removeFuture = removeQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity response = removeFuture.get(); - - Assert.assertEquals(400, response.getStatusCodeValue()); - - Assert.assertEquals("Cannot remove a running query.", Iterables.getOnlyElement(response.getBody().getExceptions()).getMessage()); - - // verify that original query was not removed - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - Assert.assertNotNull(queryStatus); - - // verify that events were published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testRemoveFailure_removeOnClosedActiveNext() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - // call next on the query - nextQuery(authUser, queryId); - - // close the query - Future> closeFuture = closeQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity closeResponse = closeFuture.get(); - - Assert.assertEquals(200, closeResponse.getStatusCodeValue()); - - // remove the query - Future> removeFuture = removeQuery(authUser, queryId); - - // the response should come back right away - ResponseEntity response = removeFuture.get(); - - Assert.assertEquals(400, response.getStatusCodeValue()); - - Assert.assertEquals("Cannot remove a running query.", Iterables.getOnlyElement(response.getBody().getExceptions()).getMessage()); - - // verify that original query was not removed - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - Assert.assertNotNull(queryStatus); - - // verify that events were published - Assert.assertEquals(3, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.NEXT, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CLOSE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @Test - public void testRemoveFailure_queryNotFound() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - String queryId = UUID.randomUUID().toString(); - - // remove the query - UriComponents uri = createUri(queryId + "/remove"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); - - // close the query - Future> resetFuture = Executors.newSingleThreadExecutor() - .submit(() -> jwtRestTemplate.exchange(requestEntity, BaseResponse.class)); - - // the response should come back right away - ResponseEntity response = resetFuture.get(); - - Assert.assertEquals(404, response.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "No query object matches this id. " + queryId, - "Exception with no cause caught", - "404-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); - } - - @DirtiesContext - @Test - public void testRemoveFailure_ownershipFailure() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - ProxiedUserDetails altAuthUser = createAltUserDetails(); - - // define a valid query - String queryId = defineQuery(authUser, createParams()); - - // remove the query - UriComponents uri = createUri(queryId + "/remove"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(altAuthUser, null, null, HttpMethod.DELETE, uri); - - // close the query - Future> resetFuture = Executors.newSingleThreadExecutor() - .submit(() -> jwtRestTemplate.exchange(requestEntity, BaseResponse.class)); - - // the response should come back right away - ResponseEntity response = resetFuture.get(); - - Assert.assertEquals(401, response.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "Current user does not match user that defined query. altuserdn != userdn", - "Exception with no cause caught", - "401-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); - } - - @DirtiesContext - @Test - public void testAdminRemoveSuccess() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - ProxiedUserDetails adminUser = createAltUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); - - // define a valid query - String queryId = defineQuery(authUser, createParams()); - - // remove the query - Future> removeFuture = adminRemoveQuery(adminUser, queryId); - - // the response should come back right away - ResponseEntity response = removeFuture.get(); - - Assert.assertEquals(200, response.getStatusCodeValue()); - - // verify that original query was removed - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - Assert.assertNull(queryStatus); - - // verify that events were published - Assert.assertEquals(0, queryRequestEvents.size()); - } - - @DirtiesContext - @Test - public void testAdminRemoveFailure_notAdminUser() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // define a valid query - String queryId = defineQuery(authUser, createParams()); - - // remove the query - UriComponents uri = createUri(queryId + "/adminRemove"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); - - // remove the queries - Future> removeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); - - // the response should come back right away - ResponseEntity response = removeFuture.get(); - - Assert.assertEquals(403, response.getStatusCodeValue()); - - // verify that original query was not removed - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - Assert.assertNotNull(queryStatus); - - // verify that events were published - Assert.assertEquals(0, queryRequestEvents.size()); - } - - @DirtiesContext - @Test - public void testAdminRemoveAllSuccess() throws Exception { - ProxiedUserDetails adminUser = createUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); - - // define a bunch of queries - for (int i = 0; i < 10; i++) { - defineQuery(adminUser, createParams()); - } - - // remove all queries as the admin user - Future> removeFuture = adminRemoveAllQueries(adminUser); - - // the response should come back right away - ResponseEntity removeResponse = removeFuture.get(); - - Assert.assertEquals(200, removeResponse.getStatusCodeValue()); - - // verify that query status was created correctly - List queryStatusList = queryStorageCache.getQueryStatus(); - - Assert.assertEquals(0, queryStatusList.size()); - - // verify that there are no events - Assert.assertEquals(0, queryRequestEvents.size()); - } - - @DirtiesContext - @Test - public void testAdminRemoveAllFailure_notAdminUser() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // define a bunch of queries - for (int i = 0; i < 10; i++) { - defineQuery(authUser, createParams()); - } - - UriComponents uri = createUri("/adminRemoveAll"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); - - // remove the queries - Future> removeFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); - - // the response should come back right away - ResponseEntity removeResponse = removeFuture.get(); - - Assert.assertEquals(403, removeResponse.getStatusCodeValue()); - - // verify that query status was created correctly - List queryStatusList = queryStorageCache.getQueryStatus(); - - Assert.assertEquals(10, queryStatusList.size()); - - // verify that there are no events - Assert.assertEquals(0, queryRequestEvents.size()); - } - - @DirtiesContext - @Test - public void testUpdateSuccess_updateOnDefined() throws Exception { - ProxiedUserDetails authUser = createUserDetails(null, Arrays.asList("ALL", "NONE")); - - // define a valid query - String queryId = defineQuery(authUser, createParams()); - - String newQuery = "SOME_OTHER_FIELD:SOME_OTHER_VALUE"; - String newAuths = "ALL,NONE"; - String newBegin = "20100101 000000.000"; - String newEnd = "20600101 000000.000"; - String newLogic = "AltEventQuery"; - int newPageSize = 100; - - MultiValueMap updateParams = new LinkedMultiValueMap<>(); - updateParams.set(QUERY, newQuery); - updateParams.set(QUERY_AUTHORIZATIONS, newAuths); - updateParams.set(BEGIN_DATE, newBegin); - updateParams.set(END_DATE, newEnd); - updateParams.set(QUERY_LOGIC_NAME, newLogic); - updateParams.set(PAGESIZE, Integer.toString(newPageSize)); - - // update the query - Future> updateFuture = updateQuery(authUser, queryId, updateParams); - - // the response should come back right away - ResponseEntity response = updateFuture.get(); - - Assert.assertEquals(200, response.getStatusCodeValue()); - - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - // make sure the query was updated - Assert.assertEquals(newQuery, queryStatus.getQuery().getQuery()); - Assert.assertEquals(newAuths, queryStatus.getQuery().getQueryAuthorizations()); - Assert.assertEquals(newBegin, DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate())); - Assert.assertEquals(newEnd, DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate())); - Assert.assertEquals(newLogic, queryStatus.getQuery().getQueryLogicName()); - Assert.assertEquals(newPageSize, queryStatus.getQuery().getPagesize()); - - // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); - } - - @DirtiesContext - @Test - public void testUpdateSuccess_updateOnCreated() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - int newPageSize = 100; - - MultiValueMap updateParams = new LinkedMultiValueMap<>(); - updateParams.set(PAGESIZE, Integer.toString(newPageSize)); - - // update the query - Future> updateFuture = updateQuery(authUser, queryId, updateParams); - - // the response should come back right away - ResponseEntity response = updateFuture.get(); - - Assert.assertEquals(200, response.getStatusCodeValue()); - - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - // make sure the query was updated - Assert.assertEquals(newPageSize, queryStatus.getQuery().getPagesize()); - - // verify that events were published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testUpdateFailure_unsafeParamUpdateQuery() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - String newQuery = "SOME_OTHER_FIELD:SOME_OTHER_VALUE"; - - MultiValueMap updateParams = new LinkedMultiValueMap<>(); - updateParams.set(QUERY, newQuery); - - // update the query - UriComponents uri = createUri(queryId + "/update"); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); - - // make the update call asynchronously - Future> updateFuture = Executors.newSingleThreadExecutor() - .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); - - // the response should come back right away - ResponseEntity response = updateFuture.get(); - - Assert.assertEquals(400, response.getStatusCodeValue()); - - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - // make sure the query was not updated - Assert.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); - - // @formatter:off - assertQueryException( - "Cannot update the following parameters for a running query: query", - "Exception with no cause caught", - "400-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // verify that events were published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testUpdateFailure_unsafeParamUpdateDate() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - String newBegin = "20100101 000000.000"; - String newEnd = "20600101 000000.000"; - - MultiValueMap updateParams = new LinkedMultiValueMap<>(); - updateParams.set(BEGIN_DATE, newBegin); - updateParams.set(END_DATE, newEnd); - - // update the query - UriComponents uri = createUri(queryId + "/update"); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); - - // make the update call asynchronously - Future> updateFuture = Executors.newSingleThreadExecutor() - .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); - - // the response should come back right away - ResponseEntity response = updateFuture.get(); - - Assert.assertEquals(400, response.getStatusCodeValue()); - - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - // make sure the query was not updated - Assert.assertEquals(TEST_QUERY_BEGIN, DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate())); - Assert.assertEquals(TEST_QUERY_END, DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate())); - - // @formatter:off - assertQueryException( - "Cannot update the following parameters for a running query: begin, end", - "Exception with no cause caught", - "400-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // verify that events were published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testUpdateFailure_unsafeParamUpdateLogic() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - String newLogic = "AltEventQuery"; - - MultiValueMap updateParams = new LinkedMultiValueMap<>(); - updateParams.set(QUERY_LOGIC_NAME, newLogic); - - // update the query - UriComponents uri = createUri(queryId + "/update"); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); - - // make the update call asynchronously - Future> updateFuture = Executors.newSingleThreadExecutor() - .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); - - // the response should come back right away - ResponseEntity response = updateFuture.get(); - - Assert.assertEquals(400, response.getStatusCodeValue()); - - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - // make sure the query was not updated - Assert.assertEquals("EventQuery", queryStatus.getQuery().getQueryLogicName()); - - // @formatter:off - assertQueryException( - "Cannot update the following parameters for a running query: logicName", - "Exception with no cause caught", - "400-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // verify that events were published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testUpdateFailure_unsafeParamUpdateAuths() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - String newAuths = "ALL,NONE"; - - MultiValueMap updateParams = new LinkedMultiValueMap<>(); - updateParams.set(QUERY_AUTHORIZATIONS, newAuths); - - // update the query - UriComponents uri = createUri(queryId + "/update"); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); - - // make the update call asynchronously - Future> updateFuture = Executors.newSingleThreadExecutor() - .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); - - // the response should come back right away - ResponseEntity response = updateFuture.get(); - - Assert.assertEquals(400, response.getStatusCodeValue()); - - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - // make sure the query was not updated - Assert.assertEquals(TEST_QUERY_AUTHORIZATIONS, queryStatus.getQuery().getQueryAuthorizations()); - - // @formatter:off - assertQueryException( - "Cannot update the following parameters for a running query: auths", - "Exception with no cause caught", - "400-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // verify that events were published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testUpdateFailure_nullParams() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - String queryId = createQuery(authUser, createParams()); - - MultiValueMap updateParams = new LinkedMultiValueMap<>(); - - // update the query - UriComponents uri = createUri(queryId + "/update"); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); - - // make the update call asynchronously - Future> updateFuture = Executors.newSingleThreadExecutor() - .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); - - // the response should come back right away - ResponseEntity response = updateFuture.get(); - - Assert.assertEquals(400, response.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "No parameters specified for update.", - "Exception with no cause caught", - "400-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // verify that events were published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @Test - public void testUpdateFailure_queryNotFound() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - String queryId = UUID.randomUUID().toString(); - - MultiValueMap updateParams = new LinkedMultiValueMap<>(); - updateParams.set(QUERY_STRING, TEST_QUERY_STRING); - - // update the query - UriComponents uri = createUri(queryId + "/update"); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); - - // make the update call asynchronously - Future> updateFuture = Executors.newSingleThreadExecutor() - .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); - - // the response should come back right away - ResponseEntity response = updateFuture.get(); - - Assert.assertEquals(404, response.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "No query object matches this id. " + queryId, - "Exception with no cause caught", - "404-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); - } - - @DirtiesContext - @Test - public void testUpdateFailure_ownershipFailure() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - ProxiedUserDetails altAuthUser = createAltUserDetails(); - - // define a valid query - String queryId = defineQuery(authUser, createParams()); - - String newQuery = "SOME_OTHER_FIELD:SOME_OTHER_VALUE"; - - MultiValueMap updateParams = new LinkedMultiValueMap<>(); - updateParams.set(QUERY, newQuery); - - // update the query - UriComponents uri = createUri(queryId + "/update"); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(altAuthUser, updateParams, null, HttpMethod.PUT, uri); - - // make the update call asynchronously - Future> updateFuture = Executors.newSingleThreadExecutor() - .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); - - // the response should come back right away - ResponseEntity response = updateFuture.get(); - - Assert.assertEquals(401, response.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "Current user does not match user that defined query. altuserdn != userdn", - "Exception with no cause caught", - "401-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - // make sure the query was not updated - Assert.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); - - // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); - } - - @DirtiesContext - @Test - public void testDuplicateSuccess_duplicateOnDefined() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // define a valid query - long currentTimeMillis = System.currentTimeMillis(); - String queryId = defineQuery(authUser, createParams()); - - MultiValueMap updateParams = new LinkedMultiValueMap<>(); - - mockServer.reset(); - auditSentSetup(); - - // duplicate the query - Future> duplicateFuture = duplicateQuery(authUser, queryId, updateParams); - - // the response should come back right away - ResponseEntity response = duplicateFuture.get(); - - Assert.assertEquals(200, response.getStatusCodeValue()); - - String dupeQueryId = (String) response.getBody().getResult(); - - // make sure an audit message was sent - assertAuditSent(dupeQueryId); - - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.DEFINED, - 0, - 0, - 0, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, - 0, - 0, - 0, - 0, - currentTimeMillis, - dupeQueryStatus); - // @formatter:on - - // make sure the queries are identical - Assert.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); - Assert.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); - Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), - DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); - Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), - DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); - Assert.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); - Assert.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); - - // verify that no events were published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - dupeQueryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testDuplicateSuccess_duplicateOnCreated() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - long currentTimeMillis = System.currentTimeMillis(); - String queryId = createQuery(authUser, createParams()); - - MultiValueMap updateParams = new LinkedMultiValueMap<>(); - - mockServer.reset(); - auditSentSetup(); - - // duplicate the query - Future> duplicateFuture = duplicateQuery(authUser, queryId, updateParams); - - // the response should come back right away - ResponseEntity response = duplicateFuture.get(); - - Assert.assertEquals(200, response.getStatusCodeValue()); - - String dupeQueryId = (String) response.getBody().getResult(); - - // make sure an audit message was sent - assertAuditSent(dupeQueryId); - - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, - 0, - 0, - 0, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, - 0, - 0, - 0, - 0, - currentTimeMillis, - dupeQueryStatus); - // @formatter:on - - // make sure the queries are identical - Assert.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); - Assert.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); - Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), - DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); - Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), - DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); - Assert.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); - Assert.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); - - // verify that no events were published - Assert.assertEquals(2, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - dupeQueryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testDuplicateSuccess_duplicateOnCanceled() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - long currentTimeMillis = System.currentTimeMillis(); - String queryId = createQuery(authUser, createParams()); - - // cancel the query - Future> cancelFuture = cancelQuery(authUser, queryId); - - // this should return immediately - ResponseEntity cancelResponse = cancelFuture.get(); - - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); - - MultiValueMap updateParams = new LinkedMultiValueMap<>(); - - mockServer.reset(); - auditSentSetup(); - - // duplicate the query - Future> duplicateFuture = duplicateQuery(authUser, queryId, updateParams); - - // the response should come back right away - ResponseEntity response = duplicateFuture.get(); - - Assert.assertEquals(200, response.getStatusCodeValue()); - - String dupeQueryId = (String) response.getBody().getResult(); - - // make sure an audit message was sent - assertAuditSent(dupeQueryId); - - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CANCELED, - 0, - 0, - 0, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, - 0, - 0, - 0, - 0, - currentTimeMillis, - dupeQueryStatus); - // @formatter:on - - // make sure the queries are identical - Assert.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); - Assert.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); - Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), - DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); - Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), - DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); - Assert.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); - Assert.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); - - // verify that no events were published - Assert.assertEquals(4, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "/query:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CANCEL, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - dupeQueryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testDuplicateSuccess_duplicateOnClosed() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - long currentTimeMillis = System.currentTimeMillis(); - String queryId = createQuery(authUser, createParams()); - - // close the query - Future> closeFuture = closeQuery(authUser, queryId); - - // this should return immediately - ResponseEntity closeResponse = closeFuture.get(); - - Assert.assertEquals(200, closeResponse.getStatusCodeValue()); - - MultiValueMap updateParams = new LinkedMultiValueMap<>(); - - mockServer.reset(); - auditSentSetup(); - - // duplicate the query - Future> duplicateFuture = duplicateQuery(authUser, queryId, updateParams); - - // the response should come back right away - ResponseEntity response = duplicateFuture.get(); - - Assert.assertEquals(200, response.getStatusCodeValue()); - - String dupeQueryId = (String) response.getBody().getResult(); - - // make sure an audit message was sent - assertAuditSent(dupeQueryId); - - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CLOSED, - 0, - 0, - 0, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, - 0, - 0, - 0, - 0, - currentTimeMillis, - dupeQueryStatus); - // @formatter:on - - // make sure the queries are identical - Assert.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); - Assert.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); - Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), - DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); - Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), - DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); - Assert.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); - Assert.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); - - // verify that no events were published - Assert.assertEquals(3, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CLOSE, - queryId, - queryRequestEvents.removeLast()); - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - dupeQueryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testDuplicateSuccess_update() throws Exception { - ProxiedUserDetails authUser = createUserDetails(null, Arrays.asList("ALL", "NONE")); - - // define a valid query - long currentTimeMillis = System.currentTimeMillis(); - String queryId = defineQuery(authUser, createParams()); - - String newQuery = "SOME_OTHER_FIELD:SOME_OTHER_VALUE"; - String newAuths = "ALL,NONE"; - String newBegin = "20100101 000000.000"; - String newEnd = "20600101 000000.000"; - String newLogic = "AltEventQuery"; - - MultiValueMap updateParams = new LinkedMultiValueMap<>(); - updateParams.set(QUERY, newQuery); - updateParams.set(QUERY_AUTHORIZATIONS, newAuths); - updateParams.set(BEGIN_DATE, newBegin); - updateParams.set(END_DATE, newEnd); - updateParams.set(QUERY_LOGIC_NAME, newLogic); - - mockServer.reset(); - auditSentSetup(); - - // duplicate the query - Future> duplicateFuture = duplicateQuery(authUser, queryId, updateParams); - - // the response should come back right away - ResponseEntity response = duplicateFuture.get(); - - Assert.assertEquals(200, response.getStatusCodeValue()); - - String dupeQueryId = (String) response.getBody().getResult(); - - // make sure an audit message was sent - assertAuditSent(dupeQueryId); - - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.DEFINED, - 0, - 0, - 0, - 0, - currentTimeMillis, - queryStatus); - // @formatter:on - - QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); - // @formatter:off - assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, - 0, - 0, - 0, - 0, - currentTimeMillis, - dupeQueryStatus); - // @formatter:on - - // make sure the original query is unchanged - Assert.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); - Assert.assertEquals(TEST_QUERY_AUTHORIZATIONS, queryStatus.getQuery().getQueryAuthorizations()); - Assert.assertEquals(TEST_QUERY_BEGIN, DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate())); - Assert.assertEquals(TEST_QUERY_END, DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate())); - Assert.assertEquals("EventQuery", queryStatus.getQuery().getQueryLogicName()); - - // make sure the duplicated query is updated - Assert.assertEquals(newQuery, dupeQueryStatus.getQuery().getQuery()); - Assert.assertEquals(newAuths, dupeQueryStatus.getQuery().getQueryAuthorizations()); - Assert.assertEquals(newBegin, DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); - Assert.assertEquals(newEnd, DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); - Assert.assertEquals(newLogic, dupeQueryStatus.getQuery().getQueryLogicName()); - - // verify that no events were published - Assert.assertEquals(1, queryRequestEvents.size()); - // @formatter:off - assertQueryRequestEvent( - "executor-unassigned:**", - QueryRequest.Method.CREATE, - dupeQueryId, - queryRequestEvents.removeLast()); - // @formatter:on - } - - @DirtiesContext - @Test - public void testDuplicateFailure_invalidUpdate() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // define a valid query - String queryId = defineQuery(authUser, createParams()); - - String newLogic = "SomeBogusLogic"; - - MultiValueMap updateParams = new LinkedMultiValueMap<>(); - updateParams.set(QUERY_LOGIC_NAME, newLogic); - - mockServer.reset(); - auditNotSentSetup(); - - // duplicate the query - UriComponents uri = createUri(queryId + "/duplicate"); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.POST, uri); - - // make the duplicate call asynchronously - Future> duplicateFuture = Executors.newSingleThreadExecutor() - .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); - - // the response should come back right away - ResponseEntity response = duplicateFuture.get(); - - Assert.assertEquals(400, response.getStatusCodeValue()); - - // make sure an audit message wasn't sent - assertAuditNotSent(); - - // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); - } - - @Test - public void testDuplicateFailure_queryNotFound() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - String queryId = UUID.randomUUID().toString(); - - MultiValueMap updateParams = new LinkedMultiValueMap<>(); - - mockServer.reset(); - auditNotSentSetup(); - - // duplicate the query - UriComponents uri = createUri(queryId + "/duplicate"); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.POST, uri); - - // make the duplicate call asynchronously - Future> duplicateFuture = Executors.newSingleThreadExecutor() - .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); - - // the response should come back right away - ResponseEntity response = duplicateFuture.get(); - - Assert.assertEquals(404, response.getStatusCodeValue()); - - // @formatter:off - assertQueryException( - "No query object matches this id. " + queryId, - "Exception with no cause caught", - "404-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - // make sure an audit message wasn't sent - assertAuditNotSent(); - - // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); - } - - @DirtiesContext - @Test - public void testDuplicateFailure_ownershipFailure() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - ProxiedUserDetails altAuthUser = createAltUserDetails(); - - // define a valid query - String queryId = defineQuery(authUser, createParams()); - - MultiValueMap updateParams = new LinkedMultiValueMap<>(); - - mockServer.reset(); - auditNotSentSetup(); - - // duplicate the query - UriComponents uri = createUri(queryId + "/duplicate"); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(altAuthUser, updateParams, null, HttpMethod.POST, uri); - - // make the duplicate call asynchronously - Future> duplicateFuture = Executors.newSingleThreadExecutor() - .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); - - // the response should come back right away - ResponseEntity response = duplicateFuture.get(); - - Assert.assertEquals(401, response.getStatusCodeValue()); - - // make sure an audit message wasn't sent - assertAuditNotSent(); - - // @formatter:off - assertQueryException( - "Current user does not match user that defined query. altuserdn != userdn", - "Exception with no cause caught", - "401-1", - Iterables.getOnlyElement(response.getBody().getExceptions())); - // @formatter:on - - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - - // make sure the query was not updated - Assert.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); - - // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); - } - - @DirtiesContext - @Test - public void testListSuccess() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - ProxiedUserDetails altAuthUser = createAltUserDetails(); - - // define a bunch of queries as the original user - List queryIds = new ArrayList<>(); - for (int i = 0; i < 5; i++) { - String queryId = createQuery(authUser, createParams()); - mockServer.reset(); - - queryIds.add(queryId); - } - - // define a bunch of queries as the alternate user - List altQueryIds = new ArrayList<>(); - for (int i = 0; i < 5; i++) { - String queryId = defineQuery(altAuthUser, createParams()); - mockServer.reset(); - - altQueryIds.add(queryId); - } - - // list queries as the original user - Future> listFuture = listQueries(authUser, null, null); - - // this should return immediately - ResponseEntity listResponse = listFuture.get(); - - Assert.assertEquals(200, listResponse.getStatusCodeValue()); - - QueryImplListResponse result = listResponse.getBody(); - - Assert.assertEquals(5, result.getNumResults()); - - List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); - - Collections.sort(queryIds); - Collections.sort(actualQueryIds); - - Assert.assertEquals(queryIds, actualQueryIds); - } - - @DirtiesContext - @Test - public void testListSuccess_filterOnQueryId() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // define a bunch of queries as the original user - List queryIds = new ArrayList<>(); - for (int i = 0; i < 5; i++) { - String queryId = createQuery(authUser, createParams()); - mockServer.reset(); - - queryIds.add(queryId); - } - - // list queries - Future> listFuture = listQueries(authUser, queryIds.get(0), null); - - // this should return immediately - ResponseEntity listResponse = listFuture.get(); - - Assert.assertEquals(200, listResponse.getStatusCodeValue()); - - QueryImplListResponse result = listResponse.getBody(); - - Assert.assertEquals(1, result.getNumResults()); - - List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); - - Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); - } - - @DirtiesContext - @Test - public void testListSuccess_filterOnQueryName() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - String uniqueQueryName = "Unique Query"; - - // define a bunch of queries as the original user - List queryIds = new ArrayList<>(); - for (int i = 0; i < 5; i++) { - MultiValueMap params = createParams(); - if (i == 0) { - params.set(QUERY_NAME, uniqueQueryName); - } - - String queryId = createQuery(authUser, params); - mockServer.reset(); - - queryIds.add(queryId); - } - - // list queries - Future> listFuture = listQueries(authUser, null, uniqueQueryName); - - // this should return immediately - ResponseEntity listResponse = listFuture.get(); - - Assert.assertEquals(200, listResponse.getStatusCodeValue()); - - QueryImplListResponse result = listResponse.getBody(); - - Assert.assertEquals(1, result.getNumResults()); - - List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); - - Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); - } - - @DirtiesContext - @Test - public void testListSuccess_filterOnMultiple() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - String uniqueQueryName = "Unique Query"; - - // define a bunch of queries as the original user - List queryIds = new ArrayList<>(); - for (int i = 0; i < 5; i++) { - MultiValueMap params = createParams(); - if (i == 0) { - params.set(QUERY_NAME, uniqueQueryName); - } - - String queryId = createQuery(authUser, params); - mockServer.reset(); - - queryIds.add(queryId); - } - - // list queries with just the query ID and a bogus name - Future> listFuture = listQueries(authUser, queryIds.get(0), "bogus name"); - - // this should return immediately - ResponseEntity listResponse = listFuture.get(); - - Assert.assertEquals(200, listResponse.getStatusCodeValue()); - - QueryImplListResponse result = listResponse.getBody(); - - Assert.assertEquals(0, result.getNumResults()); - - // list queries with just the query name and a bogus ID - listFuture = listQueries(authUser, UUID.randomUUID().toString(), uniqueQueryName); - - // this should return immediately - listResponse = listFuture.get(); - - Assert.assertEquals(200, listResponse.getStatusCodeValue()); - - result = listResponse.getBody(); - - Assert.assertEquals(0, result.getNumResults()); - - // list queries with just the query name and a bogus ID - listFuture = listQueries(authUser, queryIds.get(0), uniqueQueryName); - - // this should return immediately - listResponse = listFuture.get(); - - Assert.assertEquals(200, listResponse.getStatusCodeValue()); - - result = listResponse.getBody(); - - Assert.assertEquals(1, result.getNumResults()); - - List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); - - Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); - } - - @DirtiesContext - @Test - public void testListFailure_ownershipFailure() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - ProxiedUserDetails altAuthUser = createAltUserDetails(); - - String uniqueQueryName = "Unique Query"; - - // define a bunch of queries as the original user - List queryIds = new ArrayList<>(); - for (int i = 0; i < 5; i++) { - MultiValueMap params = createParams(); - if (i == 0) { - params.set(QUERY_NAME, uniqueQueryName); - } - - String queryId = createQuery(authUser, params); - mockServer.reset(); - - queryIds.add(queryId); - } - - // list queries with just the query ID and a bogus name - Future> listFuture = listQueries(altAuthUser, queryIds.get(0), "bogus name"); - - // this should return immediately - ResponseEntity listResponse = listFuture.get(); - - Assert.assertEquals(200, listResponse.getStatusCodeValue()); - - QueryImplListResponse result = listResponse.getBody(); - - Assert.assertEquals(0, result.getNumResults()); - - // list queries with just the query name and a bogus ID - listFuture = listQueries(altAuthUser, UUID.randomUUID().toString(), uniqueQueryName); - - // this should return immediately - listResponse = listFuture.get(); - - Assert.assertEquals(200, listResponse.getStatusCodeValue()); - - result = listResponse.getBody(); - - Assert.assertEquals(0, result.getNumResults()); - - // list queries with the query name and query ID - listFuture = listQueries(altAuthUser, queryIds.get(0), uniqueQueryName); - - // this should return immediately - listResponse = listFuture.get(); - - Assert.assertEquals(200, listResponse.getStatusCodeValue()); - - result = listResponse.getBody(); - - Assert.assertEquals(0, result.getNumResults()); - } - - @DirtiesContext - @Test - public void testAdminListSuccess() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - ProxiedUserDetails adminUser = createAltUserDetails(Arrays.asList("AuthorizedUser", "Administrator"), null); - - String user = ProxiedEntityUtils.getShortName(authUser.getPrimaryUser().getDn().subjectDN()); - - String uniqueQueryName = "Unique Query"; - - // define a bunch of queries as the original user - List queryIds = new ArrayList<>(); - for (int i = 0; i < 5; i++) { - MultiValueMap params = createParams(); - if (i == 0) { - params.set(QUERY_NAME, uniqueQueryName); - } - - String queryId = createQuery(authUser, params); - mockServer.reset(); - - queryIds.add(queryId); - } - - // list queries with just the query ID - Future> listFuture = adminListQueries(adminUser, queryIds.get(0), user, null); - - // this should return immediately - ResponseEntity listResponse = listFuture.get(); - - Assert.assertEquals(200, listResponse.getStatusCodeValue()); - - QueryImplListResponse result = listResponse.getBody(); - - Assert.assertEquals(1, result.getNumResults()); - - List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); - - Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); - - // list queries with just the query name - listFuture = adminListQueries(adminUser, null, user, uniqueQueryName); - - // this should return immediately - listResponse = listFuture.get(); - - Assert.assertEquals(200, listResponse.getStatusCodeValue()); - - result = listResponse.getBody(); - - Assert.assertEquals(1, result.getNumResults()); - - actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); - - Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); - - // list queries with the query name and query ID - listFuture = adminListQueries(adminUser, queryIds.get(0), user, uniqueQueryName); - - // this should return immediately - listResponse = listFuture.get(); - - Assert.assertEquals(200, listResponse.getStatusCodeValue()); - - result = listResponse.getBody(); - - Assert.assertEquals(1, result.getNumResults()); - - actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); - - Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); - } - - @DirtiesContext - @Test - public void testAdminListFailure_notAdminUser() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - ProxiedUserDetails altAuthUser = createAltUserDetails(); - - String user = ProxiedEntityUtils.getShortName(authUser.getPrimaryUser().getDn().subjectDN()); - - String uniqueQueryName = "Unique Query"; - - // define a bunch of queries as the original user - List queryIds = new ArrayList<>(); - for (int i = 0; i < 5; i++) { - MultiValueMap params = createParams(); - if (i == 0) { - params.set(QUERY_NAME, uniqueQueryName); - } - - String queryId = createQuery(authUser, params); - mockServer.reset(); - - queryIds.add(queryId); - } - - UriComponentsBuilder uriBuilder = uriBuilder("/adminList"); - UriComponents uri = uriBuilder.build(); - - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(altAuthUser, null, null, HttpMethod.GET, uri); - - // make the next call asynchronously - Future> listFuture = Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); - - ResponseEntity listResponse = listFuture.get(); - - Assert.assertEquals(403, listResponse.getStatusCodeValue()); - } - - @DirtiesContext - @Test - public void testGetQuerySuccess() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // define a query - String queryId = createQuery(authUser, createParams()); - mockServer.reset(); - - // get the query - Future> listFuture = getQuery(authUser, queryId); - - // this should return immediately - ResponseEntity listResponse = listFuture.get(); - - Assert.assertEquals(200, listResponse.getStatusCodeValue()); - - QueryImplListResponse result = listResponse.getBody(); - - Assert.assertEquals(1, result.getNumResults()); - - Assert.assertEquals(queryId, result.getQuery().get(0).getId().toString()); - } - - @Test - public void testListQueryLogicSuccess() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - Future> future = listQueryLogic(authUser); - - ResponseEntity response = future.get(); - - Assert.assertEquals(200, response.getStatusCodeValue()); - - QueryLogicResponse qlResponse = response.getBody(); - - Assert.assertEquals(2, qlResponse.getQueryLogicList().size()); - - List qlNames = qlResponse.getQueryLogicList().stream().map(QueryLogicDescription::getName).sorted().collect(Collectors.toList()); - - Assert.assertEquals(Arrays.asList("AltEventQuery", "EventQuery"), qlNames); - } - - private void publishEventsToQueue(String queryId, int numEvents, MultiValueMap fieldValues, String visibility) throws Exception { - for (int resultId = 0; resultId < numEvents; resultId++) { - DefaultEvent[] events = new DefaultEvent[1]; - events[0] = new DefaultEvent(); - long currentTime = System.currentTimeMillis(); - List fields = new ArrayList<>(); - for (Map.Entry> entry : fieldValues.entrySet()) { - for (String value : entry.getValue()) { - fields.add(new DefaultField(entry.getKey(), visibility, currentTime, value)); - } - } - events[0].setFields(fields); - queryQueueManager.sendMessage(queryId, new Result(Integer.toString(resultId), events)); - } - } - - private String createQuery(ProxiedUserDetails authUser, MultiValueMap map) { - return newQuery(authUser, map, "create"); - } - - private String defineQuery(ProxiedUserDetails authUser, MultiValueMap map) { - return newQuery(authUser, map, "define"); - } - - private String newQuery(ProxiedUserDetails authUser, MultiValueMap map, String createOrDefine) { - UriComponents uri = createUri("EventQuery/" + createOrDefine); - - // not testing audit with this method - auditIgnoreSetup(); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, GenericResponse.class); - - return (String) resp.getBody().getResult(); - } - - private Future> nextQuery(ProxiedUserDetails authUser, String queryId) { - UriComponents uri = createUri(queryId + "/next"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); - - // make the next call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, BaseResponse.class)); - } - - private Future> adminCloseQuery(ProxiedUserDetails authUser, String queryId) { - return stopQuery(authUser, queryId, "adminClose"); - } - - private Future> closeQuery(ProxiedUserDetails authUser, String queryId) { - return stopQuery(authUser, queryId, "close"); - } - - private Future> adminCancelQuery(ProxiedUserDetails authUser, String queryId) { - return stopQuery(authUser, queryId, "adminCancel"); - } - - private Future> cancelQuery(ProxiedUserDetails authUser, String queryId) { - return stopQuery(authUser, queryId, "cancel"); - } - - private Future> stopQuery(ProxiedUserDetails authUser, String queryId, String closeOrCancel) { - UriComponents uri = createUri(queryId + "/" + closeOrCancel); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); - - // make the next call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); - } - - private Future> adminCloseAllQueries(ProxiedUserDetails authUser) { - UriComponents uri = createUri("/adminCloseAll"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); - - // make the next call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); - } - - private Future> adminCancelAllQueries(ProxiedUserDetails authUser) { - UriComponents uri = createUri("/adminCancelAll"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); - - // make the next call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); - } - - private Future> resetQuery(ProxiedUserDetails authUser, String queryId) { - UriComponents uri = createUri(queryId + "/reset"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.PUT, uri); - - // make the next call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, GenericResponse.class)); - } - - private Future> removeQuery(ProxiedUserDetails authUser, String queryId) { - UriComponents uri = createUri(queryId + "/remove"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); - - // make the next call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); - } - - private Future> adminRemoveQuery(ProxiedUserDetails authUser, String queryId) { - UriComponents uri = createUri(queryId + "/adminRemove"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); - - // make the next call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); - } - - private Future> adminRemoveAllQueries(ProxiedUserDetails authUser) { - UriComponents uri = createUri("/adminRemoveAll"); - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.DELETE, uri); - - // make the next call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); - } - - private Future> updateQuery(ProxiedUserDetails authUser, String queryId, MultiValueMap map) { - UriComponents uri = createUri(queryId + "/update"); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.PUT, uri); - - // make the update call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, GenericResponse.class)); - } - - private Future> duplicateQuery(ProxiedUserDetails authUser, String queryId, MultiValueMap map) { - UriComponents uri = createUri(queryId + "/duplicate"); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - - // make the update call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, GenericResponse.class)); - } - - private Future> listQueries(ProxiedUserDetails authUser, String queryId, String queryName) { - UriComponentsBuilder uriBuilder = uriBuilder("/list"); - if (queryId != null) { - uriBuilder.queryParam("queryId", queryId); - } - if (queryName != null) { - uriBuilder.queryParam("queryName", queryName); - } - UriComponents uri = uriBuilder.build(); - - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); - - // make the next call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, QueryImplListResponse.class)); - } - - private Future> getQuery(ProxiedUserDetails authUser, String queryId) { - UriComponents uri = createUri(queryId); - - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); - - // make the next call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, QueryImplListResponse.class)); - } - - private Future> listQueryLogic(ProxiedUserDetails authUser) { - UriComponents uri = createUri("/listQueryLogic"); - - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); - - // make the next call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, QueryLogicResponse.class)); - } - - private Future> adminListQueries(ProxiedUserDetails authUser, String queryId, String user, String queryName) { - UriComponentsBuilder uriBuilder = uriBuilder("/adminList"); - if (queryId != null) { - uriBuilder.queryParam("queryId", queryId); - } - if (queryName != null) { - uriBuilder.queryParam("queryName", queryName); - } - if (user != null) { - uriBuilder.queryParam("user", user); - } - UriComponents uri = uriBuilder.build(); - - RequestEntity requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, null, HttpMethod.GET, uri); - - // make the next call asynchronously - return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, QueryImplListResponse.class)); - } - - private ProxiedUserDetails createUserDetails() { - return createUserDetails(null, null); - } - - private ProxiedUserDetails createUserDetails(Collection roles, Collection auths) { - Collection userRoles = roles != null ? roles : Collections.singleton("AuthorizedUser"); - Collection userAuths = auths != null ? auths : Collections.singleton("ALL"); - DatawaveUser datawaveUser = new DatawaveUser(DN, USER, userAuths, userRoles, null, System.currentTimeMillis()); - return new ProxiedUserDetails(Collections.singleton(datawaveUser), datawaveUser.getCreationTime()); - } - - private ProxiedUserDetails createAltUserDetails() { - return createAltUserDetails(null, null); - } - - private ProxiedUserDetails createAltUserDetails(Collection roles, Collection auths) { - Collection userRoles = roles != null ? roles : Collections.singleton("AuthorizedUser"); - Collection userAuths = auths != null ? auths : Collections.singleton("ALL"); - DatawaveUser datawaveUser = new DatawaveUser(altDN, USER, userAuths, userRoles, null, System.currentTimeMillis()); - return new ProxiedUserDetails(Collections.singleton(datawaveUser), datawaveUser.getCreationTime()); - } - - private UriComponentsBuilder uriBuilder(String path) { - return UriComponentsBuilder.newInstance().scheme("https").host("localhost").port(webServicePort).path("/query/v1/" + path); - } - - private UriComponents createUri(String path) { - return uriBuilder(path).build(); - } - - private MultiValueMap createParams() { - MultiValueMap map = new LinkedMultiValueMap<>(); - map.set(DefaultQueryParameters.QUERY_STRING, TEST_QUERY_STRING); - map.set(DefaultQueryParameters.QUERY_NAME, TEST_QUERY_NAME); - map.set(DefaultQueryParameters.QUERY_AUTHORIZATIONS, TEST_QUERY_AUTHORIZATIONS); - map.set(DefaultQueryParameters.QUERY_BEGIN, TEST_QUERY_BEGIN); - map.set(DefaultQueryParameters.QUERY_END, TEST_QUERY_END); - map.set(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, TEST_VISIBILITY_MARKING); - map.set(QUERY_MAX_CONCURRENT_TASKS, Integer.toString(1)); - map.set(QUERY_MAX_RESULTS_OVERRIDE, Long.toString(369)); - map.set(QUERY_PAGESIZE, Long.toString(123)); - return map; - } - - private void assertDefaultEvent(List fields, List values, DefaultEvent event) { - Assert.assertEquals(fields, event.getFields().stream().map(DefaultField::getName).collect(Collectors.toList())); - Assert.assertEquals(values, event.getFields().stream().map(DefaultField::getValueString).collect(Collectors.toList())); - } - - private void assertQueryResponse(String queryId, String logicName, long pageNumber, boolean partialResults, long operationTimeInMS, int numFields, - List fieldNames, int numEvents, DefaultEventQueryResponse queryResponse) { - Assert.assertEquals(queryId, queryResponse.getQueryId()); - Assert.assertEquals(logicName, queryResponse.getLogicName()); - Assert.assertEquals(pageNumber, queryResponse.getPageNumber()); - Assert.assertEquals(partialResults, queryResponse.isPartialResults()); - Assert.assertEquals(operationTimeInMS, queryResponse.getOperationTimeMS()); - Assert.assertEquals(numFields, queryResponse.getFields().size()); - Assert.assertEquals(fieldNames, queryResponse.getFields()); - Assert.assertEquals(numEvents, queryResponse.getEvents().size()); - } - - private void assertQueryRequestEvent(String destination, QueryRequest.Method method, String queryId, RemoteQueryRequestEvent queryRequestEvent) { - Assert.assertEquals(destination, queryRequestEvent.getDestinationService()); - Assert.assertEquals(queryId, queryRequestEvent.getRequest().getQueryId()); - Assert.assertEquals(method, queryRequestEvent.getRequest().getMethod()); - } - - private void assertQueryStatus(QueryStatus.QUERY_STATE queryState, long numResultsReturned, long numResultsGenerated, long activeNextCalls, - long lastPageNumber, long lastCallTimeMillis, QueryStatus queryStatus) { - Assert.assertEquals(queryState, queryStatus.getQueryState()); - Assert.assertEquals(numResultsReturned, queryStatus.getNumResultsReturned()); - Assert.assertEquals(numResultsGenerated, queryStatus.getNumResultsGenerated()); - Assert.assertEquals(activeNextCalls, queryStatus.getActiveNextCalls()); - Assert.assertEquals(lastPageNumber, queryStatus.getLastPageNumber()); - Assert.assertTrue(queryStatus.getLastUsedMillis() > lastCallTimeMillis); - Assert.assertTrue(queryStatus.getLastUpdatedMillis() > lastCallTimeMillis); - } - - private void assertQuery(String queryString, String queryName, String authorizations, String begin, String end, String visibility, Query query) - throws ParseException { - SimpleDateFormat sdf = new SimpleDateFormat(DefaultQueryParameters.formatPattern); - Assert.assertEquals(queryString, query.getQuery()); - Assert.assertEquals(queryName, query.getQueryName()); - Assert.assertEquals(authorizations, query.getQueryAuthorizations()); - Assert.assertEquals(sdf.parse(begin), query.getBeginDate()); - Assert.assertEquals(sdf.parse(end), query.getEndDate()); - Assert.assertEquals(visibility, query.getColumnVisibility()); - } - - private void assertTasksCreated(String queryId) throws IOException { - // verify that the query task states were created - TaskStates taskStates = queryStorageCache.getTaskStates(queryId); - Assert.assertNotNull(taskStates); - - // verify that a query task was created - List taskKeys = queryStorageCache.getTasks(queryId); - Assert.assertFalse(taskKeys.isEmpty()); - } - - private void assertTasksNotCreated(String queryId) throws IOException { - // verify that the query task states were not created - TaskStates taskStates = queryStorageCache.getTaskStates(queryId); - Assert.assertNull(taskStates); - - // verify that a query task was not created - List taskKeys = queryStorageCache.getTasks(queryId); - Assert.assertTrue(taskKeys.isEmpty()); - } - - public RequestMatcher auditIdGrabber() { - return request -> { - List params = URLEncodedUtils.parse(request.getBody().toString(), Charset.defaultCharset()); - params.stream().filter(p -> p.getName().equals(AUDIT_ID)).forEach(p -> auditIds.add(p.getValue())); - }; - } - - private void auditIgnoreSetup() { - mockServer.expect(anything()).andRespond(withSuccess()); - } - - private void auditSentSetup() { - mockServer.expect(requestTo(EXPECTED_AUDIT_URI)).andExpect(auditIdGrabber()).andRespond(withSuccess()); - } - - private void auditNotSentSetup() { - mockServer.expect(never(), requestTo(EXPECTED_AUDIT_URI)).andExpect(auditIdGrabber()).andRespond(withSuccess()); - } - - private void assertAuditSent(String queryId) { - mockServer.verify(); - Assert.assertEquals(1, auditIds.size()); - Assert.assertEquals(queryId, auditIds.get(0)); - } - - private void assertAuditNotSent() { - mockServer.verify(); - Assert.assertEquals(0, auditIds.size()); - } - - private void assertQueryException(String message, String cause, String code, QueryExceptionType queryException) { - Assert.assertEquals(message, queryException.getMessage()); - Assert.assertEquals(cause, queryException.getCause()); - Assert.assertEquals(code, queryException.getCode()); - } - - private BaseResponse assertBaseResponse(boolean hasResults, HttpStatus.Series series, ResponseEntity response) { - Assert.assertEquals(series, response.getStatusCode().series()); - Assert.assertNotNull(response); - BaseResponse baseResponse = response.getBody(); - Assert.assertNotNull(baseResponse); - Assert.assertEquals(hasResults, baseResponse.getHasResults()); - return baseResponse; - } - - @SuppressWarnings("unchecked") - private GenericResponse assertGenericResponse(boolean hasResults, HttpStatus.Series series, ResponseEntity response) { - Assert.assertEquals(series, response.getStatusCode().series()); - Assert.assertNotNull(response); - GenericResponse genericResponse = (GenericResponse) response.getBody(); - Assert.assertNotNull(genericResponse); - Assert.assertEquals(hasResults, genericResponse.getHasResults()); - return genericResponse; - } - - private static class NoOpResponseErrorHandler extends DefaultResponseErrorHandler { - @Override - public void handleError(ClientHttpResponse response) throws IOException { - // do nothing - } - } - - @Configuration - @Profile("QueryServiceTest") - @ComponentScan(basePackages = "datawave.microservice") - public static class QueryServiceTestConfiguration { - @Bean - public LinkedList queryRequestEvents() { - return new LinkedList<>(); - } - - @Bean - @Primary - public ApplicationEventPublisher eventPublisher(ApplicationEventPublisher eventPublisher) { - return new ApplicationEventPublisher() { - @Override - public void publishEvent(ApplicationEvent event) { - saveEvent(event); - } - - @Override - public void publishEvent(Object event) { - saveEvent(event); - } - - private void saveEvent(Object event) { - if (event instanceof RemoteQueryRequestEvent) { - queryRequestEvents().push(((RemoteQueryRequestEvent) event)); - } - } - }; - } - } -} diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java new file mode 100644 index 00000000..919317d3 --- /dev/null +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java @@ -0,0 +1,455 @@ +package datawave.microservice.query; + +import com.google.common.collect.Iterables; +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.result.GenericResponse; +import datawave.webservice.result.VoidResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; + +import java.util.Arrays; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; +import static datawave.webservice.common.audit.AuditParameters.QUERY_AUTHORIZATIONS; +import static datawave.webservice.common.audit.AuditParameters.QUERY_STRING; +import static datawave.webservice.query.QueryImpl.BEGIN_DATE; +import static datawave.webservice.query.QueryImpl.END_DATE; +import static datawave.webservice.query.QueryImpl.PAGESIZE; +import static datawave.webservice.query.QueryImpl.QUERY; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +public class QueryServiceUpdateTest extends AbstractQueryServiceTest { + @Before + public void setup() { + super.setup(); + } + + @After + public void teardown() throws Exception { + super.teardown(); + } + + @Test + public void testUpdateSuccess_updateOnDefined() throws Exception { + ProxiedUserDetails authUser = createUserDetails(null, Arrays.asList("ALL", "NONE")); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + String newQuery = "SOME_OTHER_FIELD:SOME_OTHER_VALUE"; + String newAuths = "ALL,NONE"; + String newBegin = "20100101 000000.000"; + String newEnd = "20600101 000000.000"; + String newLogic = "AltEventQuery"; + int newPageSize = 100; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY, newQuery); + updateParams.set(QUERY_AUTHORIZATIONS, newAuths); + updateParams.set(BEGIN_DATE, newBegin); + updateParams.set(END_DATE, newEnd); + updateParams.set(QUERY_LOGIC_NAME, newLogic); + updateParams.set(PAGESIZE, Integer.toString(newPageSize)); + + // update the query + Future> updateFuture = updateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was updated + Assert.assertEquals(newQuery, queryStatus.getQuery().getQuery()); + Assert.assertEquals(newAuths, queryStatus.getQuery().getQueryAuthorizations()); + Assert.assertEquals(newBegin, DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate())); + Assert.assertEquals(newEnd, DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate())); + Assert.assertEquals(newLogic, queryStatus.getQuery().getQueryLogicName()); + Assert.assertEquals(newPageSize, queryStatus.getQuery().getPagesize()); + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testUpdateSuccess_updateOnCreated() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + int newPageSize = 100; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(PAGESIZE, Integer.toString(newPageSize)); + + // update the query + Future> updateFuture = updateQuery(authUser, queryId, updateParams); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was updated + Assert.assertEquals(newPageSize, queryStatus.getQuery().getPagesize()); + + // verify that events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testUpdateFailure_unsafeParamUpdateQuery() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + String newQuery = "SOME_OTHER_FIELD:SOME_OTHER_VALUE"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY, newQuery); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was not updated + Assert.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); + + // @formatter:off + assertQueryException( + "Cannot update the following parameters for a running query: query", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testUpdateFailure_unsafeParamUpdateDate() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + String newBegin = "20100101 000000.000"; + String newEnd = "20600101 000000.000"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(BEGIN_DATE, newBegin); + updateParams.set(END_DATE, newEnd); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was not updated + Assert.assertEquals(TEST_QUERY_BEGIN, DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate())); + Assert.assertEquals(TEST_QUERY_END, DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate())); + + // @formatter:off + assertQueryException( + "Cannot update the following parameters for a running query: begin, end", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testUpdateFailure_unsafeParamUpdateLogic() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + String newLogic = "AltEventQuery"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY_LOGIC_NAME, newLogic); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was not updated + Assert.assertEquals("EventQuery", queryStatus.getQuery().getQueryLogicName()); + + // @formatter:off + assertQueryException( + "Cannot update the following parameters for a running query: logicName", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testUpdateFailure_unsafeParamUpdateAuths() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + String newAuths = "ALL,NONE"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY_AUTHORIZATIONS, newAuths); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was not updated + Assert.assertEquals(TEST_QUERY_AUTHORIZATIONS, queryStatus.getQuery().getQueryAuthorizations()); + + // @formatter:off + assertQueryException( + "Cannot update the following parameters for a running query: auths", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testUpdateFailure_nullParams() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "No parameters specified for update.", + "Exception with no cause caught", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that events were published + Assert.assertEquals(1, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testUpdateFailure_queryNotFound() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + String queryId = UUID.randomUUID().toString(); + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY_STRING, TEST_QUERY_STRING); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assert.assertEquals(404, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "No query object matches this id. " + queryId, + "Exception with no cause caught", + "404-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } + + @Test + public void testUpdateFailure_ownershipFailure() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + ProxiedUserDetails altAuthUser = createAltUserDetails(); + + // define a valid query + String queryId = defineQuery(authUser, createParams()); + + String newQuery = "SOME_OTHER_FIELD:SOME_OTHER_VALUE"; + + MultiValueMap updateParams = new LinkedMultiValueMap<>(); + updateParams.set(QUERY, newQuery); + + // update the query + UriComponents uri = createUri(queryId + "/update"); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(altAuthUser, updateParams, null, HttpMethod.PUT, uri); + + // make the update call asynchronously + Future> updateFuture = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, VoidResponse.class)); + + // the response should come back right away + ResponseEntity response = updateFuture.get(); + + Assert.assertEquals(401, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Current user does not match user that defined query. altuserdn != userdn", + "Exception with no cause caught", + "401-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + + // make sure the query was not updated + Assert.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); + + // verify that no events were published + Assert.assertEquals(0, queryRequestEvents.size()); + } +} diff --git a/query-microservices/query-service/service/src/test/resources/log4j2-test.xml b/query-microservices/query-service/service/src/test/resources/log4j2-test.xml index 43cd58ff..0600e42c 100644 --- a/query-microservices/query-service/service/src/test/resources/log4j2-test.xml +++ b/query-microservices/query-service/service/src/test/resources/log4j2-test.xml @@ -10,8 +10,8 @@ - - + + \ No newline at end of file From 99128ce2bbad4cbe06623b171bfb252a66b759dd Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 6 Aug 2021 11:40:53 -0400 Subject: [PATCH 083/218] Disabled the query metrics client for tests which don't run as web applications. --- .../microservice/query/QueryController.java | 3 --- .../microservice/query/QueryManagementService.java | 2 -- .../microservice/query/monitor/MonitorTask.java | 2 +- .../microservice/query/runner/NextCall.java | 1 - .../query/web/BaseQueryResponseAdvice.java | 1 - .../microservice/query/web/QueryMetrics.java | 13 ------------- .../query/web/QuerySessionIdAdvice.java | 2 -- 7 files changed, 1 insertion(+), 23 deletions(-) delete mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 0f219726..1ab65e9d 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -163,7 +163,6 @@ public QueryImplListResponse get(@PathVariable String queryId, @AuthenticationPr return queryManagementService.list(queryId, null, currentUser); } - // TODO: Update this to cancel all queries on a per-pool basis? @Timed(name = "dw.query.adminCancelAll", absolute = true) @RolesAllowed({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "adminCancelAll", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", @@ -172,7 +171,6 @@ public VoidResponse adminCancelAll(@AuthenticationPrincipal ProxiedUserDetails c return queryManagementService.adminCancelAll(currentUser); } - // TODO: Update this to close all queries on a per-pool basis? @Timed(name = "dw.query.adminCloseAll", absolute = true) @RolesAllowed({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "adminCloseAll", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", @@ -181,7 +179,6 @@ public VoidResponse adminCloseAll(@AuthenticationPrincipal ProxiedUserDetails cu return queryManagementService.adminCloseAll(currentUser); } - // TODO: Update this to remove all queries on a per-pool basis? @Timed(name = "dw.query.adminRemoveAll", absolute = true) @RolesAllowed({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "adminRemoveAll", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 39cd96cb..b290e132 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -82,8 +82,6 @@ import static datawave.webservice.query.QueryImpl.QUERY_ID; import static datawave.webservice.query.QueryImpl.USER_DN; -// TODO: Make sure close is called automatically after a query finishes. I think this is more of a query metrics thing than anything. - @Service public class QueryManagementService implements QueryRequestHandler { private final Logger log = LoggerFactory.getLogger(this.getClass()); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java index 983060f2..411e6f1e 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java @@ -58,7 +58,7 @@ public Void call() throws Exception { return null; } - // TODO: Check for the following conditions + // Check for the following conditions // 1) Is query progress idle? If so, poke the query // 2) Is the user idle? If so, close the query // 3) Are there any other conditions that we should check for? diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index e8a286f9..cb1256b7 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -234,7 +234,6 @@ private boolean isFinished(String queryId) { } } - // TODO: We need to stop relying on query metrics for these values // 7) have we reached the "max work" limit? (i.e. next count + seek count) if (!finished && logicMaxWork > 0 && (queryStatus.getNextCount() + queryStatus.getSeekCount()) >= logicMaxWork) { log.info("Query [{}]: logic max work has been reached, aborting next call", queryId); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java index 5c5ee97d..69397e74 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java @@ -11,7 +11,6 @@ import org.springframework.web.bind.annotation.ControllerAdvice; import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyAdvice; -// TODO: JWO: Should this be in the API or in a starter? /** * A {@link ControllerAdvice} that implements {@link ResponseBodyAdvice} in order to allow access to {@link BaseQueryResponse} objects before they are written * out to the response body. This is primarily used to write the page number, is last page, and partial results headers for the response. diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java deleted file mode 100644 index fdac9df1..00000000 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/QueryMetrics.java +++ /dev/null @@ -1,13 +0,0 @@ -package datawave.microservice.query.web; - -import datawave.webservice.query.metric.BaseQueryMetric; -import org.springframework.stereotype.Component; - -// TODO: JWO: Remove this placeholder and replace it with the real thing once it's ready. -@Component -public class QueryMetrics { - - public void updateMetric(BaseQueryMetric baseQueryMetric) { - System.out.println(baseQueryMetric); - } -} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java index f473d0f9..31dc7fb5 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java @@ -19,8 +19,6 @@ import java.util.Arrays; import java.util.UUID; -// TODO: JWO: Update to enable based on properties -// TODO: JWO: Should this be in the API or in a starter? @ControllerAdvice public class QuerySessionIdAdvice implements ResponseBodyAdvice { private final Logger log = Logger.getLogger(QuerySessionIdAdvice.class); From 8f10a8e4763f2c6bb82af541d99c0d55486e9684 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Thu, 12 Aug 2021 14:57:11 +0000 Subject: [PATCH 084/218] Updated result object to hold only one underlying result. --- .../microservice/query/runner/NextCall.java | 30 +++++++------------ .../query/AbstractQueryServiceTest.java | 7 ++--- 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index cb1256b7..8f11fa7e 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -110,27 +110,19 @@ public ResultsPage call() { while (!isFinished(queryId)) { Message message = resultListener.receive(nextCallProperties.getResultPollIntervalMillis()); if (message != null) { - Object[] payload = message.getPayload().getPayload(); - for (int resultIdx = 0; resultIdx < payload.length; resultIdx++) { - Object result = payload[resultIdx]; - - // have to check to make sure we haven't reached the page size - if (!isFinished(queryId)) { - if (result != null) { - results.add(result); - - if (logicBytesPerPage > 0) { - pageSizeBytes += ObjectSizeOf.Sizer.getObjectSize(result); - } - } else { - log.debug("Null result encountered, no more results"); - break; + Object result = message.getPayload().getPayload(); + + // have to check to make sure we haven't reached the page size + if (!isFinished(queryId)) { + if (result != null) { + results.add(result); + + if (logicBytesPerPage > 0) { + pageSizeBytes += ObjectSizeOf.Sizer.getObjectSize(result); } } else { - // TODO: This will no longer be possible once the result queues are updated to pass single results instead of arrays - if ((resultIdx + 1) != payload.length) { - log.error("Next call is dropping " + (payload.length - (resultIdx + 1)) + " results"); - } + log.debug("Null result encountered, no more results"); + break; } } } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java index bf05fae4..3cf6c882 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java @@ -140,8 +140,7 @@ public void teardown() throws Exception { protected void publishEventsToQueue(String queryId, int numEvents, MultiValueMap fieldValues, String visibility) throws Exception { for (int resultId = 0; resultId < numEvents; resultId++) { - DefaultEvent[] events = new DefaultEvent[1]; - events[0] = new DefaultEvent(); + DefaultEvent event = new DefaultEvent(); long currentTime = System.currentTimeMillis(); List fields = new ArrayList<>(); for (Map.Entry> entry : fieldValues.entrySet()) { @@ -149,8 +148,8 @@ protected void publishEventsToQueue(String queryId, int numEvents, MultiValueMap fields.add(new DefaultField(entry.getKey(), visibility, currentTime, value)); } } - events[0].setFields(fields); - queryQueueManager.sendMessage(queryId, new Result(Integer.toString(resultId), events)); + event.setFields(fields); + queryQueueManager.sendMessage(queryId, new Result(Integer.toString(resultId), event)); } } From 9b2cf864fd1aea070fe6ef7c13afda274b670adf Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 19 Aug 2021 16:04:19 -0400 Subject: [PATCH 085/218] Updated the query starter to include a default QueryLogicFactory.xml, and updated the query service to include that file as a resource. --- .../query-service/service/pom.xml | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/query-microservices/query-service/service/pom.xml b/query-microservices/query-service/service/pom.xml index a317a852..517140fe 100644 --- a/query-microservices/query-service/service/pom.xml +++ b/query-microservices/query-service/service/pom.xml @@ -115,6 +115,29 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + + unpack + package + + unpack + + + + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-query + **/QueryLogicFactory.xml + ${project.build.outputDirectory} + + + + + + org.springframework.boot spring-boot-maven-plugin From 3d1980e43b69f4a75aa02557b438ceb349984ae8 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 19 Aug 2021 16:06:06 -0400 Subject: [PATCH 086/218] Updated request filters to to use request scoped beans. --- .../web/filter/BaseMethodStatsFilter.java | 69 +++++++++-------- .../QueryMetricsEnrichmentFilterAdvice.java | 75 +++++++++++-------- 2 files changed, 80 insertions(+), 64 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java index 25668a65..99e7eb0d 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java @@ -1,9 +1,14 @@ package datawave.microservice.query.web.filter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; import org.springframework.http.MediaType; import org.springframework.lang.NonNull; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; +import org.springframework.web.context.annotation.RequestScope; import org.springframework.web.filter.OncePerRequestFilter; import javax.servlet.FilterChain; @@ -28,6 +33,9 @@ public abstract class BaseMethodStatsFilter extends OncePerRequestFilter { private static final String START_NS_ATTRIBUTE = "STATS_START_NS"; private static final String STOP_NS_ATTRIBUTE = "STATS_STOP_NS"; + @Autowired + private BaseMethodStatsContext baseMethodStatsContext; + protected static class RequestMethodStats { private String uri; private String method; @@ -110,13 +118,13 @@ public void doFilterInternal(@NonNull HttpServletRequest request, @NonNull HttpS } public void preProcess(HttpServletRequest request, HttpServletResponse response) { - if (BaseMethodStatsContext.getRequestStats() == null) { - BaseMethodStatsContext.setRequestStats(createRequestMethodStats(request, response)); + if (baseMethodStatsContext.getRequestStats() == null) { + baseMethodStatsContext.setRequestStats(createRequestMethodStats(request, response)); long start = System.nanoTime(); request.setAttribute(START_NS_ATTRIBUTE, start); } - preProcess(BaseMethodStatsContext.getRequestStats()); + preProcess(baseMethodStatsContext.getRequestStats()); } public void preProcess(RequestMethodStats requestStats) { @@ -124,18 +132,16 @@ public void preProcess(RequestMethodStats requestStats) { } public void postProcess(HttpServletRequest request, HttpServletResponse response) { - if (BaseMethodStatsContext.getResponseStats() == null) { + if (baseMethodStatsContext.getResponseStats() == null) { long stop = System.nanoTime(); request.setAttribute(STOP_NS_ATTRIBUTE, stop); - BaseMethodStatsContext.setResponseStats(createResponseMethodStats(request, response)); + baseMethodStatsContext.setResponseStats(createResponseMethodStats(request, response)); } - postProcess(BaseMethodStatsContext.getResponseStats()); + postProcess(baseMethodStatsContext.getResponseStats()); } - public void postProcess(ResponseMethodStats responseStats) { - // do nothing - } + abstract public void postProcess(ResponseMethodStats responseStats); protected RequestMethodStats createRequestMethodStats(HttpServletRequest request, HttpServletResponse response) { RequestMethodStats requestStats = new RequestMethodStats(); @@ -190,8 +196,8 @@ private ResponseMethodStats createResponseMethodStats(HttpServletRequest request } responseStats.serializationTime = TimeUnit.NANOSECONDS.toMillis(stop - start); - responseStats.loginTime = BaseMethodStatsContext.getRequestStats().getLoginTime(); - responseStats.callTime = TimeUnit.NANOSECONDS.toMillis(stop - BaseMethodStatsContext.getRequestStats().getCallStartTime()); + responseStats.loginTime = baseMethodStatsContext.getRequestStats().getLoginTime(); + responseStats.callTime = TimeUnit.NANOSECONDS.toMillis(stop - baseMethodStatsContext.getRequestStats().getCallStartTime()); if (response instanceof CountingHttpServletResponseWrapper) { responseStats.bytesWritten = ((CountingHttpServletResponseWrapper) response).getByteCount(); @@ -204,11 +210,6 @@ private ResponseMethodStats createResponseMethodStats(HttpServletRequest request return responseStats; } - @Override - public void destroy() { - BaseMethodStatsContext.remove(); - } - private static class CountingHttpServletResponseWrapper extends HttpServletResponseWrapper { private final ServletResponse response; private CountingServletOutputStream cos; @@ -274,30 +275,34 @@ public long getByteCount() { } } - public static class BaseMethodStatsContext { - - private static final ThreadLocal requestStats = new ThreadLocal<>(); - private static final ThreadLocal responseStats = new ThreadLocal<>(); + private static class BaseMethodStatsContext { + private RequestMethodStats requestStats; + private ResponseMethodStats responseStats; - public static RequestMethodStats getRequestStats() { - return requestStats.get(); + public RequestMethodStats getRequestStats() { + return requestStats; } - public static void setRequestStats(RequestMethodStats requestStats) { - BaseMethodStatsContext.requestStats.set(requestStats); + public void setRequestStats(RequestMethodStats requestStats) { + this.requestStats = requestStats; } - public static ResponseMethodStats getResponseStats() { - return responseStats.get(); + public ResponseMethodStats getResponseStats() { + return responseStats; } - public static void setResponseStats(ResponseMethodStats responseStats) { - BaseMethodStatsContext.responseStats.set(responseStats); + public void setResponseStats(ResponseMethodStats responseStats) { + this.responseStats = responseStats; } - - private static void remove() { - requestStats.remove(); - responseStats.remove(); + } + + @Configuration + public static class BaseMethodStatsFilterConfig { + @Bean + @ConditionalOnMissingBean + @RequestScope + public BaseMethodStatsContext baseMethodStatsContext() { + return new BaseMethodStatsContext(); } } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java index 6ed1a8e8..06ec9ba6 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java @@ -6,9 +6,13 @@ import datawave.microservice.query.web.annotation.EnrichQueryMetrics; import datawave.microservice.querymetric.BaseQueryMetric; import datawave.microservice.querymetric.QueryMetricClient; +import datawave.microservice.querymetric.QueryMetricType; import datawave.webservice.result.BaseQueryResponse; import datawave.webservice.result.GenericResponse; import org.apache.log4j.Logger; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; import org.springframework.core.MethodParameter; import org.springframework.http.MediaType; import org.springframework.http.converter.HttpMessageConverter; @@ -16,9 +20,11 @@ import org.springframework.http.server.ServerHttpResponse; import org.springframework.lang.NonNull; import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.context.annotation.RequestScope; import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyAdvice; import java.util.Arrays; +import java.util.Date; import java.util.List; import java.util.Objects; @@ -36,12 +42,16 @@ public class QueryMetricsEnrichmentFilterAdvice extends BaseMethodStatsFilter im // Note: BaseQueryMetric needs to be request scoped private final BaseQueryMetric baseQueryMetric; + // Note: QueryMetricsEnrichmentContext needs to be request scoped + private final QueryMetricsEnrichmentContext queryMetricsEnrichmentContext; + public QueryMetricsEnrichmentFilterAdvice(QueryLogicFactory queryLogicFactory, QueryStorageCache queryStorageCache, QueryMetricClient queryMetricClient, - BaseQueryMetric baseQueryMetric) { + BaseQueryMetric baseQueryMetric, QueryMetricsEnrichmentContext queryMetricsEnrichmentContext) { this.queryLogicFactory = queryLogicFactory; this.queryStorageCache = queryStorageCache; this.queryMetricClient = queryMetricClient; this.baseQueryMetric = baseQueryMetric; + this.queryMetricsEnrichmentContext = queryMetricsEnrichmentContext; } @Override @@ -54,10 +64,10 @@ public boolean supports(MethodParameter returnType, @NonNull Class returnClass = Objects.requireNonNull(returnType.getMethod()).getReturnType(); if (GenericResponse.class.isAssignableFrom(returnClass)) { supports = true; - QueryMetricsEnrichmentContext.setMethodType(annotation.methodType()); + queryMetricsEnrichmentContext.setMethodType(annotation.methodType()); } else if (BaseQueryResponse.class.isAssignableFrom(returnClass)) { supports = true; - QueryMetricsEnrichmentContext.setMethodType(annotation.methodType()); + queryMetricsEnrichmentContext.setMethodType(annotation.methodType()); } else { log.error("Unexpected response class for metrics annotated query method " + returnType.getMethod().getName() + ". Response class was " + returnClass.toString()); @@ -77,10 +87,10 @@ public Object beforeBodyWrite(Object body, @NonNull MethodParameter returnType, if (body instanceof GenericResponse) { @SuppressWarnings({"unchecked", "rawtypes"}) GenericResponse genericResponse = (GenericResponse) body; - QueryMetricsEnrichmentContext.setQueryId(genericResponse.getResult()); + queryMetricsEnrichmentContext.setQueryId(genericResponse.getResult()); } else if (body instanceof BaseQueryResponse) { BaseQueryResponse baseResponse = (BaseQueryResponse) body; - QueryMetricsEnrichmentContext.setQueryId(baseResponse.getQueryId()); + queryMetricsEnrichmentContext.setQueryId(baseResponse.getQueryId()); } return body; @@ -88,8 +98,8 @@ public Object beforeBodyWrite(Object body, @NonNull MethodParameter returnType, @Override public void postProcess(ResponseMethodStats responseStats) { - String queryId = QueryMetricsEnrichmentContext.getQueryId(); - EnrichQueryMetrics.MethodType methodType = QueryMetricsEnrichmentContext.getMethodType(); + String queryId = queryMetricsEnrichmentContext.getQueryId(); + EnrichQueryMetrics.MethodType methodType = queryMetricsEnrichmentContext.getMethodType(); if (queryId != null && methodType != null) { // determine which query logic is being used @@ -113,7 +123,7 @@ public void postProcess(ResponseMethodStats responseStats) { if (isMetricsEnabled) { try { - switch (QueryMetricsEnrichmentContext.getMethodType()) { + switch (queryMetricsEnrichmentContext.getMethodType()) { case CREATE: baseQueryMetric.setCreateCallTime(responseStats.getCallTime()); baseQueryMetric.setLoginTime(responseStats.getLoginTime()); @@ -142,49 +152,50 @@ public void postProcess(ResponseMethodStats responseStats) { break; } + baseQueryMetric.setLastUpdated(new Date()); // @formatter:off queryMetricClient.submit( new QueryMetricClient.Request.Builder() - .withMetric(baseQueryMetric) + .withMetric(baseQueryMetric.duplicate()) + .withMetricType(QueryMetricType.DISTRIBUTED) .build()); // @formatter:on } catch (Exception e) { - log.error("Unable to record metrics for query '" + QueryMetricsEnrichmentContext.getQueryId() + "' and method '" - + QueryMetricsEnrichmentContext.getMethodType() + "': " + e.getLocalizedMessage(), e); + log.error("Unable to record metrics for query '" + queryMetricsEnrichmentContext.getQueryId() + "' and method '" + + queryMetricsEnrichmentContext.getMethodType() + "': " + e.getLocalizedMessage(), e); } } } } - @Override - public void destroy() { - super.destroy(); - QueryMetricsEnrichmentContext.remove(); - } - - public static class QueryMetricsEnrichmentContext { + private static class QueryMetricsEnrichmentContext { + private String queryId; + private EnrichQueryMetrics.MethodType methodType; - private static final ThreadLocal queryId = new ThreadLocal<>(); - private static final ThreadLocal methodType = new ThreadLocal<>(); - - public static String getQueryId() { - return queryId.get(); + public String getQueryId() { + return queryId; } - public static void setQueryId(String queryId) { - QueryMetricsEnrichmentContext.queryId.set(queryId); + public void setQueryId(String queryId) { + this.queryId = queryId; } - public static EnrichQueryMetrics.MethodType getMethodType() { - return methodType.get(); + public EnrichQueryMetrics.MethodType getMethodType() { + return methodType; } - public static void setMethodType(EnrichQueryMetrics.MethodType methodType) { - QueryMetricsEnrichmentContext.methodType.set(methodType); + public void setMethodType(EnrichQueryMetrics.MethodType methodType) { + this.methodType = methodType; } - - private static void remove() { - queryId.remove(); + } + + @Configuration + public static class QueryMetricsEnrichmentFilterAdviceConfig { + @Bean + @ConditionalOnMissingBean + @RequestScope + public QueryMetricsEnrichmentContext queryMetricsEnrichmentContext() { + return new QueryMetricsEnrichmentContext(); } } } From 3b7c44e86719d8c227169b35ea3b9f5c06a8eb05 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 19 Aug 2021 16:07:00 -0400 Subject: [PATCH 087/218] Updated the query service to correctly populate query metrics. --- .../query/QueryManagementService.java | 11 ++++++++-- .../microservice/query/runner/NextCall.java | 22 +++++++++---------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index b290e132..da50e272 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -21,6 +21,7 @@ import datawave.microservice.query.util.QueryUtil; import datawave.microservice.querymetric.BaseQueryMetric; import datawave.microservice.querymetric.QueryMetricClient; +import datawave.microservice.querymetric.QueryMetricType; import datawave.security.util.ProxiedEntityUtils; import datawave.webservice.common.audit.AuditParameters; import datawave.webservice.common.audit.Auditor; @@ -848,12 +849,15 @@ public void cancel(String queryId, boolean publishEvent) throws InterruptedExcep publishExecutorEvent(cancelRequest, queryStatus.getQueryKey().getQueryPool()); // update query metrics + baseQueryMetric.setQueryId(queryId); baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.CANCELLED); + baseQueryMetric.setLastUpdated(new Date()); try { // @formatter:off queryMetricClient.submit( new QueryMetricClient.Request.Builder() - .withMetric(baseQueryMetric) + .withMetric(baseQueryMetric.duplicate()) + .withMetricType(QueryMetricType.DISTRIBUTED) .build()); // @formatter:on } catch (Exception e) { @@ -1039,12 +1043,15 @@ public void close(String queryId) throws InterruptedException, QueryException { publishExecutorEvent(QueryRequest.close(queryId), queryStatus.getQueryKey().getQueryPool()); // update query metrics + baseQueryMetric.setQueryId(queryId); baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.CLOSED); + baseQueryMetric.setLastUpdated(new Date()); try { // @formatter:off queryMetricClient.submit( new QueryMetricClient.Request.Builder() - .withMetric(baseQueryMetric) + .withMetric(baseQueryMetric.duplicate()) + .withMetricType(QueryMetricType.DISTRIBUTED) .build()); // @formatter:on } catch (Exception e) { diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 8f11fa7e..23c771ab 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -105,15 +105,13 @@ public ResultsPage call() { startTimeMillis = System.currentTimeMillis(); QueryQueueListener resultListener = queryQueueManager.createListener(UUID.randomUUID().toString(), queryId); - - // keep waiting for results until we're finished - while (!isFinished(queryId)) { - Message message = resultListener.receive(nextCallProperties.getResultPollIntervalMillis()); - if (message != null) { - Object result = message.getPayload().getPayload(); - - // have to check to make sure we haven't reached the page size - if (!isFinished(queryId)) { + try { + // keep waiting for results until we're finished + // Note: isFinished should be checked once per result + while (!isFinished(queryId)) { + Message message = resultListener.receive(nextCallProperties.getResultPollIntervalMillis()); + if (message != null) { + Object result = message.getPayload().getPayload(); if (result != null) { results.add(result); @@ -126,11 +124,11 @@ public ResultsPage call() { } } } + } finally { + // stop the result listener + resultListener.stop(); } - // stop the result listener - resultListener.stop(); - // update some values for metrics stopTimeMillis = System.currentTimeMillis(); if (lifecycle == null && !results.isEmpty()) { From 13a695ae6c7e4bf74e14107f76d2c6b59f8f583d Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 1 Sep 2021 15:18:45 -0400 Subject: [PATCH 088/218] Updated logging for the query service to be more useful. --- .../microservice/query/QueryController.java | 4 +- .../query/QueryManagementService.java | 118 +++++++++++++----- 2 files changed, 89 insertions(+), 33 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 1ab65e9d..48033d89 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -32,8 +32,8 @@ public QueryController(QueryManagementService queryManagementService) { @Timed(name = "dw.query.listQueryLogic", absolute = true) @RequestMapping(path = "listQueryLogic", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "text/html"}) - public QueryLogicResponse listQueryLogic() { - return queryManagementService.listQueryLogic(); + public QueryLogicResponse listQueryLogic(@AuthenticationPrincipal ProxiedUserDetails currentUser) { + return queryManagementService.listQueryLogic(currentUser); } @Timed(name = "dw.query.defineQuery", absolute = true) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index da50e272..da916bbe 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -142,7 +142,9 @@ public QueryManagementService(QueryProperties queryProperties, ApplicationContex * * @return the query logic descriptions */ - public QueryLogicResponse listQueryLogic() { + public QueryLogicResponse listQueryLogic(ProxiedUserDetails currentUser) { + log.info("Request: listQueryLogic from {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); + QueryLogicResponse response = new QueryLogicResponse(); List> queryLogicList = queryLogicFactory.getQueryLogicList(); List logicConfigurationList = new ArrayList<>(); @@ -181,7 +183,7 @@ public QueryLogicResponse listQueryLogic() { try { logicDesc.setResponseClass(queryLogic.getResponseClass(q)); } catch (QueryException e) { - log.error("Unable to get response class for query logic: " + queryLogic.getLogicName(), e); + log.error("Unable to get response class for query logic: {}", queryLogic.getLogicName(), e); response.addException(e); logicDesc.setResponseClass("unknown"); } @@ -196,7 +198,7 @@ public QueryLogicResponse listQueryLogic() { querySyntax.add(o.toString()); } } catch (Exception e) { - log.warn("Unable to get query syntax for query logic: " + queryLogic.getClass().getCanonicalName()); + log.warn("Unable to get query syntax for query logic: {}", queryLogic.getClass().getCanonicalName()); } if (querySyntax.isEmpty()) { querySyntax.add("CUSTOM"); @@ -246,6 +248,13 @@ public QueryLogicResponse listQueryLogic() { */ public GenericResponse define(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: {}/define from {} with params: {}", queryLogicName, user, parameters); + } else { + log.info("Request: {}/define from {}", queryLogicName, user); + } + try { TaskKey taskKey = storeQuery(queryLogicName, parameters, currentUser, false); GenericResponse response = new GenericResponse<>(); @@ -295,6 +304,13 @@ public GenericResponse define(String queryLogicName, MultiValueMap create(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: {}/create from {} with params: {}", queryLogicName, user, parameters); + } else { + log.info("Request: {}/create from {}", queryLogicName, user); + } + try { TaskKey taskKey = storeQuery(queryLogicName, parameters, currentUser, true); GenericResponse response = new GenericResponse<>(); @@ -353,7 +369,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p QueryLogic queryLogic = validateQuery(queryLogicName, parameters, currentUser); String userId = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); - log.trace(userId + " has authorizations " + currentUser.getPrimaryUser().getAuths()); + log.trace("{} has authorizations {}", userId, currentUser.getPrimaryUser().getAuths()); // set some audit parameters which are used internally String userDn = currentUser.getPrimaryUser().getDn().subjectDN(); @@ -454,6 +470,13 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p */ public BaseQueryResponse createAndNext(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: {}/createAndNext from {} with params: {}", queryLogicName, user, parameters); + } else { + log.info("Request: {}/createAndNext from {}", queryLogicName, user); + } + String queryId = null; try { queryId = create(queryLogicName, parameters, currentUser).getResult(); @@ -463,7 +486,7 @@ public BaseQueryResponse createAndNext(String queryLogicName, MultiValueMap userRoles) thr * if there is an unknown error */ public VoidResponse cancel(String queryId, ProxiedUserDetails currentUser) throws QueryException { + log.info("Request: cancel from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId); + return cancel(queryId, currentUser, false); } @@ -707,7 +734,8 @@ public VoidResponse cancel(String queryId, ProxiedUserDetails currentUser) throw * if there is an unknown error */ public VoidResponse adminCancel(String queryId, ProxiedUserDetails currentUser) throws QueryException { - log.info("Cancel '" + queryId + "'called by admin: " + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); + log.info("Request: adminCancel from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId); + return cancel(queryId, currentUser, true); } @@ -732,7 +760,8 @@ public VoidResponse adminCancel(String queryId, ProxiedUserDetails currentUser) * if there is an unknown error */ public VoidResponse adminCancelAll(ProxiedUserDetails currentUser) throws QueryException { - log.info("Cancel All called by admin: " + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); + log.info("Request: adminCancelAll from {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); + try { List queryStatuses = queryStorageCache.getQueryStatus(); queryStatuses.removeIf(s -> s.getQueryState() != CREATED); @@ -799,7 +828,7 @@ private VoidResponse cancel(String queryId, ProxiedUserDetails currentUser, bool throw e; } catch (Exception e) { QueryException queryException = new QueryException(DatawaveErrorCode.CANCELLATION_ERROR, e, "Unknown error canceling query " + queryId); - log.error("Unknown error canceling query " + queryId, queryException); + log.error("Unknown error canceling query {}", queryId, queryException); throw queryException; } } @@ -861,7 +890,7 @@ public void cancel(String queryId, boolean publishEvent) throws InterruptedExcep .build()); // @formatter:on } catch (Exception e) { - log.error("Error updateing query metric", e); + log.error("Error updating query metric", e); } } } @@ -892,6 +921,8 @@ public void cancel(String queryId, boolean publishEvent) throws InterruptedExcep * if there is an unknown error */ public VoidResponse close(String queryId, ProxiedUserDetails currentUser) throws QueryException { + log.info("Request: close from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId); + return close(queryId, currentUser, false); } @@ -919,7 +950,8 @@ public VoidResponse close(String queryId, ProxiedUserDetails currentUser) throws * if there is an unknown error */ public VoidResponse adminClose(String queryId, ProxiedUserDetails currentUser) throws QueryException { - log.info("Close '" + queryId + "'called by admin: " + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); + log.info("Request: adminClose from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId); + return close(queryId, currentUser, true); } @@ -944,7 +976,8 @@ public VoidResponse adminClose(String queryId, ProxiedUserDetails currentUser) t * if there is an unknown error */ public VoidResponse adminCloseAll(ProxiedUserDetails currentUser) throws QueryException { - log.info("Close All called by admin: " + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); + log.info("Request: adminCloseAll from {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); + try { List queryStatuses = queryStorageCache.getQueryStatus(); queryStatuses.removeIf(s -> s.getQueryState() != CREATED); @@ -1009,7 +1042,7 @@ private VoidResponse close(String queryId, ProxiedUserDetails currentUser, boole throw e; } catch (Exception e) { QueryException queryException = new QueryException(DatawaveErrorCode.QUERY_CLOSE_ERROR, e, "Unknown error closing query " + queryId); - log.error("Unknown error closing query " + queryId, queryException); + log.error("Unknown error closing query {}", queryId, queryException); throw queryException; } } @@ -1097,6 +1130,8 @@ public void close(String queryId) throws InterruptedException, QueryException { * if there is an unknown error */ public GenericResponse reset(String queryId, ProxiedUserDetails currentUser) throws QueryException { + log.info("Request: reset from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId); + try { // make sure the query is valid, and the user can act on it QueryStatus queryStatus = validateRequest(queryId, currentUser); @@ -1117,7 +1152,7 @@ public GenericResponse reset(String queryId, ProxiedUserDetails currentU } catch (QueryException e) { throw e; } catch (Exception e) { - log.error("Unknown error resetting query " + queryId, e); + log.error("Unknown error resetting query {}", queryId, e); throw new QueryException(DatawaveErrorCode.QUERY_RESET_ERROR, e, "Unknown error resetting query " + queryId); } } @@ -1143,6 +1178,8 @@ public GenericResponse reset(String queryId, ProxiedUserDetails currentU * if there is an unknown error */ public VoidResponse remove(String queryId, ProxiedUserDetails currentUser) throws QueryException { + log.info("Request: remove from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId); + return remove(queryId, currentUser, false); } @@ -1165,7 +1202,8 @@ public VoidResponse remove(String queryId, ProxiedUserDetails currentUser) throw * if there is an unknown error */ public VoidResponse adminRemove(String queryId, ProxiedUserDetails currentUser) throws QueryException { - log.info("Remove '" + queryId + "'called by admin: " + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); + log.info("Request: adminRemove from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId); + return remove(queryId, currentUser, true); } @@ -1185,7 +1223,8 @@ public VoidResponse adminRemove(String queryId, ProxiedUserDetails currentUser) * if there is an unknown error */ public VoidResponse adminRemoveAll(ProxiedUserDetails currentUser) throws QueryException { - log.info("Remove All called by admin: " + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); + log.info("Request: adminRemoveAll from {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); + try { List queryStatuses = queryStorageCache.getQueryStatus(); queryStatuses.removeIf(QueryStatus::isRunning); @@ -1246,7 +1285,7 @@ private VoidResponse remove(String queryId, ProxiedUserDetails currentUser, bool } catch (QueryException e) { throw e; } catch (Exception e) { - log.error("Unknown error removing query " + queryId, e); + log.error("Unknown error removing query {}", queryId, e); throw new QueryException(DatawaveErrorCode.QUERY_REMOVAL_ERROR, e, "Unknown error removing query " + queryId); } } @@ -1301,6 +1340,13 @@ private boolean remove(QueryStatus queryStatus) throws IOException { * if there is an unknown error */ public GenericResponse update(String queryId, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: {}/update from {} with params: {}", queryId, user, parameters); + } else { + log.info("Request: {}/update from {}", queryId, user); + } + try { // make sure the query is valid, and the user can act on it QueryStatus queryStatus = validateRequest(queryId, currentUser); @@ -1370,7 +1416,7 @@ public GenericResponse update(String queryId, MultiValueMap update(String queryId, MultiValueMap duplicate(String queryId, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: {}/duplicate from {} with params: {}", queryId, user, parameters); + } else { + log.info("Request: {}/duplicate from {}", queryId, user); + } + try { // make sure the query is valid, and the user can act on it QueryStatus queryStatus = validateRequest(queryId, currentUser); @@ -1428,7 +1481,7 @@ public GenericResponse duplicate(String queryId, MultiValueMap parameterNames, MultiValueMa * if there is an unknown error */ public QueryImplListResponse list(String queryId, String queryName, ProxiedUserDetails currentUser) throws QueryException { + log.info("Request: list from {} for queryId: {}, queryName: {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId, + queryName); + return list(queryId, queryName, ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getDn().subjectDN())); } @@ -1557,8 +1613,9 @@ public QueryImplListResponse list(String queryId, String queryName, ProxiedUserD * if there is an unknown error */ public QueryImplListResponse adminList(String queryId, String queryName, String userId, ProxiedUserDetails currentUser) throws QueryException { - log.info("List '" + String.join(",", Arrays.asList(queryId, queryName, userId)) + "'called by admin: " - + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); + log.info("Request: adminList from {} for queryId: {}, queryName: {}, userId: {}", + ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId, queryName, userId); + return list(queryId, queryName, userId); } @@ -1598,7 +1655,7 @@ private QueryImplListResponse list(String queryId, String queryName, String user response.setQuery(queries); return response; } catch (Exception e) { - log.error("Unknown error listing queries for " + userId, e); + log.error("Unknown error listing queries for {}", userId, e); throw new QueryException(DatawaveErrorCode.QUERY_LISTING_ERROR, e, "Unknown error listing queries for " + userId); } } @@ -1662,7 +1719,7 @@ public void handleRemoteRequest(QueryRequest queryRequest) { log.debug("No handling specified for remote query request method: {}", queryRequest.getMethod()); } } catch (Exception e) { - log.error("Unknown error handling remote request:" + queryRequest); + log.error("Unknown error handling remote request: {}", queryRequest); } } @@ -1917,7 +1974,7 @@ protected void validateParameters(String queryLogicName, MultiValueMap pageMaxTimeoutMillis)) { - log.error("Invalid page timeout: " + queryParameters.getPageTimeout()); + log.error("Invalid page timeout: {}", queryParameters.getPageTimeout()); throw new BadRequestQueryException(DatawaveErrorCode.INVALID_PAGE_TIMEOUT); } // Ensure begin date does not occur after the end date (if dates are not null) if ((queryParameters.getBeginDate() != null && queryParameters.getEndDate() != null) && queryParameters.getBeginDate().after(queryParameters.getEndDate())) { - log.error("Invalid begin and/or end date: " + queryParameters.getBeginDate() + " - " + queryParameters.getEndDate()); + log.error("Invalid begin and/or end date: {}", queryParameters.getBeginDate() + " - " + queryParameters.getEndDate()); throw new BadRequestQueryException(DatawaveErrorCode.BEGIN_DATE_AFTER_END_DATE); } } @@ -1957,7 +2014,7 @@ protected QueryLogic createQueryLogic(String queryLogicName, ProxiedUserDetai try { return queryLogicFactory.getQueryLogic(queryLogicName, currentUser.getPrimaryUser().getRoles()); } catch (Exception e) { - log.error("Failed to get query logic for " + queryLogicName, e); + log.error("Failed to get query logic for {}", queryLogicName, e); throw new BadRequestQueryException(DatawaveErrorCode.QUERY_LOGIC_ERROR, e); } } @@ -1993,7 +2050,7 @@ protected void validateQueryLogic(QueryLogic queryLogic, MultiValueMap 0 && queryParameters.getPagesize() > queryLogic.getMaxPageSize()) { - log.error("Invalid page size: " + queryParameters.getPagesize() + " vs " + queryLogic.getMaxPageSize()); + log.error("Invalid page size: {} vs {}", queryParameters.getPagesize(), queryLogic.getMaxPageSize()); throw new BadRequestQueryException(DatawaveErrorCode.PAGE_SIZE_TOO_LARGE, MessageFormat.format("Max = {0}.", queryLogic.getMaxPageSize())); } @@ -2003,7 +2060,7 @@ protected void validateQueryLogic(QueryLogic queryLogic, MultiValueMap= 0) { if (queryParameters.getMaxResultsOverride() < 0 || (queryLogic.getMaxResults() < queryParameters.getMaxResultsOverride())) { - log.error("Invalid max results override: " + queryParameters.getMaxResultsOverride() + " vs " + queryLogic.getMaxResults()); + log.error("Invalid max results override: {} vs {}", queryParameters.getMaxResultsOverride(), queryLogic.getMaxResults()); throw new BadRequestQueryException(DatawaveErrorCode.INVALID_MAX_RESULTS_OVERRIDE, MessageFormat.format("Max = {0}.", queryLogic.getMaxResults())); } @@ -2013,8 +2070,7 @@ protected void validateQueryLogic(QueryLogic queryLogic, MultiValueMap= 0) { if (queryParameters.getMaxConcurrentTasks() < 0 || (queryLogic.getMaxConcurrentTasks() < queryParameters.getMaxConcurrentTasks())) { - log.error("Invalid max concurrent tasks override: " + queryParameters.getMaxConcurrentTasks() + " vs " - + queryLogic.getMaxConcurrentTasks()); + log.error("Invalid max concurrent tasks override: {} vs {}", queryParameters.getMaxConcurrentTasks(), queryLogic.getMaxConcurrentTasks()); throw new BadRequestQueryException(DatawaveErrorCode.INVALID_MAX_CONCURRENT_TASKS_OVERRIDE, MessageFormat.format("Max = {0}.", queryLogic.getMaxConcurrentTasks())); } From 55f1a136134ecaa15897c8e97d6257493a47f4b2 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Tue, 7 Sep 2021 16:14:01 -0400 Subject: [PATCH 089/218] Fixed service to service communication via spring cloud bus. --- .../query/QueryManagementService.java | 25 ++++++++++++------- .../query/QueryServiceCancelTest.java | 10 ++++---- .../query/QueryServiceDuplicateTest.java | 2 +- .../query/QueryServiceNextTest.java | 6 ++--- .../query/QueryServiceRemoveTest.java | 2 +- .../query/QueryServiceResetTest.java | 4 +-- 6 files changed, 28 insertions(+), 21 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index da916bbe..7e067332 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -50,7 +50,6 @@ import org.slf4j.LoggerFactory; import org.springframework.cloud.bus.BusProperties; import org.springframework.cloud.bus.event.RemoteQueryRequestEvent; -import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationEventPublisher; import org.springframework.core.task.TaskRejectedException; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; @@ -91,7 +90,6 @@ public class QueryManagementService implements QueryRequestHandler { private final QueryProperties queryProperties; - private final ApplicationContext appCtx; private final ApplicationEventPublisher eventPublisher; private final BusProperties busProperties; @@ -113,13 +111,13 @@ public class QueryManagementService implements QueryRequestHandler { private final QueryStatusUpdateHelper queryStatusUpdateHelper; private final MultiValueMap nextCallMap = new LinkedMultiValueMap<>(); - public QueryManagementService(QueryProperties queryProperties, ApplicationContext appCtx, ApplicationEventPublisher eventPublisher, - BusProperties busProperties, QueryParameters queryParameters, SecurityMarking securityMarking, BaseQueryMetric baseQueryMetric, - QueryLogicFactory queryLogicFactory, QueryMetricClient queryMetricClient, ResponseObjectFactory responseObjectFactory, - QueryStorageCache queryStorageCache, QueryQueueManager queryQueueManager, AuditClient auditClient, - ThreadPoolTaskExecutor nextCallExecutor) { + private final String selfDestination; + + public QueryManagementService(QueryProperties queryProperties, ApplicationEventPublisher eventPublisher, BusProperties busProperties, + QueryParameters queryParameters, SecurityMarking securityMarking, BaseQueryMetric baseQueryMetric, QueryLogicFactory queryLogicFactory, + QueryMetricClient queryMetricClient, ResponseObjectFactory responseObjectFactory, QueryStorageCache queryStorageCache, + QueryQueueManager queryQueueManager, AuditClient auditClient, ThreadPoolTaskExecutor nextCallExecutor) { this.queryProperties = queryProperties; - this.appCtx = appCtx; this.eventPublisher = eventPublisher; this.busProperties = busProperties; this.queryParameters = queryParameters; @@ -133,6 +131,7 @@ public QueryManagementService(QueryProperties queryProperties, ApplicationContex this.auditClient = auditClient; this.nextCallExecutor = nextCallExecutor; this.queryStatusUpdateHelper = new QueryStatusUpdateHelper(this.queryProperties, this.queryStorageCache); + this.selfDestination = getSelfDestination(); } /** @@ -1834,11 +1833,19 @@ private void publishSelfEvent(QueryRequest queryRequest) { new RemoteQueryRequestEvent( this, busProperties.getId(), - appCtx.getApplicationName(), + selfDestination, queryRequest)); // @formatter:on } + private String getSelfDestination() { + String id = busProperties.getId(); + if (id.contains(":")) { + return id.substring(0, id.indexOf(":")); + } + return id; + } + /** * Gets the maximum number of concurrent query tasks allowed for this logic. *

diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java index 56d80ffe..c2629027 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java @@ -88,7 +88,7 @@ public void testCancelSuccess() throws Exception { queryId, queryRequestEvents.removeLast()); assertQueryRequestEvent( - "/query:**", + "query:**", QueryRequest.Method.CANCEL, queryId, queryRequestEvents.removeLast()); @@ -168,7 +168,7 @@ public void testCancelSuccess_activeNextCall() throws Exception { queryId, queryRequestEvents.removeLast()); assertQueryRequestEvent( - "/query:**", + "query:**", QueryRequest.Method.CANCEL, queryId, queryRequestEvents.removeLast()); @@ -279,7 +279,7 @@ public void testCancelFailure_queryNotRunning() throws Exception { queryId, queryRequestEvents.removeLast()); assertQueryRequestEvent( - "/query:**", + "query:**", QueryRequest.Method.CANCEL, queryId, queryRequestEvents.removeLast()); @@ -337,7 +337,7 @@ public void testAdminCancelSuccess() throws Exception { queryId, queryRequestEvents.removeLast()); assertQueryRequestEvent( - "/query:**", + "query:**", QueryRequest.Method.CANCEL, queryId, queryRequestEvents.removeLast()); @@ -433,7 +433,7 @@ public void testAdminCancelAllSuccess() throws Exception { // @formatter:off assertQueryRequestEvent( - "/query:**", + "query:**", QueryRequest.Method.CANCEL, queryId, queryRequestEvents.removeLast()); diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java index 2facee7c..18c40803 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java @@ -271,7 +271,7 @@ public void testDuplicateSuccess_duplicateOnCanceled() throws Exception { queryId, queryRequestEvents.removeLast()); assertQueryRequestEvent( - "/query:**", + "query:**", QueryRequest.Method.CANCEL, queryId, queryRequestEvents.removeLast()); diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java index c826d7a2..b0d4aef4 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java @@ -293,7 +293,7 @@ public void testNextSuccess_cancelPartialResults() throws Exception { queryId, queryRequestEvents.removeLast()); assertQueryRequestEvent( - "/query:**", + "query:**", QueryRequest.Method.CANCEL, queryId, queryRequestEvents.removeLast()); @@ -539,7 +539,7 @@ public void testNextFailure_queryNotRunning() throws Exception { queryId, queryRequestEvents.removeLast()); assertQueryRequestEvent( - "/query:**", + "query:**", QueryRequest.Method.CANCEL, queryId, queryRequestEvents.removeLast()); @@ -739,7 +739,7 @@ public void testNextFailure_nextOnCanceled() throws Exception { queryId, queryRequestEvents.removeLast()); assertQueryRequestEvent( - "/query:**", + "query:**", QueryRequest.Method.CANCEL, queryId, queryRequestEvents.removeLast()); diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java index fac5f558..94c9fa6a 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java @@ -145,7 +145,7 @@ public void testRemoveSuccess_removeOnCanceled() throws Exception { queryId, queryRequestEvents.removeLast()); assertQueryRequestEvent( - "/query:**", + "query:**", QueryRequest.Method.CANCEL, queryId, queryRequestEvents.removeLast()); diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java index 30c40407..7247500d 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java @@ -192,7 +192,7 @@ public void testResetSuccess_resetOnCreated() throws Exception { queryId, queryRequestEvents.removeLast()); assertQueryRequestEvent( - "/query:**", + "query:**", QueryRequest.Method.CANCEL, queryId, queryRequestEvents.removeLast()); @@ -387,7 +387,7 @@ public void testResetSuccess_resetOnCanceled() throws Exception { queryId, queryRequestEvents.removeLast()); assertQueryRequestEvent( - "/query:**", + "query:**", QueryRequest.Method.CANCEL, queryId, queryRequestEvents.removeLast()); From 4f5a0e57ef29777bef92da0ea1e8a85d7b2afa96 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 10 Sep 2021 12:05:46 -0400 Subject: [PATCH 090/218] Updated query service to log audit messages before sending. --- .../microservice/query/QueryManagementService.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 7e067332..7e008e6e 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -1763,15 +1763,19 @@ protected void audit(Query query, QueryLogic queryLogic, MultiValueMap Date: Fri, 10 Sep 2021 18:09:35 +0000 Subject: [PATCH 091/218] Fixed the test configurations --- .../service/src/test/resources/config/application.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/query-microservices/query-service/service/src/test/resources/config/application.yml b/query-microservices/query-service/service/src/test/resources/config/application.yml index 2183202b..24cac37a 100644 --- a/query-microservices/query-service/service/src/test/resources/config/application.yml +++ b/query-microservices/query-service/service/src/test/resources/config/application.yml @@ -200,7 +200,7 @@ datawave: zookeeperConfig: ${warehouse.accumulo.zookeepers} ivaratorCacheDirConfigs: - basePathURI: "hdfs:///IvaratorCache" - ivaratorFstHdfsBaseURIs: ${warehouse.accumulo.ivaratorFstHdfsBaseURIs} + ivaratorFstHdfsBaseURIs: ${warehouse.defaults.ivaratorFstHdfsBaseURIs} ivaratorCacheBufferSize: ${warehouse.defaults.ivaratorCacheBufferSize} ivaratorMaxOpenFiles: ${warehouse.defaults.ivaratorMaxOpenFiles} ivaratorCacheScanPersistThreshold: ${warehouse.defaults.ivaratorCacheScanPersistThreshold} From 1eda4d55e402297a3534b3ec75178caad0231c87 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Mon, 13 Sep 2021 14:26:58 -0400 Subject: [PATCH 092/218] Updated the query service to wait for a create event from the executor service before returning to the user. --- .../query/config/QueryProperties.java | 10 ++++ .../query/QueryManagementService.java | 48 +++++++++++++++++-- .../src/test/resources/config/application.yml | 1 + 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java index e8179f98..db115471 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java @@ -33,6 +33,8 @@ public class QueryProperties { private String executorServiceName = "executor"; // These are the only parameters that can be updated for a running query private List updatableParams = Arrays.asList(QUERY_EXPIRATION, QUERY_PAGESIZE, QUERY_PAGETIMEOUT, QUERY_MAX_RESULTS_OVERRIDE); + // Whether or not to wait for an executor create response before returning to the caller + private boolean awaitExecutorCreateResponse = true; private QueryExpirationProperties expiration = new QueryExpirationProperties(); private NextCallProperties nextCall = new NextCallProperties(); @@ -104,6 +106,14 @@ public void setUpdatableParams(List updatableParams) { this.updatableParams = updatableParams; } + public boolean isAwaitExecutorCreateResponse() { + return awaitExecutorCreateResponse; + } + + public void setAwaitExecutorCreateResponse(boolean awaitExecutorCreateResponse) { + this.awaitExecutorCreateResponse = awaitExecutorCreateResponse; + } + public QueryExpirationProperties getExpiration() { return expiration; } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 7e008e6e..ad1f7e52 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -69,6 +69,8 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -113,6 +115,8 @@ public class QueryManagementService implements QueryRequestHandler { private final String selfDestination; + private final Map createLatchMap = new ConcurrentHashMap<>(); + public QueryManagementService(QueryProperties queryProperties, ApplicationEventPublisher eventPublisher, BusProperties busProperties, QueryParameters queryParameters, SecurityMarking securityMarking, BaseQueryMetric baseQueryMetric, QueryLogicFactory queryLogicFactory, QueryMetricClient queryMetricClient, ResponseObjectFactory responseObjectFactory, QueryStorageCache queryStorageCache, @@ -402,8 +406,32 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p getMaxConcurrentTasks(queryLogic)); // @formatter:on + if (queryProperties.isAwaitExecutorCreateResponse()) { + // before publishing the message, create a latch based on the query ID + createLatchMap.put(taskKey.getQueryId(), new CountDownLatch(1)); + } + // publish a create event to the executor pool publishExecutorEvent(QueryRequest.create(taskKey.getQueryId()), getPoolName()); + + if (queryProperties.isAwaitExecutorCreateResponse()) { + log.info("Waiting on query create response from the executor."); + // wait for the executor to finish creating the query + try { + // TODO: Should we incorporate the call start time into this check? + if (!createLatchMap.get(taskKey.getQueryId()).await(queryProperties.getExpiration().getCallTimeout(), + queryProperties.getExpiration().getCallTimeUnit())) { + throw new QueryException(DatawaveErrorCode.QUERY_TIMEOUT, "Timed out waiting for query create to finish."); + } else { + log.info("Received query create response from the executor."); + } + } catch (InterruptedException e) { + log.warn("Interrupted while waiting for query create latch for queryId {}", queryId); + throw new QueryException(DatawaveErrorCode.QUERY_TIMEOUT, "Interrupted while waiting for query create to finish."); + } finally { + createLatchMap.remove(taskKey.getQueryId()); + } + } } else { // @formatter:off taskKey = queryStorageCache.defineQuery( @@ -1707,18 +1735,30 @@ private QueryStatus validateRequest(String queryId, ProxiedUserDetails currentUs * * @param queryRequest * the remote query request, not null + * @param originService + * the address of the service which sent the request + * @param destinationService + * the address of the service which this event was sent to */ @Override - public void handleRemoteRequest(QueryRequest queryRequest) { + public void handleRemoteRequest(QueryRequest queryRequest, String originService, String destinationService) { try { if (queryRequest.getMethod() == QueryRequest.Method.CANCEL) { - log.trace("Received remote cancel request."); + log.trace("Received remote cancel request from {} for {}.", originService, destinationService); cancel(queryRequest.getQueryId(), false); + } else if (queryRequest.getMethod() == QueryRequest.Method.CREATE) { + log.trace("Received remote create request from {} for {}.", originService, destinationService); + if (createLatchMap.containsKey(queryRequest.getQueryId())) { + createLatchMap.get(queryRequest.getQueryId()).countDown(); + } else { + log.warn("Unable to decrement create latch for query {}", queryRequest.getQueryId()); + } } else { - log.debug("No handling specified for remote query request method: {}", queryRequest.getMethod()); + log.debug("No handling specified for remote query request method: {} from {} for {}", queryRequest.getMethod(), originService, + destinationService); } } catch (Exception e) { - log.error("Unknown error handling remote request: {}", queryRequest); + log.error("Unknown error handling remote request: {} from {} for {}", queryRequest, originService, destinationService); } } diff --git a/query-microservices/query-service/service/src/test/resources/config/application.yml b/query-microservices/query-service/service/src/test/resources/config/application.yml index 24cac37a..afe8cd72 100644 --- a/query-microservices/query-service/service/src/test/resources/config/application.yml +++ b/query-microservices/query-service/service/src/test/resources/config/application.yml @@ -110,6 +110,7 @@ datawave: "[datawave.data.type.DateType]": "datawave.data.type.RawDateType" query: + awaitExecutorCreateResponse: false nextCall: resultPollInterval: 500 statusUpdateInterval: 500 From f73833c34cb720b2fec598c0006c9edb408bfbd1 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Tue, 14 Sep 2021 15:00:27 -0400 Subject: [PATCH 093/218] Updated the extracted modules to produce jboss-compatible jars. Got the wildfly webservice working again. --- query-microservices/query-service/api/pom.xml | 26 +++++++++++++++++++ .../api/src/main/resources/META-INF/beans.xml | 9 +++++++ .../main/resources/META-INF/jboss-ejb3.xml | 17 ++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 query-microservices/query-service/api/src/main/resources/META-INF/beans.xml create mode 100644 query-microservices/query-service/api/src/main/resources/META-INF/jboss-ejb3.xml diff --git a/query-microservices/query-service/api/pom.xml b/query-microservices/query-service/api/pom.xml index 6269478f..ba5b7216 100644 --- a/query-microservices/query-service/api/pom.xml +++ b/query-microservices/query-service/api/pom.xml @@ -68,4 +68,30 @@ https://raw.githubusercontent.com/NationalSecurityAgency/datawave/mvn-repo + + + + org.apache.maven.plugins + maven-jar-plugin + + + jboss + + jar + + + jboss + + + + + + + META-INF/beans.xml + META-INF/jboss-ejb3.xml + + + + + diff --git a/query-microservices/query-service/api/src/main/resources/META-INF/beans.xml b/query-microservices/query-service/api/src/main/resources/META-INF/beans.xml new file mode 100644 index 00000000..4ca201f8 --- /dev/null +++ b/query-microservices/query-service/api/src/main/resources/META-INF/beans.xml @@ -0,0 +1,9 @@ + + + + \ No newline at end of file diff --git a/query-microservices/query-service/api/src/main/resources/META-INF/jboss-ejb3.xml b/query-microservices/query-service/api/src/main/resources/META-INF/jboss-ejb3.xml new file mode 100644 index 00000000..f29fb358 --- /dev/null +++ b/query-microservices/query-service/api/src/main/resources/META-INF/jboss-ejb3.xml @@ -0,0 +1,17 @@ + + + + + + + * + datawave + + + + \ No newline at end of file From c3e183e861972e7b78ee8b4166b3cc40e5797232 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 17 Sep 2021 16:46:26 -0400 Subject: [PATCH 094/218] Refactored the common, query, and connection-pool modules into a separate, independent services module, which is required by both the webservice and microservices. --- .../api/src/main/resources/META-INF/jboss-ejb3.xml | 1 - query-microservices/query-service/service/pom.xml | 8 ++++---- .../microservice/query/QueryManagementService.java | 6 +++--- .../java/datawave/microservice/query/runner/NextCall.java | 2 +- .../web/filter/QueryMetricsEnrichmentFilterAdvice.java | 2 +- .../src/test/resources/MyTestQueryLogicFactory.xml | 4 ++-- 6 files changed, 11 insertions(+), 12 deletions(-) diff --git a/query-microservices/query-service/api/src/main/resources/META-INF/jboss-ejb3.xml b/query-microservices/query-service/api/src/main/resources/META-INF/jboss-ejb3.xml index f29fb358..8cf49db8 100644 --- a/query-microservices/query-service/api/src/main/resources/META-INF/jboss-ejb3.xml +++ b/query-microservices/query-service/api/src/main/resources/META-INF/jboss-ejb3.xml @@ -2,7 +2,6 @@ diff --git a/query-microservices/query-service/service/pom.xml b/query-microservices/query-service/service/pom.xml index 517140fe..719f755e 100644 --- a/query-microservices/query-service/service/pom.xml +++ b/query-microservices/query-service/service/pom.xml @@ -50,10 +50,6 @@ - - gov.nsa.datawave.microservice - query - gov.nsa.datawave.microservice query-api @@ -70,6 +66,10 @@ gov.nsa.datawave.microservice spring-boot-starter-datawave-query + + gov.nsa.datawave.services + datawave-services-query + gov.nsa.datawave.microservice spring-boot-starter-datawave-query diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index ad1f7e52..0280787e 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -6,10 +6,7 @@ import datawave.microservice.audit.AuditClient; import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.authorization.util.AuthorizationsUtil; -import datawave.microservice.common.audit.PrivateAuditConstants; import datawave.microservice.query.config.QueryProperties; -import datawave.microservice.query.logic.QueryLogic; -import datawave.microservice.query.logic.QueryLogicFactory; import datawave.microservice.query.remote.QueryRequest; import datawave.microservice.query.remote.QueryRequestHandler; import datawave.microservice.query.runner.NextCall; @@ -23,6 +20,9 @@ import datawave.microservice.querymetric.QueryMetricClient; import datawave.microservice.querymetric.QueryMetricType; import datawave.security.util.ProxiedEntityUtils; +import datawave.services.common.audit.PrivateAuditConstants; +import datawave.services.query.logic.QueryLogic; +import datawave.services.query.logic.QueryLogicFactory; import datawave.webservice.common.audit.AuditParameters; import datawave.webservice.common.audit.Auditor; import datawave.webservice.query.Query; diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 23c771ab..829c528e 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -3,7 +3,6 @@ import datawave.microservice.query.config.NextCallProperties; import datawave.microservice.query.config.QueryExpirationProperties; import datawave.microservice.query.config.QueryProperties; -import datawave.microservice.query.logic.QueryLogic; import datawave.microservice.query.storage.QueryQueueListener; import datawave.microservice.query.storage.QueryQueueManager; import datawave.microservice.query.storage.QueryStatus; @@ -12,6 +11,7 @@ import datawave.microservice.query.storage.TaskStates; import datawave.microservice.querymetric.BaseQueryMetric; import datawave.microservice.querymetric.QueryMetric; +import datawave.services.query.logic.QueryLogic; import datawave.webservice.query.cache.ResultsPage; import datawave.webservice.query.data.ObjectSizeOf; import org.slf4j.Logger; diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java index 06ec9ba6..e8c33c39 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java @@ -1,12 +1,12 @@ package datawave.microservice.query.web.filter; -import datawave.microservice.query.logic.QueryLogicFactory; import datawave.microservice.query.storage.QueryStatus; import datawave.microservice.query.storage.QueryStorageCache; import datawave.microservice.query.web.annotation.EnrichQueryMetrics; import datawave.microservice.querymetric.BaseQueryMetric; import datawave.microservice.querymetric.QueryMetricClient; import datawave.microservice.querymetric.QueryMetricType; +import datawave.services.query.logic.QueryLogicFactory; import datawave.webservice.result.BaseQueryResponse; import datawave.webservice.result.GenericResponse; import org.apache.log4j.Logger; diff --git a/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml b/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml index 7fcd6122..7988bf95 100644 --- a/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml +++ b/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml @@ -8,7 +8,7 @@ http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd"> - + @@ -342,7 +342,7 @@ - + From 9fff9183abe020fed45032ee216dc4e2be0a7bfb Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Mon, 20 Sep 2021 13:59:44 -0400 Subject: [PATCH 095/218] Created datawave-services-common-util to contain the classes from datawave-ws-common-util which are not jboss-specific. --- .../datawave/microservice/query/QueryManagementService.java | 2 +- .../main/java/datawave/microservice/query/runner/NextCall.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 0280787e..fa635bde 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -21,13 +21,13 @@ import datawave.microservice.querymetric.QueryMetricType; import datawave.security.util.ProxiedEntityUtils; import datawave.services.common.audit.PrivateAuditConstants; +import datawave.services.query.cache.ResultsPage; import datawave.services.query.logic.QueryLogic; import datawave.services.query.logic.QueryLogicFactory; import datawave.webservice.common.audit.AuditParameters; import datawave.webservice.common.audit.Auditor; import datawave.webservice.query.Query; import datawave.webservice.query.QueryImpl; -import datawave.webservice.query.cache.ResultsPage; import datawave.webservice.query.exception.BadRequestQueryException; import datawave.webservice.query.exception.DatawaveErrorCode; import datawave.webservice.query.exception.NoResultsQueryException; diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 829c528e..5e1cc999 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -11,8 +11,8 @@ import datawave.microservice.query.storage.TaskStates; import datawave.microservice.querymetric.BaseQueryMetric; import datawave.microservice.querymetric.QueryMetric; +import datawave.services.query.cache.ResultsPage; import datawave.services.query.logic.QueryLogic; -import datawave.webservice.query.cache.ResultsPage; import datawave.webservice.query.data.ObjectSizeOf; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From 21a1a1621d22f473432da858176adac80a8ae3ea Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 22 Sep 2021 15:41:05 -0400 Subject: [PATCH 096/218] Moved the remote dictionary services out of datawave-query-core. --- .../service/src/test/resources/MyTestQueryLogicFactory.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml b/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml index 7988bf95..6e915fee 100644 --- a/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml +++ b/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml @@ -816,7 +816,10 @@ - + + + + From d185687110641d99fc26846fd79b70046902d6d5 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Wed, 22 Sep 2021 21:21:40 +0000 Subject: [PATCH 097/218] Updated to store the originService in the QueryStatus cache for use by the executor to notify the origin service of completion. Updated the next and seek counts in the QueryStatus cache for use by the QueryManagement service. --- .../java/datawave/microservice/query/QueryManagementService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index fa635bde..b0c5952d 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -402,6 +402,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p taskKey = queryStorageCache.createQuery( getPoolName(), query, + busProperties.getId(), downgradedAuthorizations, getMaxConcurrentTasks(queryLogic)); // @formatter:on From 84103af2c84582e597e1416fa10f33c65b7623eb Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Wed, 29 Sep 2021 18:36:00 +0000 Subject: [PATCH 098/218] Added eror code into the query status, and added looking for a failed query in the NextCall --- .../microservice/query/runner/NextCall.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 5e1cc999..369b526f 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -14,6 +14,7 @@ import datawave.services.query.cache.ResultsPage; import datawave.services.query.logic.QueryLogic; import datawave.webservice.query.data.ObjectSizeOf; +import datawave.webservice.query.exception.QueryException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.messaging.Message; @@ -124,6 +125,9 @@ public ResultsPage call() { } } } + } catch (QueryException e) { + // TODO Is there a better way to pass this exception up? + throw new RuntimeException(e); } finally { // stop the result listener resultListener.stop(); @@ -143,13 +147,20 @@ public void updateQueryMetric(BaseQueryMetric baseQueryMetric) { baseQueryMetric.setLifecycle(lifecycle); } - private boolean isFinished(String queryId) { + private boolean isFinished(String queryId) throws QueryException { boolean finished = false; long callTimeMillis = System.currentTimeMillis() - startTimeMillis; final QueryStatus queryStatus = getQueryStatus(); + // 0) has the query failed? + if (queryStatus.getQueryState() == QueryStatus.QUERY_STATE.FAILED) { + log.error("Query [{}]: failed: {}\n{}", queryId, queryStatus.getFailureMessage(), queryStatus.getStackTrace()); + + throw new QueryException(queryStatus.getErrorCode(), queryStatus.getFailureMessage()); + } + // 1) have we hit the user's results-per-page limit? - if (results.size() >= userResultsPerPage) { + if (!finished && results.size() >= userResultsPerPage) { log.info("Query [{}]: user requested max page size has been reached, aborting next call", queryId); finished = true; From 1816ecd92f2862093b82f73b9bc05c53ea08f559 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Wed, 27 Oct 2021 21:33:34 +0000 Subject: [PATCH 099/218] Merge branch 'integration' into feature/queryMicroservices --- .../query/DefaultQueryParameters.java | 49 +++++++++---------- .../microservice/query/QueryParameters.java | 10 ++-- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java index 1e1b53ee..2c31d178 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java @@ -12,6 +12,7 @@ import java.util.Arrays; import java.util.Calendar; import java.util.Date; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -38,7 +39,7 @@ public class DefaultQueryParameters implements QueryParameters { protected String pool; protected boolean isMaxConcurrentTasksOverridden; protected int maxConcurrentTasks; - protected MultiValueMap requestHeaders; + protected Map> requestHeaders; public DefaultQueryParameters() { clear(); @@ -276,9 +277,9 @@ public static synchronized Date parseDate(String s, String defaultTime, String d } /** - * Convenience method to generate a MultivaluedMap from the specified arguments. If an argument is null, it's associated parameter name (key) will not be - * added to the map, which is why Integer and Boolean wrappers are used for greater flexibility. - * + * Convenience method to generate a {@code Map>} from the specified arguments. If an argument is null, it's associated parameter name + * (key) will not be added to the map, which is why Integer and Boolean wrappers are used for greater flexibility. + * * The 'parameters' argument will not be parsed, so its internal elements will not be placed into the map. If non-null, the 'parameters' value will be * mapped directly to the QUERY_PARAMS key. * @@ -316,54 +317,54 @@ public static synchronized Date parseDate(String s, String defaultTime, String d * @throws ParseException * on date parse/format error */ - public static MultiValueMap paramsToMap(String queryLogicName, String query, String queryName, String queryVisibility, Date beginDate, + public static Map> paramsToMap(String queryLogicName, String query, String queryName, String queryVisibility, Date beginDate, Date endDate, String queryAuthorizations, Date expirationDate, Integer pagesize, Integer pageTimeout, Long maxResultsOverride, QueryPersistence persistenceMode, String parameters, Boolean trace) throws ParseException { MultiValueMap p = new LinkedMultiValueMap<>(); if (queryLogicName != null) { - p.add(QueryParameters.QUERY_LOGIC_NAME, queryLogicName); + p.set(QueryParameters.QUERY_LOGIC_NAME, queryLogicName); } if (query != null) { - p.add(QueryParameters.QUERY_STRING, query); + p.set(QueryParameters.QUERY_STRING, query); } if (queryName != null) { - p.add(QueryParameters.QUERY_NAME, queryName); + p.set(QueryParameters.QUERY_NAME, queryName); } if (queryVisibility != null) { - p.add(QueryParameters.QUERY_VISIBILITY, queryVisibility); + p.set(QueryParameters.QUERY_VISIBILITY, queryVisibility); } if (beginDate != null) { - p.add(QueryParameters.QUERY_BEGIN, formatDate(beginDate)); + p.set(QueryParameters.QUERY_BEGIN, formatDate(beginDate)); } if (endDate != null) { - p.add(QueryParameters.QUERY_END, formatDate(endDate)); + p.set(QueryParameters.QUERY_END, formatDate(endDate)); } if (queryAuthorizations != null) { // ensure that auths are comma separated with no empty values or spaces Splitter splitter = Splitter.on(',').omitEmptyStrings().trimResults(); - p.add(QueryParameters.QUERY_AUTHORIZATIONS, StringUtils.join(splitter.splitToList(queryAuthorizations), ",")); + p.set(QueryParameters.QUERY_AUTHORIZATIONS, StringUtils.join(splitter.splitToList(queryAuthorizations), ",")); } if (expirationDate != null) { - p.add(QueryParameters.QUERY_EXPIRATION, formatDate(expirationDate)); + p.set(QueryParameters.QUERY_EXPIRATION, formatDate(expirationDate)); } if (pagesize != null) { - p.add(QueryParameters.QUERY_PAGESIZE, pagesize.toString()); + p.set(QueryParameters.QUERY_PAGESIZE, pagesize.toString()); } if (pageTimeout != null) { - p.add(QueryParameters.QUERY_PAGETIMEOUT, pageTimeout.toString()); + p.set(QueryParameters.QUERY_PAGETIMEOUT, pageTimeout.toString()); } if (maxResultsOverride != null) { - p.add(QueryParameters.QUERY_MAX_RESULTS_OVERRIDE, maxResultsOverride.toString()); + p.set(QueryParameters.QUERY_MAX_RESULTS_OVERRIDE, maxResultsOverride.toString()); } if (persistenceMode != null) { - p.add(QueryParameters.QUERY_PERSISTENCE, persistenceMode.name()); + p.set(QueryParameters.QUERY_PERSISTENCE, persistenceMode.name()); } if (trace != null) { - p.add(QueryParameters.QUERY_TRACE, trace.toString()); + p.set(QueryParameters.QUERY_TRACE, trace.toString()); } if (parameters != null) { - p.add(QueryParameters.QUERY_PARAMS, parameters); + p.set(QueryParameters.QUERY_PARAMS, parameters); } return p; @@ -530,23 +531,21 @@ public boolean isMaxConcurrentTasksOverridden() { } @Override - public MultiValueMap getRequestHeaders() { + public Map> getRequestHeaders() { return requestHeaders; } @Override - public void setRequestHeaders(MultiValueMap requestHeaders) { + public void setRequestHeaders(Map> requestHeaders) { this.requestHeaders = requestHeaders; } @Override - public MultiValueMap getUnknownParameters(MultiValueMap allQueryParameters) { + public MultiValueMap getUnknownParameters(Map> allQueryParameters) { MultiValueMap p = new LinkedMultiValueMap<>(); for (String key : allQueryParameters.keySet()) { if (!KNOWN_PARAMS.contains(key)) { - for (String value : allQueryParameters.get(key)) { - p.add(key, value); - } + p.put(key, allQueryParameters.get(key)); } } return p; diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParameters.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParameters.java index 02345c48..a9b95fed 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParameters.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParameters.java @@ -5,6 +5,10 @@ import java.util.Date; +import java.util.Date; +import java.util.List; +import java.util.Map; + /** * QueryParameters passed in from a client, they are validated and passed through to the iterator stack as QueryOptions. * @@ -92,11 +96,11 @@ public interface QueryParameters extends ParameterValidator { boolean isMaxConcurrentTasksOverridden(); - MultiValueMap getRequestHeaders(); + Map> getRequestHeaders(); - void setRequestHeaders(MultiValueMap requestHeaders); + void setRequestHeaders(Map> requestHeaders); - MultiValueMap getUnknownParameters(MultiValueMap allQueryParameters); + MultiValueMap getUnknownParameters(Map> allQueryParameters); void clear(); From 0fb6bf5ab4e618b2d80dd390c1c76d582c418c89 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 28 Oct 2021 17:18:19 -0400 Subject: [PATCH 100/218] Updated query checkpoints to store the query configuration. Updated query service next call to throw exception without wrapping as RuntimeException. --- .../datawave/microservice/query/runner/NextCall.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 369b526f..1101e65b 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -102,7 +102,7 @@ private NextCall(Builder builder) { } @Override - public ResultsPage call() { + public ResultsPage call() throws Exception { startTimeMillis = System.currentTimeMillis(); QueryQueueListener resultListener = queryQueueManager.createListener(UUID.randomUUID().toString(), queryId); @@ -125,9 +125,6 @@ public ResultsPage call() { } } } - } catch (QueryException e) { - // TODO Is there a better way to pass this exception up? - throw new RuntimeException(e); } finally { // stop the result listener resultListener.stop(); @@ -152,15 +149,15 @@ private boolean isFinished(String queryId) throws QueryException { long callTimeMillis = System.currentTimeMillis() - startTimeMillis; final QueryStatus queryStatus = getQueryStatus(); - // 0) has the query failed? + // if the query state is FAILED, throw an exception up to the query management service with the failure message if (queryStatus.getQueryState() == QueryStatus.QUERY_STATE.FAILED) { - log.error("Query [{}]: failed: {}\n{}", queryId, queryStatus.getFailureMessage(), queryStatus.getStackTrace()); + log.error("Query [{}]: query failed, aborting next call. Cause: {}", queryId, queryStatus.getFailureMessage()); throw new QueryException(queryStatus.getErrorCode(), queryStatus.getFailureMessage()); } // 1) have we hit the user's results-per-page limit? - if (!finished && results.size() >= userResultsPerPage) { + if (results.size() >= userResultsPerPage) { log.info("Query [{}]: user requested max page size has been reached, aborting next call", queryId); finished = true; From edc398f399ba88bbf2d1558ed39515945ff0f7eb Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 29 Oct 2021 17:11:36 -0400 Subject: [PATCH 101/218] Updated ingest config to use correct date for tvmaze data. Fixed an intermittent bug with the query service cancel tests. --- .../datawave/microservice/query/QueryServiceCancelTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java index c2629027..8ca9f175 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java @@ -134,12 +134,16 @@ public void testCancelSuccess_activeNextCall() throws Exception { // verify that query status was created correctly QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + // wait for the next call to drop out before checking status + // since we canceled, this should quit immediately, but just in case, we add a timeout + nextFuture.get(10, TimeUnit.SECONDS); + // @formatter:off assertQueryStatus( QueryStatus.QUERY_STATE.CANCELED, 0, 0, - 1, + 0, 0, currentTimeMillis, queryStatus); From 501240f46bc82699ec760c1600201432e646ae64 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Wed, 3 Nov 2021 17:44:03 +0000 Subject: [PATCH 102/218] Renamed some of the executor classes to be more understandable. changed the QueryExecutor to hear self notifications Updated the checkpoint mechanism to send a notification for each task Updated the executor task to send a notification when a task is not complete Updated the next call to ensure no results are missed (but now get duplicate results?) --- .../microservice/query/runner/NextCall.java | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 1101e65b..a35a0a31 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -110,19 +110,8 @@ public ResultsPage call() throws Exception { // keep waiting for results until we're finished // Note: isFinished should be checked once per result while (!isFinished(queryId)) { - Message message = resultListener.receive(nextCallProperties.getResultPollIntervalMillis()); - if (message != null) { - Object result = message.getPayload().getPayload(); - if (result != null) { - results.add(result); - - if (logicBytesPerPage > 0) { - pageSizeBytes += ObjectSizeOf.Sizer.getObjectSize(result); - } - } else { - log.debug("Null result encountered, no more results"); - break; - } + if (pullResult(resultListener) == null) { + break; } } } finally { @@ -130,6 +119,13 @@ public ResultsPage call() throws Exception { resultListener.stop(); } + // now that we have stopped the listener, make sure we use any results that may have arrived in the meantime + while (resultListener.hasResults()) { + if (pullResult(resultListener) != null) { + log.info("Got result after we were finished with the page!"); + } + } + // update some values for metrics stopTimeMillis = System.currentTimeMillis(); if (lifecycle == null && !results.isEmpty()) { @@ -139,6 +135,24 @@ public ResultsPage call() throws Exception { return new ResultsPage<>(results, status); } + private Object pullResult(QueryQueueListener resultListener) { + Message message = resultListener.receive(nextCallProperties.getResultPollIntervalMillis()); + if (message != null) { + Object result = message.getPayload().getPayload(); + if (result != null) { + results.add(result); + + if (logicBytesPerPage > 0) { + pageSizeBytes += ObjectSizeOf.Sizer.getObjectSize(result); + } + } else { + log.debug("Null result encountered, no more results"); + } + return result; + } + return null; + } + public void updateQueryMetric(BaseQueryMetric baseQueryMetric) { baseQueryMetric.addPageTime(results.size(), stopTimeMillis - startTimeMillis, startTimeMillis, stopTimeMillis); baseQueryMetric.setLifecycle(lifecycle); From 526b52b97ef187c519ad8a5ffd2acbab9b57b33d Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Wed, 3 Nov 2021 18:21:47 +0000 Subject: [PATCH 103/218] Revert next call to not check the listener after stopping it. --- .../microservice/query/runner/NextCall.java | 40 ++++++------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index a35a0a31..1101e65b 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -110,8 +110,19 @@ public ResultsPage call() throws Exception { // keep waiting for results until we're finished // Note: isFinished should be checked once per result while (!isFinished(queryId)) { - if (pullResult(resultListener) == null) { - break; + Message message = resultListener.receive(nextCallProperties.getResultPollIntervalMillis()); + if (message != null) { + Object result = message.getPayload().getPayload(); + if (result != null) { + results.add(result); + + if (logicBytesPerPage > 0) { + pageSizeBytes += ObjectSizeOf.Sizer.getObjectSize(result); + } + } else { + log.debug("Null result encountered, no more results"); + break; + } } } } finally { @@ -119,13 +130,6 @@ public ResultsPage call() throws Exception { resultListener.stop(); } - // now that we have stopped the listener, make sure we use any results that may have arrived in the meantime - while (resultListener.hasResults()) { - if (pullResult(resultListener) != null) { - log.info("Got result after we were finished with the page!"); - } - } - // update some values for metrics stopTimeMillis = System.currentTimeMillis(); if (lifecycle == null && !results.isEmpty()) { @@ -135,24 +139,6 @@ public ResultsPage call() throws Exception { return new ResultsPage<>(results, status); } - private Object pullResult(QueryQueueListener resultListener) { - Message message = resultListener.receive(nextCallProperties.getResultPollIntervalMillis()); - if (message != null) { - Object result = message.getPayload().getPayload(); - if (result != null) { - results.add(result); - - if (logicBytesPerPage > 0) { - pageSizeBytes += ObjectSizeOf.Sizer.getObjectSize(result); - } - } else { - log.debug("Null result encountered, no more results"); - } - return result; - } - return null; - } - public void updateQueryMetric(BaseQueryMetric baseQueryMetric) { baseQueryMetric.addPageTime(results.size(), stopTimeMillis - startTimeMillis, startTimeMillis, stopTimeMillis); baseQueryMetric.setLifecycle(lifecycle); From 424960136897715739a9b87fe089785a33f3ecad Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Fri, 5 Nov 2021 13:59:53 +0000 Subject: [PATCH 104/218] Added a latch mechanism to acknowledge results correctly. --- .../main/java/datawave/microservice/query/runner/NextCall.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 1101e65b..46794ce9 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -112,7 +112,7 @@ public ResultsPage call() throws Exception { while (!isFinished(queryId)) { Message message = resultListener.receive(nextCallProperties.getResultPollIntervalMillis()); if (message != null) { - Object result = message.getPayload().getPayload(); + Object result = message.getPayload().getAndAcknowledgePayload(); if (result != null) { results.add(result); From 3c4c2e0f20db6fe0414699313a3fa4d529572f3c Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 10 Nov 2021 22:49:06 -0500 Subject: [PATCH 105/218] Updated executor to indicate when all tasks are generated, and to disallow asynchronous create task execution. Also updated executor tests to be more deterministic, and to use the hazelcast cache instead of caffeine. --- .../microservice/query/DefaultQueryParameters.java | 1 - .../datawave/microservice/query/QueryParameters.java | 2 -- .../microservice/query/config/QueryProperties.java | 10 ++++++++++ .../datawave/microservice/query/runner/NextCall.java | 6 +++--- .../microservice/query/QueryServiceNextTest.java | 7 +++++-- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java index 2c31d178..98f212a9 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java @@ -12,7 +12,6 @@ import java.util.Arrays; import java.util.Calendar; import java.util.Date; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParameters.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParameters.java index a9b95fed..93bfec13 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParameters.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParameters.java @@ -3,8 +3,6 @@ import datawave.validation.ParameterValidator; import org.springframework.util.MultiValueMap; -import java.util.Date; - import java.util.Date; import java.util.List; import java.util.Map; diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java index db115471..79847694 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java @@ -30,6 +30,8 @@ public class QueryProperties { @NotNull private TimeUnit lockLeaseTimeUnit = TimeUnit.MILLISECONDS; @NotEmpty + private String queryServiceName = "query"; + @NotEmpty private String executorServiceName = "executor"; // These are the only parameters that can be updated for a running query private List updatableParams = Arrays.asList(QUERY_EXPIRATION, QUERY_PAGESIZE, QUERY_PAGETIMEOUT, QUERY_MAX_RESULTS_OVERRIDE); @@ -90,6 +92,14 @@ public QueryProperties setLockLeaseTimeUnit(TimeUnit lockLeaseTimeUnit) { return this; } + public String getQueryServiceName() { + return queryServiceName; + } + + public void setQueryServiceName(String queryServiceName) { + this.queryServiceName = queryServiceName; + } + public String getExecutorServiceName() { return executorServiceName; } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 46794ce9..d958f508 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -158,14 +158,14 @@ private boolean isFinished(String queryId) throws QueryException { // 1) have we hit the user's results-per-page limit? if (results.size() >= userResultsPerPage) { - log.info("Query [{}]: user requested max page size has been reached, aborting next call", queryId); + log.info("Query [{}]: user requested max page size [{}] has been reached, aborting next call", queryId, userResultsPerPage); finished = true; } // 2) have we hit the query logic's results-per-page limit? if (!finished && logicResultsPerPage > 0 && results.size() >= logicResultsPerPage) { - log.info("Query [{}]: query logic max page size has been reached, aborting next call", queryId); + log.info("Query [{}]: query logic max page size [{}] has been reached, aborting next call", queryId, logicResultsPerPage); finished = true; } @@ -191,7 +191,7 @@ private boolean isFinished(String queryId) throws QueryException { } // 5) have we retrieved all of the results? - if (!finished && !getTaskStates().hasUnfinishedTasks()) { + if (!finished && !getTaskStates().isCreatingTasks() && !getTaskStates().hasUnfinishedTasks()) { // if all tasks have completed (or failed), start keeping track of the result count if (tasksFinished) { // if we stop getting results, we are done diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java index b0d4aef4..3dfbdd9a 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java @@ -438,9 +438,12 @@ public void testNextSuccess_noResults() throws Exception { // create a valid query String queryId = createQuery(authUser, createParams()); - // remove the task states to make it appear that the executor has finished + // mark the task states as complete, and mark task creation as complete to make it appear that the executor has finished TaskStates taskStates = queryStorageCache.getTaskStates(queryId); - taskStates.getTaskStates().remove(TaskStates.TASK_STATE.READY); + for (int i = 0; i < taskStates.getNextTaskId(); i++) { + taskStates.setState(i, TaskStates.TASK_STATE.COMPLETED); + } + taskStates.setCreatingTasks(false); queryStorageCache.updateTaskStates(taskStates); // make the next call asynchronously From 74d29d784cfa097dfb2fc66169e090857a7aa060 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 12 Nov 2021 15:30:41 -0500 Subject: [PATCH 106/218] Fixed a bug introduced by the last change which caused multiple executors to process a task. Moved the query status update util to the query starter so it can be used by all services. --- .../query/QueryManagementService.java | 23 +++--- .../query/status/QueryStatusUpdateHelper.java | 80 ------------------- .../query/status/StatusUpdater.java | 13 --- 3 files changed, 13 insertions(+), 103 deletions(-) delete mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java delete mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/StatusUpdater.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index b0c5952d..8bc704b8 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -10,11 +10,11 @@ import datawave.microservice.query.remote.QueryRequest; import datawave.microservice.query.remote.QueryRequestHandler; import datawave.microservice.query.runner.NextCall; -import datawave.microservice.query.status.QueryStatusUpdateHelper; import datawave.microservice.query.storage.QueryQueueManager; import datawave.microservice.query.storage.QueryStatus; import datawave.microservice.query.storage.QueryStorageCache; import datawave.microservice.query.storage.TaskKey; +import datawave.microservice.query.util.QueryStatusUpdateUtil; import datawave.microservice.query.util.QueryUtil; import datawave.microservice.querymetric.BaseQueryMetric; import datawave.microservice.querymetric.QueryMetricClient; @@ -110,7 +110,7 @@ public class QueryManagementService implements QueryRequestHandler { private final AuditClient auditClient; private final ThreadPoolTaskExecutor nextCallExecutor; - private final QueryStatusUpdateHelper queryStatusUpdateHelper; + private final QueryStatusUpdateUtil queryStatusUpdateUtil; private final MultiValueMap nextCallMap = new LinkedMultiValueMap<>(); private final String selfDestination; @@ -134,7 +134,7 @@ public QueryManagementService(QueryProperties queryProperties, ApplicationEventP this.queryQueueManager = queryQueueManager; this.auditClient = auditClient; this.nextCallExecutor = nextCallExecutor; - this.queryStatusUpdateHelper = new QueryStatusUpdateHelper(this.queryProperties, this.queryStorageCache); + this.queryStatusUpdateUtil = new QueryStatusUpdateUtil(this.queryProperties, this.queryStorageCache); this.selfDestination = getSelfDestination(); } @@ -625,7 +625,7 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th private BaseQueryResponse next(String queryId, Collection userRoles) throws InterruptedException, QueryException { // before we spin up a separate thread, make sure we are allowed to call next boolean success = false; - QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(queryId, queryStatusUpdateHelper::claimNextCall); + QueryStatus queryStatus = queryStatusUpdateUtil.lockedUpdate(queryId, queryStatusUpdateUtil::claimNextCall); try { // publish a next event to the executor pool publishNextEvent(queryId, queryStatus.getQueryKey().getQueryPool()); @@ -664,8 +664,8 @@ private BaseQueryResponse next(String queryId, Collection userRoles) thr BaseQueryResponse response = queryLogic.getTransformer(queryStatus.getQuery()).createResponse(resultsPage); // after all of our work is done, perform our final query status update for this next call - queryStatus = queryStatusUpdateHelper.lockedUpdate(queryId, status -> { - queryStatusUpdateHelper.releaseNextCall(status, queryQueueManager); + queryStatus = queryStatusUpdateUtil.lockedUpdate(queryId, status -> { + queryStatusUpdateUtil.releaseNextCall(status, queryQueueManager); status.setLastPageNumber(status.getLastPageNumber() + 1); status.setNumResultsReturned(status.getNumResultsReturned() + resultsPage.getResults().size()); }); @@ -678,10 +678,13 @@ private BaseQueryResponse next(String queryId, Collection userRoles) thr return response; } else { if (nextCall.isCanceled()) { + log.debug("Query [{}]: Canceled while handling next call", queryId); throw new QueryCanceledQueryException(DatawaveErrorCode.QUERY_CANCELED, MessageFormat.format("{0} canceled;", queryId)); } else if (baseQueryMetric.getLifecycle() == BaseQueryMetric.Lifecycle.NEXTTIMEOUT) { + log.debug("Query [{}]: Timed out during next call", queryId); throw new TimeoutQueryException(DatawaveErrorCode.QUERY_TIMEOUT, MessageFormat.format("{0} timed out.", queryId)); } else { + log.debug("Query [{}]: No results found for next call - closing query", queryId); // if there are no results, and we didn't timeout, close the query close(queryId); throw new NoResultsQueryException(DatawaveErrorCode.NO_QUERY_RESULTS_FOUND, MessageFormat.format("{0}", queryId)); @@ -702,7 +705,7 @@ private BaseQueryResponse next(String queryId, Collection userRoles) thr } finally { // update query status if we failed if (!success) { - queryStatusUpdateHelper.lockedUpdate(queryId, status -> queryStatusUpdateHelper.releaseNextCall(status, queryQueueManager)); + queryStatusUpdateUtil.lockedUpdate(queryId, status -> queryStatusUpdateUtil.releaseNextCall(status, queryQueueManager)); } } } @@ -889,7 +892,7 @@ public void cancel(String queryId, boolean publishEvent) throws InterruptedExcep if (publishEvent) { // only the initial event publisher should update the status - QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(queryId, status -> { + QueryStatus queryStatus = queryStatusUpdateUtil.lockedUpdate(queryId, status -> { // update query state to CANCELED status.setQueryState(CANCELED); }); @@ -1090,7 +1093,7 @@ private VoidResponse close(String queryId, ProxiedUserDetails currentUser, boole * if the cancel call is interrupted */ public void close(String queryId) throws InterruptedException, QueryException { - QueryStatus queryStatus = queryStatusUpdateHelper.lockedUpdate(queryId, status -> { + QueryStatus queryStatus = queryStatusUpdateUtil.lockedUpdate(queryId, status -> { // update query state to CLOSED status.setQueryState(CLOSED); }); @@ -1420,7 +1423,7 @@ public GenericResponse update(String queryId, MultiValueMap status.setQuery(query)); + queryStatusUpdateUtil.lockedUpdate(queryId, status -> status.setQuery(query)); } } else { throw new BadRequestQueryException("Cannot update the following parameters for a running query: " + String.join(", ", unsafeParams), diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java deleted file mode 100644 index 1a19d91f..00000000 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/QueryStatusUpdateHelper.java +++ /dev/null @@ -1,80 +0,0 @@ -package datawave.microservice.query.status; - -import datawave.microservice.query.config.QueryProperties; -import datawave.microservice.query.storage.QueryQueueManager; -import datawave.microservice.query.storage.QueryStatus; -import datawave.microservice.query.storage.QueryStorageCache; -import datawave.microservice.query.storage.QueryStorageLock; -import datawave.webservice.query.exception.DatawaveErrorCode; -import datawave.webservice.query.exception.NotFoundQueryException; -import datawave.webservice.query.exception.QueryException; - -import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CREATED; - -public class QueryStatusUpdateHelper { - - private final QueryProperties queryProperties; - private final QueryStorageCache queryStorageCache; - - public QueryStatusUpdateHelper(QueryProperties queryProperties, QueryStorageCache queryStorageCache) { - this.queryProperties = queryProperties; - this.queryStorageCache = queryStorageCache; - } - - public void claimNextCall(QueryStatus queryStatus) throws QueryException { - // we can only call next on a created query - if (queryStatus.getQueryState() == CREATED) { - // increment the concurrent next count - if (queryStatus.getActiveNextCalls() < queryProperties.getNextCall().getConcurrency()) { - queryStatus.setActiveNextCalls(queryStatus.getActiveNextCalls() + 1); - - // update the last used datetime for the query - queryStatus.setLastUsedMillis(System.currentTimeMillis()); - } else { - throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, - "Concurrent next call limit reached: " + queryProperties.getNextCall().getConcurrency()); - } - } else { - throw new QueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH, "Unable to find query status in cache with CREATED state."); - } - } - - public void releaseNextCall(QueryStatus queryStatus, QueryQueueManager queryQueueManager) throws QueryException { - // decrement the concurrent next count - if (queryStatus.getActiveNextCalls() > 0) { - queryStatus.setActiveNextCalls(queryStatus.getActiveNextCalls() - 1); - - // update the last used datetime for the query - queryStatus.setLastUsedMillis(System.currentTimeMillis()); - - // TODO: We should add a 'queueExists' call to determine whether this needs to be run - // if by the end of the call the query is no longer running, delete the results queue - if (!queryStatus.isRunning()) { - queryQueueManager.deleteQueue(queryStatus.getQueryKey().getQueryId()); - } - } else { - throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Concurrent next count can't be decremented: " + queryStatus.getActiveNextCalls()); - } - } - - public QueryStatus lockedUpdate(String queryUUID, StatusUpdater updater) throws QueryException, InterruptedException { - QueryStatus queryStatus = null; - QueryStorageLock statusLock = queryStorageCache.getQueryStatusLock(queryUUID); - if (statusLock.tryLock(queryProperties.getLockWaitTimeMillis(), queryProperties.getLockLeaseTimeMillis())) { - try { - queryStatus = queryStorageCache.getQueryStatus(queryUUID); - if (queryStatus != null) { - updater.apply(queryStatus); - queryStorageCache.updateQueryStatus(queryStatus); - } else { - throw new NotFoundQueryException(DatawaveErrorCode.NO_QUERY_OBJECT_MATCH, "Unable to find query status in cache."); - } - } finally { - statusLock.unlock(); - } - } else { - updater.onLockFailed(); - } - return queryStatus; - } -} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/StatusUpdater.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/StatusUpdater.java deleted file mode 100644 index 29c3a30e..00000000 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/status/StatusUpdater.java +++ /dev/null @@ -1,13 +0,0 @@ -package datawave.microservice.query.status; - -import datawave.microservice.query.storage.QueryStatus; -import datawave.webservice.query.exception.DatawaveErrorCode; -import datawave.webservice.query.exception.QueryException; - -public interface StatusUpdater { - void apply(QueryStatus queryStatus) throws QueryException; - - default void onLockFailed() throws QueryException { - throw new QueryException(DatawaveErrorCode.QUERY_LOCKED_ERROR, "Unable to acquire lock on query"); - } -} From 327c0af6145c8cc71877729ebff9166b8746dde6 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 19 Nov 2021 15:49:40 -0500 Subject: [PATCH 107/218] Reworked some of the messaging logic in order to ensure that the correct number of results are consistently returned. Removed a bunch of code and configuration which didn't seem to be used anymore. --- .../query/config/NextCallProperties.java | 24 ++++ .../java/datawave/microservice/lock/Lock.java | 17 --- .../microservice/lock/LockManager.java | 8 -- .../datawave/microservice/lock/Semaphore.java | 27 ---- .../distributed/hazelcast/HazelcastLock.java | 44 ------ .../hazelcast/HazelcastLockManager.java | 37 ----- .../hazelcast/HazelcastSemaphore.java | 69 ---------- .../distributed/zookeeper/ZookeeperLock.java | 46 ------- .../zookeeper/ZookeeperLockManager.java | 35 ----- .../zookeeper/ZookeeperSemaphore.java | 90 ------------- .../microservice/lock/local/LocalLock.java | 47 ------- .../lock/local/LocalLockManager.java | 23 ---- .../lock/local/LocalSemaphore.java | 71 ---------- .../query/QueryManagementService.java | 20 +-- .../query/monitor/MonitorTask.java | 8 +- .../query/monitor/QueryMonitor.java | 10 +- .../microservice/query/runner/NextCall.java | 126 +++++++++++++----- .../query/AbstractQueryServiceTest.java | 8 +- .../query/QueryServiceCancelTest.java | 3 - .../query/QueryServiceCloseTest.java | 6 - .../query/QueryServiceCreateTest.java | 3 - .../query/QueryServiceNextTest.java | 2 +- .../src/test/resources/config/application.yml | 5 - 23 files changed, 141 insertions(+), 588 deletions(-) delete mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Lock.java delete mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/LockManager.java delete mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Semaphore.java delete mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLock.java delete mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLockManager.java delete mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastSemaphore.java delete mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLock.java delete mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLockManager.java delete mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperSemaphore.java delete mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLock.java delete mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLockManager.java delete mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalSemaphore.java diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/NextCallProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/NextCallProperties.java index 53262c6b..2ce444c2 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/NextCallProperties.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/NextCallProperties.java @@ -19,6 +19,10 @@ public class NextCallProperties { private long statusUpdateInterval = TimeUnit.SECONDS.toMillis(6); @NotNull private TimeUnit statusUpdateTimeUnit = TimeUnit.MILLISECONDS; + @PositiveOrZero + private long maxResultsTimeout = 5l; + @NotNull + private TimeUnit maxResultsTimeUnit = TimeUnit.SECONDS; private ThreadPoolTaskExecutorProperties executor = new ThreadPoolTaskExecutorProperties(10, 100, 100, "nextCall-"); @@ -70,6 +74,26 @@ public void setStatusUpdateTimeUnit(TimeUnit statusUpdateTimeUnit) { this.statusUpdateTimeUnit = statusUpdateTimeUnit; } + public long getMaxResultsTimeout() { + return maxResultsTimeout; + } + + public long getMaxResultsTimeoutMillis() { + return maxResultsTimeUnit.toMillis(maxResultsTimeout); + } + + public void setMaxResultsTimeout(long maxResultsTimeout) { + this.maxResultsTimeout = maxResultsTimeout; + } + + public TimeUnit getMaxResultsTimeUnit() { + return maxResultsTimeUnit; + } + + public void setMaxResultsTimeUnit(TimeUnit maxResultsTimeUnit) { + this.maxResultsTimeUnit = maxResultsTimeUnit; + } + public ThreadPoolTaskExecutorProperties getExecutor() { return executor; } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Lock.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Lock.java deleted file mode 100644 index 96e1127c..00000000 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Lock.java +++ /dev/null @@ -1,17 +0,0 @@ -package datawave.microservice.lock; - -import java.util.concurrent.TimeUnit; - -public interface Lock { - String getName(); - - void lock() throws Exception; - - void lockInterruptibly() throws InterruptedException; - - boolean tryLock(); - - boolean tryLock(long time, TimeUnit unit) throws Exception; - - void unlock() throws Exception; -} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/LockManager.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/LockManager.java deleted file mode 100644 index 45326534..00000000 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/LockManager.java +++ /dev/null @@ -1,8 +0,0 @@ -package datawave.microservice.lock; - -public interface LockManager { - - Semaphore getSemaphore(String name, int permits) throws Exception; - - Lock getLock(String name); -} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Semaphore.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Semaphore.java deleted file mode 100644 index 8b847051..00000000 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/Semaphore.java +++ /dev/null @@ -1,27 +0,0 @@ -package datawave.microservice.lock; - -import java.util.concurrent.TimeUnit; - -public interface Semaphore { - String getName(); - - void acquire() throws Exception; - - void acquire(int permits) throws Exception; - - int availablePermits() throws Exception; - - int drainPermits() throws Exception; - - void release(); - - void release(int permits); - - boolean tryAcquire(); - - boolean tryAcquire(int permits); - - boolean tryAcquire(long timeout, TimeUnit unit) throws InterruptedException; - - boolean tryAcquire(int permits, long timeout, TimeUnit unit) throws InterruptedException; -} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLock.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLock.java deleted file mode 100644 index c9735de3..00000000 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLock.java +++ /dev/null @@ -1,44 +0,0 @@ -package datawave.microservice.lock.distributed.hazelcast; - -import com.hazelcast.cp.lock.FencedLock; -import datawave.microservice.lock.Lock; - -import java.util.concurrent.TimeUnit; - -public class HazelcastLock implements Lock { - private final FencedLock lock; - - HazelcastLock(FencedLock lock) { - this.lock = lock; - } - - @Override - public String getName() { - return lock.getName(); - } - - @Override - public void lock() { - lock.lock(); - } - - @Override - public void lockInterruptibly() throws InterruptedException { - lock.lockInterruptibly(); - } - - @Override - public boolean tryLock() { - return lock.tryLock(); - } - - @Override - public boolean tryLock(long time, TimeUnit unit) throws InterruptedException { - return lock.tryLock(time, unit); - } - - @Override - public void unlock() { - lock.unlock(); - } -} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLockManager.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLockManager.java deleted file mode 100644 index da3fe0f1..00000000 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastLockManager.java +++ /dev/null @@ -1,37 +0,0 @@ -package datawave.microservice.lock.distributed.hazelcast; - -import com.hazelcast.core.HazelcastInstance; -import com.hazelcast.cp.CPSubsystem; -import com.hazelcast.cp.ISemaphore; -import datawave.microservice.lock.Lock; -import datawave.microservice.lock.LockManager; -import datawave.microservice.lock.Semaphore; -import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; - -@Component("distributedLockManager") -@ConditionalOnBean(HazelcastInstance.class) -@ConditionalOnProperty(name = "datawave.lock.type", havingValue = "hazelcast") -public class HazelcastLockManager implements LockManager { - - private CPSubsystem cpSubsystem; - - public HazelcastLockManager(HazelcastInstance hazelcastInstance) { - this.cpSubsystem = hazelcastInstance.getCPSubsystem(); - } - - @Override - public Semaphore getSemaphore(String name, int permits) throws Exception { - ISemaphore iSemaphore = cpSubsystem.getSemaphore(name); - if (!iSemaphore.init(permits)) { - throw new Exception("Unable to initialize ISemaphore"); - } - return new HazelcastSemaphore(iSemaphore); - } - - @Override - public Lock getLock(String name) { - return new HazelcastLock(cpSubsystem.getLock(name)); - } -} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastSemaphore.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastSemaphore.java deleted file mode 100644 index 4a1a30df..00000000 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/hazelcast/HazelcastSemaphore.java +++ /dev/null @@ -1,69 +0,0 @@ -package datawave.microservice.lock.distributed.hazelcast; - -import com.hazelcast.cp.ISemaphore; -import datawave.microservice.lock.Semaphore; - -import java.util.concurrent.TimeUnit; - -public class HazelcastSemaphore implements Semaphore { - private final ISemaphore semaphore; - - HazelcastSemaphore(ISemaphore semaphore) { - this.semaphore = semaphore; - } - - @Override - public String getName() { - return semaphore.getName(); - } - - @Override - public void acquire() throws InterruptedException { - semaphore.acquire(); - } - - @Override - public void acquire(int permits) throws InterruptedException { - semaphore.acquire(permits); - } - - @Override - public int availablePermits() { - return semaphore.availablePermits(); - } - - @Override - public int drainPermits() { - return semaphore.drainPermits(); - } - - @Override - public void release() { - semaphore.release(); - } - - @Override - public void release(int permits) { - semaphore.release(permits); - } - - @Override - public boolean tryAcquire() { - return semaphore.tryAcquire(); - } - - @Override - public boolean tryAcquire(int permits) { - return semaphore.tryAcquire(permits); - } - - @Override - public boolean tryAcquire(long timeout, TimeUnit unit) throws InterruptedException { - return semaphore.tryAcquire(timeout, unit); - } - - @Override - public boolean tryAcquire(int permits, long timeout, TimeUnit unit) throws InterruptedException { - return semaphore.tryAcquire(permits, timeout, unit); - } -} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLock.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLock.java deleted file mode 100644 index d1a37ff9..00000000 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLock.java +++ /dev/null @@ -1,46 +0,0 @@ -package datawave.microservice.lock.distributed.zookeeper; - -import datawave.microservice.lock.Lock; -import org.apache.curator.framework.recipes.locks.InterProcessMutex; - -import java.util.concurrent.TimeUnit; - -public class ZookeeperLock implements Lock { - private final String path; - private final InterProcessMutex lock; - - ZookeeperLock(String path, InterProcessMutex lock) { - this.path = path; - this.lock = lock; - } - - @Override - public String getName() { - return path; - } - - @Override - public void lock() throws Exception { - lock.acquire(); - } - - @Override - public void lockInterruptibly() throws InterruptedException { - throw new UnsupportedOperationException("Unable to lockInterruptibly for ZookeeperLock"); - } - - @Override - public boolean tryLock() { - throw new UnsupportedOperationException("Unable to tryLock for ZookeeperLock"); - } - - @Override - public boolean tryLock(long time, TimeUnit unit) throws Exception { - return lock.acquire(time, unit); - } - - @Override - public void unlock() throws Exception { - lock.release(); - } -} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLockManager.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLockManager.java deleted file mode 100644 index a5bda2ae..00000000 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperLockManager.java +++ /dev/null @@ -1,35 +0,0 @@ -package datawave.microservice.lock.distributed.zookeeper; - -import datawave.microservice.lock.Lock; -import datawave.microservice.lock.LockManager; -import datawave.microservice.lock.Semaphore; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.locks.InterProcessMutex; -import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreV2; -import org.apache.curator.framework.recipes.shared.SharedCount; -import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; - -@Component("distributedLockManager") -@ConditionalOnBean(CuratorFramework.class) -@ConditionalOnProperty(name = "datawave.lock.type", havingValue = "zookeeper") -public class ZookeeperLockManager implements LockManager { - - final private CuratorFramework curatorFramework; - - public ZookeeperLockManager(CuratorFramework curatorFramework) { - this.curatorFramework = curatorFramework; - } - - @Override - public Semaphore getSemaphore(String name, int permits) throws Exception { - return new ZookeeperSemaphore(name, new InterProcessSemaphoreV2(curatorFramework, name, new SharedCount(curatorFramework, name + "/permits", permits)), - curatorFramework); - } - - @Override - public Lock getLock(String name) { - return new ZookeeperLock(name, new InterProcessMutex(curatorFramework, name)); - } -} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperSemaphore.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperSemaphore.java deleted file mode 100644 index 36708df3..00000000 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/distributed/zookeeper/ZookeeperSemaphore.java +++ /dev/null @@ -1,90 +0,0 @@ -package datawave.microservice.lock.distributed.zookeeper; - -import datawave.microservice.lock.Semaphore; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreV2; -import org.apache.curator.framework.recipes.locks.Lease; - -import java.util.LinkedList; -import java.util.concurrent.TimeUnit; - -public class ZookeeperSemaphore implements Semaphore { - private final String path; - private final InterProcessSemaphoreV2 semaphore; - private final LinkedList leases = new LinkedList<>(); - private final CuratorFramework curatorFramework; - - ZookeeperSemaphore(String path, InterProcessSemaphoreV2 semaphore, CuratorFramework curatorFramework) { - this.path = path; - this.semaphore = semaphore; - this.curatorFramework = curatorFramework; - ; - } - - @Override - public String getName() { - return path; - } - - @Override - public void acquire() throws Exception { - synchronized (leases) { - leases.push(semaphore.acquire()); - } - } - - @Override - public void acquire(int permits) throws Exception { - synchronized (leases) { - semaphore.acquire(permits).forEach(leases::push); - } - } - - @Override - public int availablePermits() throws Exception { - return curatorFramework.getChildren().forPath(path).size(); - } - - @Override - public int drainPermits() throws Exception { - throw new UnsupportedOperationException("Unable to drainPermits for ZookeeperSemaphore"); - } - - @Override - public void release() { - synchronized (leases) { - if (leases.size() > 0) { - semaphore.returnLease(leases.pop()); - } - } - } - - @Override - public void release(int permits) { - synchronized (leases) { - while (permits-- > 0 && !leases.isEmpty()) { - semaphore.returnLease(leases.pop()); - } - } - } - - @Override - public boolean tryAcquire() { - throw new UnsupportedOperationException("Unable to tryAcquire for ZookeeperSemaphore"); - } - - @Override - public boolean tryAcquire(int permits) { - throw new UnsupportedOperationException("Unable to tryAcquire for ZookeeperSemaphore"); - } - - @Override - public boolean tryAcquire(long timeout, TimeUnit unit) throws InterruptedException { - throw new UnsupportedOperationException("Unable to tryAcquire for ZookeeperSemaphore"); - } - - @Override - public boolean tryAcquire(int permits, long timeout, TimeUnit unit) throws InterruptedException { - throw new UnsupportedOperationException("Unable to tryAcquire for ZookeeperSemaphore"); - } -} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLock.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLock.java deleted file mode 100644 index a7838409..00000000 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLock.java +++ /dev/null @@ -1,47 +0,0 @@ -package datawave.microservice.lock.local; - -import datawave.microservice.lock.Lock; - -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; - -public class LocalLock implements Lock { - - private final String name; - private final ReentrantLock lock; - - LocalLock(String name, ReentrantLock lock) { - this.name = name; - this.lock = lock; - } - - @Override - public String getName() { - return name; - } - - @Override - public void lock() { - lock.lock(); - } - - @Override - public void lockInterruptibly() throws InterruptedException { - lock.lockInterruptibly(); - } - - @Override - public boolean tryLock() { - return lock.tryLock(); - } - - @Override - public boolean tryLock(long time, TimeUnit unit) throws InterruptedException { - return lock.tryLock(time, unit); - } - - @Override - public void unlock() { - lock.unlock(); - } -} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLockManager.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLockManager.java deleted file mode 100644 index 67738a5d..00000000 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalLockManager.java +++ /dev/null @@ -1,23 +0,0 @@ -package datawave.microservice.lock.local; - -import datawave.microservice.lock.Lock; -import datawave.microservice.lock.LockManager; -import datawave.microservice.lock.Semaphore; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Component; - -import java.util.concurrent.locks.ReentrantLock; - -@Component("distributedLockManager") -@ConditionalOnProperty(name = "datawave.lock.type", havingValue = "local") -public class LocalLockManager implements LockManager { - @Override - public Semaphore getSemaphore(String name, int permits) { - return new LocalSemaphore(name, new java.util.concurrent.Semaphore(permits)); - } - - @Override - public Lock getLock(String name) { - return new LocalLock(name, new ReentrantLock()); - } -} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalSemaphore.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalSemaphore.java deleted file mode 100644 index e0abe842..00000000 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/lock/local/LocalSemaphore.java +++ /dev/null @@ -1,71 +0,0 @@ -package datawave.microservice.lock.local; - -import datawave.microservice.lock.Semaphore; - -import java.util.concurrent.TimeUnit; - -public class LocalSemaphore implements Semaphore { - - private final String name; - private final java.util.concurrent.Semaphore semaphore; - - LocalSemaphore(String name, java.util.concurrent.Semaphore semaphore) { - this.name = name; - this.semaphore = semaphore; - } - - @Override - public String getName() { - return name; - } - - @Override - public void acquire() throws InterruptedException { - semaphore.acquire(); - } - - @Override - public void acquire(int permits) throws InterruptedException { - semaphore.acquire(permits); - } - - @Override - public int availablePermits() { - return semaphore.availablePermits(); - } - - @Override - public int drainPermits() { - return semaphore.drainPermits(); - } - - @Override - public void release() { - semaphore.release(); - } - - @Override - public void release(int permits) { - semaphore.release(); - } - - @Override - public boolean tryAcquire() { - return semaphore.tryAcquire(); - } - - @Override - public boolean tryAcquire(int permits) { - return semaphore.tryAcquire(permits); - } - - @Override - public boolean tryAcquire(long timeout, TimeUnit unit) throws InterruptedException { - return semaphore.tryAcquire(timeout, unit); - } - - @Override - public boolean tryAcquire(int permits, long timeout, TimeUnit unit) throws InterruptedException { - return semaphore.tryAcquire(permits, timeout, unit); - } -} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 8bc704b8..160c59d1 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -7,10 +7,10 @@ import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.authorization.util.AuthorizationsUtil; import datawave.microservice.query.config.QueryProperties; +import datawave.microservice.query.messaging.QueryResultsManager; import datawave.microservice.query.remote.QueryRequest; import datawave.microservice.query.remote.QueryRequestHandler; import datawave.microservice.query.runner.NextCall; -import datawave.microservice.query.storage.QueryQueueManager; import datawave.microservice.query.storage.QueryStatus; import datawave.microservice.query.storage.QueryStorageCache; import datawave.microservice.query.storage.TaskKey; @@ -106,7 +106,7 @@ public class QueryManagementService implements QueryRequestHandler { private final QueryMetricClient queryMetricClient; private final ResponseObjectFactory responseObjectFactory; private final QueryStorageCache queryStorageCache; - private final QueryQueueManager queryQueueManager; + private final QueryResultsManager queryResultsManager; private final AuditClient auditClient; private final ThreadPoolTaskExecutor nextCallExecutor; @@ -120,7 +120,7 @@ public class QueryManagementService implements QueryRequestHandler { public QueryManagementService(QueryProperties queryProperties, ApplicationEventPublisher eventPublisher, BusProperties busProperties, QueryParameters queryParameters, SecurityMarking securityMarking, BaseQueryMetric baseQueryMetric, QueryLogicFactory queryLogicFactory, QueryMetricClient queryMetricClient, ResponseObjectFactory responseObjectFactory, QueryStorageCache queryStorageCache, - QueryQueueManager queryQueueManager, AuditClient auditClient, ThreadPoolTaskExecutor nextCallExecutor) { + QueryResultsManager queryResultsManager, AuditClient auditClient, ThreadPoolTaskExecutor nextCallExecutor) { this.queryProperties = queryProperties; this.eventPublisher = eventPublisher; this.busProperties = busProperties; @@ -131,7 +131,7 @@ public QueryManagementService(QueryProperties queryProperties, ApplicationEventP this.queryMetricClient = queryMetricClient; this.responseObjectFactory = responseObjectFactory; this.queryStorageCache = queryStorageCache; - this.queryQueueManager = queryQueueManager; + this.queryResultsManager = queryResultsManager; this.auditClient = auditClient; this.nextCallExecutor = nextCallExecutor; this.queryStatusUpdateUtil = new QueryStatusUpdateUtil(this.queryProperties, this.queryStorageCache); @@ -402,7 +402,6 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p taskKey = queryStorageCache.createQuery( getPoolName(), query, - busProperties.getId(), downgradedAuthorizations, getMaxConcurrentTasks(queryLogic)); // @formatter:on @@ -641,8 +640,9 @@ private BaseQueryResponse next(String queryId, Collection userRoles) thr // @formatter:off final NextCall nextCall = new NextCall.Builder() .setQueryProperties(queryProperties) - .setQueryQueueManager(queryQueueManager) + .setResultsQueueManager(queryResultsManager) .setQueryStorageCache(queryStorageCache) + .setQueryStatusUpdateUtil(queryStatusUpdateUtil) .setQueryId(queryId) .setQueryLogic(queryLogic) .build(); @@ -665,7 +665,7 @@ private BaseQueryResponse next(String queryId, Collection userRoles) thr // after all of our work is done, perform our final query status update for this next call queryStatus = queryStatusUpdateUtil.lockedUpdate(queryId, status -> { - queryStatusUpdateUtil.releaseNextCall(status, queryQueueManager); + queryStatusUpdateUtil.releaseNextCall(status, queryResultsManager); status.setLastPageNumber(status.getLastPageNumber() + 1); status.setNumResultsReturned(status.getNumResultsReturned() + resultsPage.getResults().size()); }); @@ -705,7 +705,7 @@ private BaseQueryResponse next(String queryId, Collection userRoles) thr } finally { // update query status if we failed if (!success) { - queryStatusUpdateUtil.lockedUpdate(queryId, status -> queryStatusUpdateUtil.releaseNextCall(status, queryQueueManager)); + queryStatusUpdateUtil.lockedUpdate(queryId, status -> queryStatusUpdateUtil.releaseNextCall(status, queryResultsManager)); } } } @@ -898,7 +898,7 @@ public void cancel(String queryId, boolean publishEvent) throws InterruptedExcep }); // delete the results queue - queryQueueManager.deleteQueue(queryId); + queryResultsManager.deleteQuery(queryId); QueryRequest cancelRequest = QueryRequest.cancel(queryId); @@ -1100,7 +1100,7 @@ public void close(String queryId) throws InterruptedException, QueryException { // if the query has no active next calls, delete the results queue if (queryStatus.getActiveNextCalls() == 0) { - queryQueueManager.deleteQueue(queryId); + queryResultsManager.deleteQuery(queryId); } // publish a close event to the executor pool diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java index 411e6f1e..b1c8588d 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java @@ -2,10 +2,10 @@ import datawave.microservice.query.QueryManagementService; import datawave.microservice.query.config.QueryExpirationProperties; +import datawave.microservice.query.messaging.QueryResultsManager; import datawave.microservice.query.monitor.cache.MonitorStatus; import datawave.microservice.query.monitor.cache.MonitorStatusCache; import datawave.microservice.query.monitor.config.MonitorProperties; -import datawave.microservice.query.storage.QueryQueueManager; import datawave.microservice.query.storage.QueryStatus; import datawave.microservice.query.storage.QueryStorageCache; import datawave.webservice.query.exception.QueryException; @@ -22,11 +22,11 @@ public class MonitorTask implements Callable { private final QueryExpirationProperties expirationProperties; private final MonitorStatusCache monitorStatusCache; private final QueryStorageCache queryStorageCache; - private final QueryQueueManager queryQueueManager; + private final QueryResultsManager queryQueueManager; private final QueryManagementService queryManagementService; public MonitorTask(MonitorProperties monitorProperties, QueryExpirationProperties expirationProperties, MonitorStatusCache monitorStatusCache, - QueryStorageCache queryStorageCache, QueryQueueManager queryQueueManager, QueryManagementService queryManagementService) { + QueryStorageCache queryStorageCache, QueryResultsManager queryQueueManager, QueryManagementService queryManagementService) { this.monitorProperties = monitorProperties; this.expirationProperties = expirationProperties; this.monitorStatusCache = monitorStatusCache; @@ -76,7 +76,7 @@ private void monitor(long currentTimeMillis) { // delete the results queue if it exists else { // TODO: add in a check to see if the queue exists first - queryQueueManager.deleteQueue(queryId); + queryQueueManager.deleteQuery(queryId); } } // if the query is running diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java index b283fe79..8287c665 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java @@ -3,9 +3,9 @@ import datawave.microservice.query.QueryManagementService; import datawave.microservice.query.config.QueryExpirationProperties; import datawave.microservice.query.config.QueryProperties; +import datawave.microservice.query.messaging.QueryResultsManager; import datawave.microservice.query.monitor.cache.MonitorStatusCache; import datawave.microservice.query.monitor.config.MonitorProperties; -import datawave.microservice.query.storage.QueryQueueManager; import datawave.microservice.query.storage.QueryStorageCache; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,7 +27,7 @@ public class QueryMonitor { private final QueryExpirationProperties expirationProperties; private final MonitorStatusCache monitorStatusCache; private final QueryStorageCache queryStorageCache; - private final QueryQueueManager queryQueueManager; + private final QueryResultsManager queryResultsManager; private final QueryManagementService queryManagementService; private final ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -35,12 +35,12 @@ public class QueryMonitor { private Future taskFuture; public QueryMonitor(MonitorProperties monitorProperties, QueryProperties queryProperties, MonitorStatusCache monitorStatusCache, - QueryStorageCache queryStorageCache, QueryQueueManager queryQueueManager, QueryManagementService queryManagementService) { + QueryStorageCache queryStorageCache, QueryResultsManager queryResultsManager, QueryManagementService queryManagementService) { this.monitorProperties = monitorProperties; this.expirationProperties = queryProperties.getExpiration(); this.monitorStatusCache = monitorStatusCache; this.queryStorageCache = queryStorageCache; - this.queryQueueManager = queryQueueManager; + this.queryResultsManager = queryResultsManager; this.queryManagementService = queryManagementService; } @@ -74,7 +74,7 @@ public void monitorTaskScheduler() { expirationProperties, monitorStatusCache, queryStorageCache, - queryQueueManager, + queryResultsManager, queryManagementService)); // @formatter:on } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index d958f508..e1050fcc 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -3,12 +3,13 @@ import datawave.microservice.query.config.NextCallProperties; import datawave.microservice.query.config.QueryExpirationProperties; import datawave.microservice.query.config.QueryProperties; -import datawave.microservice.query.storage.QueryQueueListener; -import datawave.microservice.query.storage.QueryQueueManager; +import datawave.microservice.query.messaging.QueryResultsListener; +import datawave.microservice.query.messaging.QueryResultsManager; +import datawave.microservice.query.messaging.Result; import datawave.microservice.query.storage.QueryStatus; import datawave.microservice.query.storage.QueryStorageCache; -import datawave.microservice.query.storage.Result; import datawave.microservice.query.storage.TaskStates; +import datawave.microservice.query.util.QueryStatusUpdateUtil; import datawave.microservice.querymetric.BaseQueryMetric; import datawave.microservice.querymetric.QueryMetric; import datawave.services.query.cache.ResultsPage; @@ -17,7 +18,6 @@ import datawave.webservice.query.exception.QueryException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.messaging.Message; import java.util.LinkedList; import java.util.List; @@ -26,13 +26,16 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import static datawave.microservice.query.messaging.AcknowledgementCallback.Status.ACK; + public class NextCall implements Callable> { private final Logger log = LoggerFactory.getLogger(this.getClass()); private final NextCallProperties nextCallProperties; - private final QueryQueueManager queryQueueManager; + private final QueryResultsManager queryResultsManager; private final QueryStorageCache queryStorageCache; private final String queryId; + private final QueryStatusUpdateUtil queryStatusUpdateUtil; private volatile boolean canceled = false; private volatile Future> future = null; @@ -60,16 +63,18 @@ public class NextCall implements Callable> { private QueryStatus queryStatus; private long lastTaskStatesUpdateTime = 0L; private TaskStates taskStates; - private boolean tasksFinished = false; - private long prevResultCount = 0L; + private long numResultsConsumed = 0L; + + private long hitMaxResultsTimeMillis = 0L; private BaseQueryMetric.Lifecycle lifecycle; private NextCall(Builder builder) { this.nextCallProperties = builder.nextCallProperties; - this.queryQueueManager = builder.queryQueueManager; + this.queryResultsManager = builder.queryResultsManager; this.queryStorageCache = builder.queryStorageCache; this.queryId = builder.queryId; + this.queryStatusUpdateUtil = builder.queryStatusUpdateUtil; QueryStatus status = getQueryStatus(); long pageTimeoutMillis = TimeUnit.MINUTES.toMillis(status.getQuery().getPageTimeout()); @@ -105,19 +110,22 @@ private NextCall(Builder builder) { public ResultsPage call() throws Exception { startTimeMillis = System.currentTimeMillis(); - QueryQueueListener resultListener = queryQueueManager.createListener(UUID.randomUUID().toString(), queryId); - try { + try (QueryResultsListener resultListener = queryResultsManager.createListener(UUID.randomUUID().toString(), queryId)) { // keep waiting for results until we're finished // Note: isFinished should be checked once per result while (!isFinished(queryId)) { - Message message = resultListener.receive(nextCallProperties.getResultPollIntervalMillis()); - if (message != null) { - Object result = message.getPayload().getAndAcknowledgePayload(); - if (result != null) { - results.add(result); + Result result = resultListener.receive(nextCallProperties.getResultPollInterval(), nextCallProperties.getResultPollTimeUnit()); + if (result != null) { + result.acknowledge(ACK); + + Object payload = result.getPayload(); + if (payload != null) { + results.add(payload); + + numResultsConsumed++; if (logicBytesPerPage > 0) { - pageSizeBytes += ObjectSizeOf.Sizer.getObjectSize(result); + pageSizeBytes += ObjectSizeOf.Sizer.getObjectSize(payload); } } else { log.debug("Null result encountered, no more results"); @@ -125,9 +133,6 @@ public ResultsPage call() throws Exception { } } } - } finally { - // stop the result listener - resultListener.stop(); } // update some values for metrics @@ -136,6 +141,9 @@ public ResultsPage call() throws Exception { lifecycle = BaseQueryMetric.Lifecycle.RESULTS; } + // update num results consumed for query status + updateNumResultsConsumed(); + return new ResultsPage<>(results, status); } @@ -147,7 +155,7 @@ public void updateQueryMetric(BaseQueryMetric baseQueryMetric) { private boolean isFinished(String queryId) throws QueryException { boolean finished = false; long callTimeMillis = System.currentTimeMillis() - startTimeMillis; - final QueryStatus queryStatus = getQueryStatus(); + QueryStatus queryStatus = getQueryStatus(); // if the query state is FAILED, throw an exception up to the query management service with the failure message if (queryStatus.getQueryState() == QueryStatus.QUERY_STATE.FAILED) { @@ -191,19 +199,53 @@ private boolean isFinished(String queryId) throws QueryException { } // 5) have we retrieved all of the results? - if (!finished && !getTaskStates().isCreatingTasks() && !getTaskStates().hasUnfinishedTasks()) { - // if all tasks have completed (or failed), start keeping track of the result count - if (tasksFinished) { - // if we stop getting results, we are done - if (prevResultCount == results.size()) { + if (!finished && !getTaskStates().isCreatingTasks() && !getTaskStates().hasUnfinishedTasks() + && queryResultsManager.getNumResultsRemaining(queryId) == 0) { + // update the number of results consumed + queryStatus = updateNumResultsConsumed(); + + // how many results do the query services think are left + long queryResultsRemaining = queryStatus.getNumResultsGenerated() - queryStatus.getNumResultsConsumed(); + + // check to see if the number of results consumed is >= to the number of results generated + if (queryResultsRemaining < 0) { + // TODO: Consider this as a potential backend-agnostic exit condition, but be mindful of the possibility of a next call failing before the + // consumed results can be returned + log.warn("Query [{}]: The number of results consumed [{}] exceeds the number of results generated [{}]", queryId, + queryStatus.getNumResultsConsumed(), queryStatus.getNumResultsGenerated()); + } + + // how many results does the broker think are left + long brokerResultsRemaining = queryResultsManager.getNumResultsRemaining(queryId); + + // if the broker thinks there are not results left, we may be done + if (brokerResultsRemaining == 0) { + + // if the query service thinks there are no results left, we are done + // we can have negative results remaining if we consumed duplicate records + if (queryResultsRemaining <= 0) { log.info("Query [{}]: all query tasks complete, and all results retrieved, aborting next call", queryId); status = ResultsPage.Status.PARTIAL; finished = true; } - } else { - tasksFinished = true; + // if the query services think there are results left, we may need to wait + // this can happen if messages are in flux with the message broker due to nacking + else { + // if we aren't in a max results waiting period, start waiting + if (hitMaxResultsTimeMillis == 0) { + hitMaxResultsTimeMillis = System.currentTimeMillis(); + } + // if we are finished waiting, we are done + else if (System.currentTimeMillis() >= (hitMaxResultsTimeMillis + nextCallProperties.getMaxResultsTimeoutMillis())) { + log.info("Query [{}]: all query tasks complete, but timed out waiting for all results to be retrieved, aborting next call", queryId); + + status = ResultsPage.Status.PARTIAL; + + finished = true; + } + } } } @@ -265,12 +307,24 @@ private boolean isFinished(String queryId) throws QueryException { finished = true; } - // save the previous result count - prevResultCount = results.size(); - return finished; } + private QueryStatus updateNumResultsConsumed() { + if (numResultsConsumed > 0) { + try { + queryStatus = queryStatusUpdateUtil.lockedUpdate(queryId, queryStatus1 -> { + queryStatus1.incrementNumResultsConsumed(numResultsConsumed); + numResultsConsumed = 0; + }); + lastQueryStatusUpdateTime = System.currentTimeMillis(); + } catch (Exception e) { + log.warn("Unable to update number of results consumed for query {}", queryId); + } + } + return queryStatus; + } + private boolean shortCircuitTimeout(long callTimeMillis) { boolean timeout = false; @@ -342,9 +396,10 @@ public void setFuture(Future> future) { public static class Builder { private NextCallProperties nextCallProperties; private QueryExpirationProperties expirationProperties; - private QueryQueueManager queryQueueManager; + private QueryResultsManager queryResultsManager; private QueryStorageCache queryStorageCache; private String queryId; + private QueryStatusUpdateUtil queryStatusUpdateUtil; private QueryLogic queryLogic; public Builder setQueryProperties(QueryProperties queryProperties) { @@ -363,8 +418,8 @@ public Builder setExpirationProperties(QueryExpirationProperties expirationPrope return this; } - public Builder setQueryQueueManager(QueryQueueManager queryQueueManager) { - this.queryQueueManager = queryQueueManager; + public Builder setResultsQueueManager(QueryResultsManager queryResultsManager) { + this.queryResultsManager = queryResultsManager; return this; } @@ -378,6 +433,11 @@ public Builder setQueryId(String queryId) { return this; } + public Builder setQueryStatusUpdateUtil(QueryStatusUpdateUtil queryStatusUpdateUtil) { + this.queryStatusUpdateUtil = queryStatusUpdateUtil; + return this; + } + public Builder setQueryLogic(QueryLogic queryLogic) { this.queryLogic = queryLogic; return this; diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java index 3cf6c882..343053f6 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java @@ -5,13 +5,13 @@ import datawave.microservice.authorization.jwt.JWTRestTemplate; import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.query.config.QueryProperties; +import datawave.microservice.query.messaging.Result; +import datawave.microservice.query.messaging.TestQueryResultsManager; import datawave.microservice.query.remote.QueryRequest; import datawave.microservice.query.storage.QueryStatus; import datawave.microservice.query.storage.QueryStorageCache; -import datawave.microservice.query.storage.Result; import datawave.microservice.query.storage.TaskKey; import datawave.microservice.query.storage.TaskStates; -import datawave.microservice.query.storage.queue.TestQueryQueueManager; import datawave.security.authorization.DatawaveUser; import datawave.security.authorization.SubjectIssuerDNPair; import datawave.webservice.query.Query; @@ -104,7 +104,7 @@ public abstract class AbstractQueryServiceTest { protected QueryStorageCache queryStorageCache; @Autowired - protected TestQueryQueueManager queryQueueManager; + protected TestQueryResultsManager queryQueueManager; @Autowired protected AuditClient auditClient; @@ -149,7 +149,7 @@ protected void publishEventsToQueue(String queryId, int numEvents, MultiValueMap } } event.setFields(fields); - queryQueueManager.sendMessage(queryId, new Result(Integer.toString(resultId), event)); + queryQueueManager.createPublisher(queryId).publish(new Result(Integer.toString(resultId), event)); } } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java index 8ca9f175..d03d7ea0 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java @@ -503,9 +503,6 @@ public void testAdminCancelAllFailure_notAdminUser() throws Exception { queryStatus); // @formatter:on - // verify that the result queue is still present - Assert.assertTrue(queryQueueManager.queueExists(queryStatus.getQueryKey().getQueryId())); - // verify that the query tasks are still present assertTasksCreated(queryStatus.getQueryKey().getQueryId()); } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java index 88e92730..38bf637a 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java @@ -142,9 +142,6 @@ public void testCloseSuccess_activeNextCall() throws Exception { queryStatus); // @formatter:on - // verify that the result queue is still present - Assert.assertTrue(queryQueueManager.queueExists(queryId)); - // send enough results to return a page // pump enough results into the queue to trigger a complete page int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); @@ -494,9 +491,6 @@ public void testAdminCloseAllFailure_notAdminUser() throws Exception { queryStatus); // @formatter:on - // verify that the result queue is still present - Assert.assertTrue(queryQueueManager.queueExists(queryStatus.getQueryKey().getQueryId())); - // verify that the query tasks are still present assertTasksCreated(queryStatus.getQueryKey().getQueryId()); } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java index 51d5ac0f..66274a98 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java @@ -114,9 +114,6 @@ public void testCreateSuccess() throws ParseException, IOException { // verify that an audit message was sent and the the audit id matches the query id assertAuditSent(queryId); - // verify that the results queue was created - Assert.assertTrue(queryQueueManager.queueExists(queryId)); - // verify that query tasks were created assertTasksCreated(queryId); } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java index 3dfbdd9a..bc3dd7df 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java @@ -232,7 +232,7 @@ public void testNextSuccess_cancelPartialResults() throws Exception { Future> nextFuture = nextQuery(authUser, queryId); // make sure all events were consumed before canceling - while (queryQueueManager.getQueueSize(queryId) != 0) { + while (queryQueueManager.getNumResultsRemaining(queryId) != 0) { Thread.sleep(100); } diff --git a/query-microservices/query-service/service/src/test/resources/config/application.yml b/query-microservices/query-service/service/src/test/resources/config/application.yml index afe8cd72..c029ae3f 100644 --- a/query-microservices/query-service/service/src/test/resources/config/application.yml +++ b/query-microservices/query-service/service/src/test/resources/config/application.yml @@ -97,11 +97,6 @@ warehouse: ivaratorCacheScanTimeoutMinutes: 60 modelName: 'DATAWAVE' -query: - storage: - syncStorage: true - sendNotifications: true - datawave: metadata: all-metadata-auths: From 3c8a3c89e3a6f9563bcdc542faa9eb0972767841 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Mon, 22 Nov 2021 16:37:22 -0500 Subject: [PATCH 108/218] Added plan and predict rest endpoints to the query service. Plan is fully implemented, predict is not. --- query-microservices/query-service/README.md | 4 +- .../query/DefaultQueryParameters.java | 34 +++ .../microservice/query/QueryParameters.java | 10 + .../microservice/query/QueryController.java | 16 ++ .../query/QueryManagementService.java | 255 ++++++++++++++---- .../query/AbstractQueryServiceTest.java | 17 +- .../query/QueryServicePlanTest.java | 113 ++++++++ 7 files changed, 393 insertions(+), 56 deletions(-) create mode 100644 query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java diff --git a/query-microservices/query-service/README.md b/query-microservices/query-service/README.md index da8598d4..b5f823cb 100644 --- a/query-microservices/query-service/README.md +++ b/query-microservices/query-service/README.md @@ -16,8 +16,8 @@ query capabilities. | ✓ | | | `GET` | /listQueryLogic | List QueryLogic types that are currently available | N/A | N/A | [QueryLogicResponse] | | ✓ | | | `POST` | /{queryLogic}/define | Define a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | | ✓ | | | `POST` | /{queryLogic}/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | -| | | | `POST` | /{queryLogic}/plan | Generate a query plan using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | -| | | | `POST` | /{queryLogic}/predict | Generate a query prediction using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| ✓ | | | `POST` | /{queryLogic}/plan | Generate a query plan using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | +| ✓ | | | `POST` | /{queryLogic}/predict | Generate a query prediction using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | | ✓ | | | `POST` | /{queryLogic}/async/create | Create a query using the specified query logic and params | [QueryLogicName] | [QueryParameters] | [GenericResponse] | | ✓ | | | `PUT` `POST` | /{id}/reset | Resets the specified query | [QueryId] | N/A | [VoidResponse]
[GenericResponse] | | ✓ | | | `POST` | /{queryLogic}/createAndNext | Create a query using the specified query logic and params, and get the first page | [QueryLogicName] | [QueryParameters] | [BaseQueryResponse] | diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java index 98f212a9..035caa50 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java @@ -38,6 +38,8 @@ public class DefaultQueryParameters implements QueryParameters { protected String pool; protected boolean isMaxConcurrentTasksOverridden; protected int maxConcurrentTasks; + protected boolean expandFields; + protected boolean expandValues; protected Map> requestHeaders; public DefaultQueryParameters() { @@ -125,6 +127,10 @@ public void validate(Map> parameters) throws IllegalArgument this.logicName = values.get(0); } else if (QUERY_POOL.equals(param)) { this.pool = values.get(0); + } else if (QUERY_PLAN_EXPAND_FIELDS.equals(param)) { + this.expandFields = Boolean.parseBoolean(values.get(0)); + } else if (QUERY_PLAN_EXPAND_VALUES.equals(param)) { + this.expandValues = Boolean.parseBoolean(values.get(0)); } else { throw new IllegalArgumentException("Unknown condition."); } @@ -165,6 +171,10 @@ public boolean equals(Object o) { return false; if (maxConcurrentTasks != that.maxConcurrentTasks) return false; + if (expandFields != that.expandFields) + return false; + if (expandValues != that.expandValues) + return false; if (trace != that.trace) return false; if (!auths.equals(that.auths)) @@ -205,6 +215,8 @@ public int hashCode() { if (isMaxConcurrentTasksOverridden) { result = 31 * result + maxConcurrentTasks; } + result = 31 * result + Boolean.hashCode(expandFields); + result = 31 * result + Boolean.hashCode(expandValues); result = 31 * result + auths.hashCode(); result = 31 * result + expirationDate.hashCode(); result = 31 * result + (trace ? 1 : 0); @@ -529,6 +541,26 @@ public boolean isMaxConcurrentTasksOverridden() { return isMaxConcurrentTasksOverridden; } + @Override + public boolean isExpandFields() { + return expandFields; + } + + @Override + public void setExpandFields(boolean expandFields) { + this.expandFields = expandFields; + } + + @Override + public boolean isExpandValues() { + return expandValues; + } + + @Override + public void setExpandValues(boolean expandValues) { + this.expandValues = expandValues; + } + @Override public Map> getRequestHeaders() { return requestHeaders; @@ -568,6 +600,8 @@ public void clear() { this.pool = null; this.isMaxConcurrentTasksOverridden = false; this.maxConcurrentTasks = 0; + this.expandFields = true; + this.expandValues = false; this.requestHeaders = null; } } diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParameters.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParameters.java index 93bfec13..be4733c4 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParameters.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParameters.java @@ -29,6 +29,8 @@ public interface QueryParameters extends ParameterValidator { String QUERY_LOGIC_NAME = "logicName"; String QUERY_POOL = "pool"; String QUERY_MAX_CONCURRENT_TASKS = "maxConcurrentTasks"; + String QUERY_PLAN_EXPAND_FIELDS = "expand.fields"; + String QUERY_PLAN_EXPAND_VALUES = "expand.values"; String getQuery(); @@ -94,6 +96,14 @@ public interface QueryParameters extends ParameterValidator { boolean isMaxConcurrentTasksOverridden(); + void setExpandFields(boolean expandFields); + + boolean isExpandFields(); + + void setExpandValues(boolean expandVues); + + boolean isExpandValues(); + Map> getRequestHeaders(); void setRequestHeaders(Map> requestHeaders); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 48033d89..766d4ce2 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -54,6 +54,22 @@ public GenericResponse create(@PathVariable String queryLogic, @RequestP return queryManagementService.create(queryLogic, parameters, currentUser); } + @Timed(name = "dw.query.planQuery", absolute = true) + @RequestMapping(path = "{queryLogic}/plan", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse plan(@PathVariable String queryLogic, @RequestParam MultiValueMap parameters, + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return queryManagementService.plan(queryLogic, parameters, currentUser); + } + + @Timed(name = "dw.query.predictQuery", absolute = true) + @RequestMapping(path = "{queryLogic}/predict", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse predict(@PathVariable String queryLogic, @RequestParam MultiValueMap parameters, + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return queryManagementService.predict(queryLogic, parameters, currentUser); + } + @Timed(name = "dw.query.createAndNext", absolute = true) @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE_AND_NEXT) @RequestMapping(path = "{queryLogic}/createAndNext", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 160c59d1..593fb6ce 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -80,6 +80,8 @@ import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CLOSED; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CREATED; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.DEFINED; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.PLANNED; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.PREDICTED; import static datawave.webservice.query.QueryImpl.DN_LIST; import static datawave.webservice.query.QueryImpl.QUERY_ID; import static datawave.webservice.query.QueryImpl.USER_DN; @@ -115,7 +117,7 @@ public class QueryManagementService implements QueryRequestHandler { private final String selfDestination; - private final Map createLatchMap = new ConcurrentHashMap<>(); + private final Map queryLatchMap = new ConcurrentHashMap<>(); public QueryManagementService(QueryProperties queryProperties, ApplicationEventPublisher eventPublisher, BusProperties busProperties, QueryParameters queryParameters, SecurityMarking securityMarking, BaseQueryMetric baseQueryMetric, QueryLogicFactory queryLogicFactory, @@ -259,7 +261,7 @@ public GenericResponse define(String queryLogicName, MultiValueMap response = new GenericResponse<>(); response.setResult(taskKey.getQueryId()); return response; @@ -315,7 +317,7 @@ public GenericResponse create(String queryLogicName, MultiValueMap response = new GenericResponse<>(); response.setResult(taskKey.getQueryId()); response.setHasResults(true); @@ -328,9 +330,126 @@ public GenericResponse create(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser, boolean isCreateRequest) + /** + * Creates a query using the given query logic and parameters, but only for planning purposes. + *

+ * Created queries will begin planning immediately.
+ * Auditing is performed if we are expanding indices.
+ * Query plan will be returned in the response.
+ * Updates can be made to any parameter which doesn't affect the scope of the query using {@link #update}.
+ * Stop a running query gracefully using {@link #close} or forcefully using {@link #cancel}.
+ * Stop, and restart a running query using {@link #reset}.
+ * Create a copy of a running query using {@link #duplicate}.
+ * Aside from a limited set of admin actions, only the query owner can act on a running query. + * + * @param queryLogicName + * the requested query logic, not null + * @param parameters + * the query parameters, not null + * @param currentUser + * the user who called this method, not null + * @return a generic response containing the query plan + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws QueryException + * if there is an unknown error + */ + public GenericResponse plan(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: {}/plan from {} with params: {}", queryLogicName, user, parameters); + } else { + log.info("Request: {}/plan from {}", queryLogicName, user); + } + + try { + TaskKey taskKey = storeQuery(queryLogicName, parameters, currentUser, PLANNED); + String queryPlan = queryStorageCache.getQueryStatus(taskKey.getQueryId()).getPlan(); + queryStorageCache.deleteQuery(taskKey.getQueryId()); + GenericResponse response = new GenericResponse<>(); + response.setResult(queryPlan); + response.setHasResults(true); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error planning query", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error planning query."); + } + } + + /** + * Creates a query using the given query logic and parameters, but only for prediction purposes. + *

+ * Created queries will begin predicting immediately.
+ * Auditing is not performed.
+ * Query prediction will be returned in the response.
+ * Updates can be made to any parameter which doesn't affect the scope of the query using {@link #update}.
+ * Stop a running query gracefully using {@link #close} or forcefully using {@link #cancel}.
+ * Stop, and restart a running query using {@link #reset}.
+ * Create a copy of a running query using {@link #duplicate}.
+ * Aside from a limited set of admin actions, only the query owner can act on a running query. + * + * @param queryLogicName + * the requested query logic, not null + * @param parameters + * the query parameters, not null + * @param currentUser + * the user who called this method, not null + * @return a generic response containing the query prediction + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws QueryException + * if there is an unknown error + */ + public GenericResponse predict(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { - return storeQuery(queryLogicName, parameters, currentUser, isCreateRequest, null); + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: {}/predict from {} with params: {}", queryLogicName, user, parameters); + } else { + log.info("Request: {}/predict from {}", queryLogicName, user); + } + + try { + TaskKey taskKey = storeQuery(queryLogicName, parameters, currentUser, PREDICTED); + String queryPrediction = "This endpoint has not been implemented yet"; + queryStorageCache.deleteQuery(taskKey.getQueryId()); + GenericResponse response = new GenericResponse<>(); + response.setResult(queryPrediction); + response.setHasResults(true); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error predicting query", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error predicting query."); + } + } + + private TaskKey storeQuery(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser, + QueryStatus.QUERY_STATE queryType) throws QueryException { + return storeQuery(queryLogicName, parameters, currentUser, queryType, null); } /** @@ -346,8 +465,8 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p * the query parameters, not null * @param currentUser * the user who called this method, not null - * @param isCreateRequest - * whether this is a create call true, or a define call false + * @param queryType + * whether this is a define, create, or plan call * @param queryId * the desired query id, may be null * @return the task key returned from query storage @@ -366,8 +485,8 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p * @throws QueryException * if there is an unknown error */ - private TaskKey storeQuery(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser, boolean isCreateRequest, - String queryId) throws BadRequestQueryException, QueryException { + private TaskKey storeQuery(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser, + QueryStatus.QUERY_STATE queryType, String queryId) throws BadRequestQueryException, QueryException { // validate query and get a query logic QueryLogic queryLogic = validateQuery(queryLogicName, parameters, currentUser); @@ -389,15 +508,23 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p throw new BadRequestQueryException("Unable to downgrade authorizations", e, HttpStatus.SC_BAD_REQUEST + "-1"); } - // if this is a create request, send an audit record to the auditor - if (isCreateRequest) { + // if this is a create request, or a plan request where we are expanding values, send an audit record to the auditor + if (queryType == CREATED || (queryType == PLANNED && queryParameters.isExpandValues())) { audit(query, queryLogic, parameters, currentUser); } try { // persist the query w/ query id in the query storage cache - TaskKey taskKey; - if (isCreateRequest) { + TaskKey taskKey = null; + if (queryType == DEFINED) { + // @formatter:off + taskKey = queryStorageCache.defineQuery( + getPoolName(), + query, + downgradedAuthorizations, + getMaxConcurrentTasks(queryLogic)); + // @formatter:on + } else if (queryType == CREATED) { // @formatter:off taskKey = queryStorageCache.createQuery( getPoolName(), @@ -406,47 +533,39 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p getMaxConcurrentTasks(queryLogic)); // @formatter:on - if (queryProperties.isAwaitExecutorCreateResponse()) { - // before publishing the message, create a latch based on the query ID - createLatchMap.put(taskKey.getQueryId(), new CountDownLatch(1)); - } - - // publish a create event to the executor pool - publishExecutorEvent(QueryRequest.create(taskKey.getQueryId()), getPoolName()); + sendRequestAwaitResponse(QueryRequest.create(taskKey.getQueryId()), queryProperties.isAwaitExecutorCreateResponse()); + } else if (queryType == PLANNED) { + // @formatter:off + taskKey = queryStorageCache.planQuery( + getPoolName(), + query, + downgradedAuthorizations); + // @formatter:on - if (queryProperties.isAwaitExecutorCreateResponse()) { - log.info("Waiting on query create response from the executor."); - // wait for the executor to finish creating the query - try { - // TODO: Should we incorporate the call start time into this check? - if (!createLatchMap.get(taskKey.getQueryId()).await(queryProperties.getExpiration().getCallTimeout(), - queryProperties.getExpiration().getCallTimeUnit())) { - throw new QueryException(DatawaveErrorCode.QUERY_TIMEOUT, "Timed out waiting for query create to finish."); - } else { - log.info("Received query create response from the executor."); - } - } catch (InterruptedException e) { - log.warn("Interrupted while waiting for query create latch for queryId {}", queryId); - throw new QueryException(DatawaveErrorCode.QUERY_TIMEOUT, "Interrupted while waiting for query create to finish."); - } finally { - createLatchMap.remove(taskKey.getQueryId()); - } - } - } else { + sendRequestAwaitResponse(QueryRequest.plan(taskKey.getQueryId()), true); + } else if (queryType == PREDICTED) { // @formatter:off - taskKey = queryStorageCache.defineQuery( + taskKey = queryStorageCache.predictQuery( getPoolName(), query, - downgradedAuthorizations, - getMaxConcurrentTasks(queryLogic)); + downgradedAuthorizations); // @formatter:on + + sendRequestAwaitResponse(QueryRequest.predict(taskKey.getQueryId()), true); + } + + if (taskKey == null) { + log.error("Task Key not created for query"); + throw new QueryException(DatawaveErrorCode.RUNNING_QUERY_CACHE_ERROR); } // update the query metric - baseQueryMetric.setQueryId(taskKey.getQueryId()); - baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.DEFINED); - baseQueryMetric.populate(query); - baseQueryMetric.setProxyServers(currentUser.getProxyServers()); + if (queryType == DEFINED || queryType == CREATED) { + baseQueryMetric.setQueryId(taskKey.getQueryId()); + baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.DEFINED); + baseQueryMetric.populate(query); + baseQueryMetric.setProxyServers(currentUser.getProxyServers()); + } return taskKey; } catch (Exception e) { @@ -455,6 +574,35 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p } } + private void sendRequestAwaitResponse(QueryRequest request, boolean isAwaitResponse) throws QueryException { + if (isAwaitResponse) { + // before publishing the message, create a latch based on the query ID + queryLatchMap.put(request.getQueryId(), new CountDownLatch(1)); + } + + // publish a plan event to the executor pool + publishExecutorEvent(request, getPoolName()); + + if (isAwaitResponse) { + log.info("Waiting on query {} response from the executor.", request.getMethod().name()); + // wait for the executor to finish creating the query + try { + // TODO: Should we incorporate the call start time into this check? + if (!queryLatchMap.get(request.getQueryId()).await(queryProperties.getExpiration().getCallTimeout(), + queryProperties.getExpiration().getCallTimeUnit())) { + throw new QueryException(DatawaveErrorCode.QUERY_TIMEOUT, "Timed out waiting for query " + request.getMethod().name() + " to finish."); + } else { + log.info("Received query {} response from the executor.", request.getMethod().name()); + } + } catch (InterruptedException e) { + log.warn("Interrupted while waiting for query {} latch for queryId {}", request.getMethod().name(), request.getQueryId()); + throw new QueryException(DatawaveErrorCode.QUERY_TIMEOUT, "Interrupted while waiting for query " + request.getMethod().name() + " to finish."); + } finally { + queryLatchMap.remove(request.getQueryId()); + } + } + } + /** * Creates a query using the given query logic and parameters, and returns the first page of results. *

@@ -1400,7 +1548,7 @@ public GenericResponse update(String queryId, MultiValueMap updateParameters(parameters, currentParams); // define a duplicate query - return storeQuery(currentParams.getFirst(QUERY_LOGIC_NAME), currentParams, currentUser, true); + return storeQuery(currentParams.getFirst(QUERY_LOGIC_NAME), currentParams, currentUser, CREATED); } private boolean updateParameters(MultiValueMap newParameters, MultiValueMap currentParams) throws BadRequestQueryException { @@ -1750,12 +1898,13 @@ public void handleRemoteRequest(QueryRequest queryRequest, String originService, if (queryRequest.getMethod() == QueryRequest.Method.CANCEL) { log.trace("Received remote cancel request from {} for {}.", originService, destinationService); cancel(queryRequest.getQueryId(), false); - } else if (queryRequest.getMethod() == QueryRequest.Method.CREATE) { - log.trace("Received remote create request from {} for {}.", originService, destinationService); - if (createLatchMap.containsKey(queryRequest.getQueryId())) { - createLatchMap.get(queryRequest.getQueryId()).countDown(); + } else if (queryRequest.getMethod() == QueryRequest.Method.CREATE || queryRequest.getMethod() == QueryRequest.Method.PLAN + || queryRequest.getMethod() == QueryRequest.Method.PREDICT) { + log.trace("Received remote {} request from {} for {}.", queryRequest.getMethod().name(), originService, destinationService); + if (queryLatchMap.containsKey(queryRequest.getQueryId())) { + queryLatchMap.get(queryRequest.getQueryId()).countDown(); } else { - log.warn("Unable to decrement create latch for query {}", queryRequest.getQueryId()); + log.warn("Unable to decrement {} latch for query {}", queryRequest.getMethod().name(), queryRequest.getQueryId()); } } else { log.debug("No handling specified for remote query request method: {} from {} for {}", queryRequest.getMethod(), originService, diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java index 343053f6..43f7fbd9 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java @@ -31,12 +31,14 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.web.client.RestTemplateBuilder; import org.springframework.boot.web.server.LocalServerPort; +import org.springframework.cloud.bus.ServiceMatcher; import org.springframework.cloud.bus.event.RemoteQueryRequestEvent; import org.springframework.context.ApplicationEvent; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Primary; import org.springframework.context.annotation.Profile; import org.springframework.http.HttpMethod; @@ -506,16 +508,18 @@ public LinkedList queryRequestEvents() { @Bean @Primary - public ApplicationEventPublisher eventPublisher(ApplicationEventPublisher eventPublisher) { + public ApplicationEventPublisher eventPublisher(@Lazy QueryManagementService queryManagementService, ServiceMatcher serviceMatcher) { return new ApplicationEventPublisher() { @Override public void publishEvent(ApplicationEvent event) { saveEvent(event); + processEvent(event); } @Override public void publishEvent(Object event) { saveEvent(event); + processEvent(event); } private void saveEvent(Object event) { @@ -523,6 +527,17 @@ private void saveEvent(Object event) { queryRequestEvents().push(((RemoteQueryRequestEvent) event)); } } + + private void processEvent(Object event) { + if (event instanceof RemoteQueryRequestEvent) { + RemoteQueryRequestEvent queryEvent = (RemoteQueryRequestEvent) event; + boolean isSelfRequest = serviceMatcher.isFromSelf(queryEvent); + if (!isSelfRequest) { + queryManagementService.handleRemoteRequest(queryEvent.getRequest(), queryEvent.getOriginService(), + queryEvent.getDestinationService()); + } + } + } }; } } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java new file mode 100644 index 00000000..145ca515 --- /dev/null +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java @@ -0,0 +1,113 @@ +package datawave.microservice.query; + +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.result.GenericResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.cloud.bus.event.RemoteQueryRequestEvent; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; + +import java.io.IOException; +import java.text.ParseException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +public class QueryServicePlanTest extends AbstractQueryServiceTest { + + @Autowired + public ApplicationEventPublisher eventPublisher; + + @Before + public void setup() { + super.setup(); + } + + @After + public void teardown() throws Exception { + super.teardown(); + } + + @Test + public void testPlanSuccess() throws ParseException, IOException, ExecutionException, InterruptedException { + ProxiedUserDetails authUser = createUserDetails(); + UriComponents uri = createUri("EventQuery/plan"); + MultiValueMap map = createParams(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + + // setup a mock audit service + auditSentSetup(); + + // make the plan call asynchronously + Future> futureResp = Executors.newSingleThreadExecutor() + .submit(() -> jwtRestTemplate.exchange(requestEntity, GenericResponse.class)); + + long startTime = System.currentTimeMillis(); + while (queryRequestEvents.size() == 0 && (System.currentTimeMillis() - startTime) < TimeUnit.SECONDS.toMillis(5)) { + Thread.sleep(500); + } + + // verify that the plan event was published + Assert.assertEquals(1, queryRequestEvents.size()); + RemoteQueryRequestEvent requestEvent = queryRequestEvents.removeLast(); + + Assert.assertEquals("executor-unassigned:**", requestEvent.getDestinationService()); + Assert.assertEquals(QueryRequest.Method.PLAN, requestEvent.getRequest().getMethod()); + + String queryId = requestEvent.getRequest().getQueryId(); + String plan = "some plan"; + + // save the plan to the query status object + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + queryStatus.setPlan(plan); + queryStorageCache.updateQueryStatus(queryStatus); + + // send the plan response + eventPublisher.publishEvent(new RemoteQueryRequestEvent(this, "executor-unassigned:**", "query:**", QueryRequest.plan(queryId))); + + ResponseEntity resp = futureResp.get(); + + // @formatter:off + GenericResponse genericResponse = assertGenericResponse( + true, + HttpStatus.Series.SUCCESSFUL, + resp); + // @formatter:on + + String receivedPlan = genericResponse.getResult(); + Assert.assertEquals(receivedPlan, plan); + + // @formatter:off + assertQueryRequestEvent( + "query:**", + QueryRequest.Method.PLAN, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + + // verify that the query status was deleted + queryStatus = queryStorageCache.getQueryStatus(queryId); + Assert.assertNull(queryStatus); + } +} From 97600847a4524c4b7f65dce5126ddf04146e7a91 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Fri, 19 Nov 2021 22:04:08 +0000 Subject: [PATCH 109/218] Got the DiscoveryQuery working as checkpointable and noncheckpointable. --- .../src/test/resources/config/application.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/query-microservices/query-service/service/src/test/resources/config/application.yml b/query-microservices/query-service/service/src/test/resources/config/application.yml index c029ae3f..10da535e 100644 --- a/query-microservices/query-service/service/src/test/resources/config/application.yml +++ b/query-microservices/query-service/service/src/test/resources/config/application.yml @@ -142,6 +142,7 @@ datawave: pageByteTrigger: 0 logics: BaseEventQuery: + checkpointable: true accumuloPassword: ${warehouse.accumulo.password} tableName: ${warehouse.tables.shard.name} dateIndexTableName: ${warehouse.tables.dateIndex.name} @@ -230,6 +231,20 @@ datawave: requiredRoles: - "AuthorizedUser" + DiscoveryQuery: + checkpointable: true + tableName: ${warehouse.tables.shard.name} + indexTableName: ${warehouse.tables.index.name} + reverseIndexTableName: ${warehouse.tables.reverseIndex.name} + metadataTableName: ${warehouse.tables.metadata.name} + modelTableName: ${warehouse.tables.model.name} + modelName: ${warehouse.defaults.modelName} + fullTableScanEnabled: ${warehouse.defaults.fullTableScanEnabled} + allowLeadingWildcard: true + auditType: "NONE" + maxResults: -1 + + audit-client: discovery: enabled: false From 86d5c2b226efd649b51422a7a2bddf19cb018661 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 24 Nov 2021 17:30:11 -0500 Subject: [PATCH 110/218] Added a hazelcast query results manager. Added a hazelcast claim check mechanism to handle messages which are too big for rabbitmq. Fixed some of the locking to use the intended cache. --- .../java/datawave/microservice/query/runner/NextCall.java | 3 +++ .../service/src/test/resources/config/application.yml | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index e1050fcc..0b228a55 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -133,6 +133,9 @@ public ResultsPage call() throws Exception { } } } + } catch (Exception e) { + log.error("Encountered an error while fetching results from the listener", e); + throw e; } // update some values for metrics diff --git a/query-microservices/query-service/service/src/test/resources/config/application.yml b/query-microservices/query-service/service/src/test/resources/config/application.yml index 10da535e..1ba5fd54 100644 --- a/query-microservices/query-service/service/src/test/resources/config/application.yml +++ b/query-microservices/query-service/service/src/test/resources/config/application.yml @@ -244,6 +244,10 @@ datawave: auditType: "NONE" maxResults: -1 +query: + messaging: + claimCheck: + enabled: false audit-client: discovery: From c96079fd1738d35a3954ec43c4bab54f3e12ddbd Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 26 Nov 2021 11:16:16 -0500 Subject: [PATCH 111/218] Added rest endpoints to retrieve the plan and predictions. --- query-microservices/query-service/README.md | 4 +- .../microservice/query/QueryController.java | 14 ++++ .../query/QueryManagementService.java | 80 +++++++++++++++++++ 3 files changed, 96 insertions(+), 2 deletions(-) diff --git a/query-microservices/query-service/README.md b/query-microservices/query-service/README.md index b5f823cb..5fd2ba9e 100644 --- a/query-microservices/query-service/README.md +++ b/query-microservices/query-service/README.md @@ -26,8 +26,8 @@ query capabilities. | | | | `POST` | /lookupContentUUID | Returns content associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] or [StreamingOutput] | | | | | `GET` | /lookupUUID/{uuidType}/{uuid} | Returns event associated with the given batch of UUID | [UUIDType], [UUID] | N/A | [BaseQueryResponse] or [StreamingOutput] | | | | | `POST` | /lookupUUID | Returns event(s) associated with the given batch of UUIDs | N/A | [QueryParameters] | [BaseQueryResponse] or [StreamingOutput] | -| | | | `GET` | /{id}/plan | Returns the plan for the specified query | [QueryId] | N/A | [GenericResponse] | -| | | | `GET` | /{id}/predictions | Returns the predictions for the specified query | [QueryId] | N/A | [GenericResponse] | +| ✓ | | | `GET` | /{id}/plan | Returns the plan for the specified query | [QueryId] | N/A | [GenericResponse] | +| ✓ | | | `GET` | /{id}/predictions | Returns the predictions for the specified query | [QueryId] | N/A | [GenericResponse] | | ✓ | | | `GET` | /{id}/async/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | | ✓ | | | `GET` | /{id}/next | Returns the next page of results for the specified query | [QueryId] | N/A | [BaseQueryResponse] | | ✓ | | | `PUT` `POST` | /{id}/close | Closes the specified query | [QueryId] | N/A | [VoidResponse] | diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 766d4ce2..43e3b511 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -179,6 +179,20 @@ public QueryImplListResponse get(@PathVariable String queryId, @AuthenticationPr return queryManagementService.list(queryId, null, currentUser); } + @Timed(name = "dw.query.plan", absolute = true) + @RequestMapping(path = "{queryId}/plan", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse plan(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return queryManagementService.plan(queryId, currentUser); + } + + @Timed(name = "dw.query.predictions", absolute = true) + @RequestMapping(path = "{queryId}/predictions", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public GenericResponse predictions(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return queryManagementService.predictions(queryId, currentUser); + } + @Timed(name = "dw.query.adminCancelAll", absolute = true) @RolesAllowed({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "adminCancelAll", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 593fb6ce..d4647414 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -1839,6 +1839,86 @@ private QueryImplListResponse list(String queryId, String queryName, String user } } + /** + * Gets the plan for the given query for the calling user. + * + * @param queryId + * the query id, may be null + * @param currentUser + * the user who called this method, not null + * @return the query plan for the matching query + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if query storage fails + * @throws QueryException + * if there is an unknown error + */ + public GenericResponse plan(String queryId, ProxiedUserDetails currentUser) throws QueryException { + log.info("Request: plan from {} for queryId: {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId); + + try { + // make sure the query is valid, and the user can act on it + QueryStatus queryStatus = validateRequest(queryId, currentUser); + + GenericResponse response = new GenericResponse<>(); + if (queryStatus.getPlan() != null) { + response.setResult(queryStatus.getPlan()); + response.setHasResults(true); + } + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error getting plan for query {}", queryId, e); + throw new QueryException(DatawaveErrorCode.QUERY_PLAN_ERROR, e, "Unknown error getting plan for query " + queryId); + } + } + + /** + * Gets the predictions for the given query for the calling user. + * + * @param queryId + * the query id, may be null + * @param currentUser + * the user who called this method, not null + * @return the query predictions for the matching query + * @throws NotFoundQueryException + * if the query cannot be found + * @throws UnauthorizedQueryException + * if the user doesn't own the query + * @throws QueryException + * if query lock acquisition fails + * @throws QueryException + * if query storage fails + * @throws QueryException + * if there is an unknown error + */ + public GenericResponse predictions(String queryId, ProxiedUserDetails currentUser) throws QueryException { + log.info("Request: predictions from {} for queryId: {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), queryId); + + try { + // make sure the query is valid, and the user can act on it + QueryStatus queryStatus = validateRequest(queryId, currentUser); + + GenericResponse response = new GenericResponse<>(); + if (queryStatus.getPredictions() != null && !queryStatus.getPredictions().isEmpty()) { + response.setResult(queryStatus.getPredictions().toString()); + response.setHasResults(true); + } + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error getting predictions for query {}", queryId, e); + throw new QueryException(DatawaveErrorCode.QUERY_PLAN_ERROR, e, "Unknown error getting predictions for query " + queryId); + } + } + private QueryStatus validateRequest(String queryId, ProxiedUserDetails currentUser) throws QueryException { return validateRequest(queryId, currentUser, false); } From e915fdc675e661385fd054f1e394a06fb84a2948 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Sun, 12 Dec 2021 13:56:33 -0500 Subject: [PATCH 112/218] Added initial lookupUUID & batchLookupUUID functionality to the query service. Restructured use of QueryLogicFactory.xml to use a single instance of that file from the query starter. Updated query microservices to use a common application-QueryStarterDefaults.yml file for property defaults, with microservice-specific application-QueryStarterOverrides.yml for property overrides. Added scripts to test lookup UUID functionality. --- .../config/DefaultParameterProperties.java | 31 - .../query/config/NextCallProperties.java | 104 --- .../query/config/QueryProperties.java | 150 ---- .../ThreadPoolTaskExecutorProperties.java | 62 -- .../query-service/service/pom.xml | 18 + .../microservice/query/QueryController.java | 37 +- .../query/config/QueryServiceConfig.java | 8 - .../query/uuid/LookupUUIDService.java | 274 ++++++ .../query/QueryServiceCancelTest.java | 2 +- .../query/QueryServiceCloseTest.java | 2 +- .../query/QueryServiceCreateTest.java | 2 +- .../query/QueryServiceDefineTest.java | 2 +- .../query/QueryServiceDuplicateTest.java | 2 +- .../query/QueryServiceListTest.java | 10 +- .../query/QueryServiceNextTest.java | 2 +- .../query/QueryServicePlanTest.java | 2 +- .../query/QueryServiceRemoveTest.java | 2 +- .../query/QueryServiceResetTest.java | 2 +- .../query/QueryServiceUpdateTest.java | 2 +- .../uuid/QueryServiceLookupUUIDTest.java | 99 +++ .../resources/MyTestQueryLogicFactory.xml | 826 ------------------ .../test/resources/TestQueryLogicFactory.xml | 15 + .../application-QueryStarterOverrides.yml | 20 + .../src/test/resources/config/application.yml | 209 ----- 24 files changed, 479 insertions(+), 1404 deletions(-) delete mode 100644 query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/DefaultParameterProperties.java delete mode 100644 query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/NextCallProperties.java delete mode 100644 query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java delete mode 100644 query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/ThreadPoolTaskExecutorProperties.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/uuid/LookupUUIDService.java create mode 100644 query-microservices/query-service/service/src/test/java/datawave/microservice/query/uuid/QueryServiceLookupUUIDTest.java delete mode 100644 query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml create mode 100644 query-microservices/query-service/service/src/test/resources/TestQueryLogicFactory.xml create mode 100644 query-microservices/query-service/service/src/test/resources/config/application-QueryStarterOverrides.yml diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/DefaultParameterProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/DefaultParameterProperties.java deleted file mode 100644 index 4516ae8b..00000000 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/DefaultParameterProperties.java +++ /dev/null @@ -1,31 +0,0 @@ -package datawave.microservice.query.config; - -import org.springframework.validation.annotation.Validated; - -import javax.annotation.Nonnegative; -import javax.validation.constraints.NotEmpty; - -@Validated -public class DefaultParameterProperties { - @NotEmpty - private String pool = "unassigned"; - @Nonnegative - private int maxConcurrentTasks = 10; - - public String getPool() { - return pool; - } - - public void setPool(String pool) { - this.pool = pool; - } - - public int getMaxConcurrentTasks() { - return maxConcurrentTasks; - } - - public void setMaxConcurrentTasks(int maxConcurrentTasks) { - this.maxConcurrentTasks = maxConcurrentTasks; - } - -} diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/NextCallProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/NextCallProperties.java deleted file mode 100644 index 2ce444c2..00000000 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/NextCallProperties.java +++ /dev/null @@ -1,104 +0,0 @@ -package datawave.microservice.query.config; - -import org.springframework.validation.annotation.Validated; - -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Positive; -import javax.validation.constraints.PositiveOrZero; -import java.util.concurrent.TimeUnit; - -@Validated -public class NextCallProperties { - @PositiveOrZero - private long resultPollInterval = TimeUnit.SECONDS.toMillis(6); - @NotNull - private TimeUnit resultPollTimeUnit = TimeUnit.MILLISECONDS; - @Positive - private int concurrency = 1; - @PositiveOrZero - private long statusUpdateInterval = TimeUnit.SECONDS.toMillis(6); - @NotNull - private TimeUnit statusUpdateTimeUnit = TimeUnit.MILLISECONDS; - @PositiveOrZero - private long maxResultsTimeout = 5l; - @NotNull - private TimeUnit maxResultsTimeUnit = TimeUnit.SECONDS; - - private ThreadPoolTaskExecutorProperties executor = new ThreadPoolTaskExecutorProperties(10, 100, 100, "nextCall-"); - - public long getResultPollInterval() { - return resultPollInterval; - } - - public long getResultPollIntervalMillis() { - return resultPollTimeUnit.toMillis(resultPollInterval); - } - - public void setResultPollInterval(long resultPollInterval) { - this.resultPollInterval = resultPollInterval; - } - - public TimeUnit getResultPollTimeUnit() { - return resultPollTimeUnit; - } - - public void setResultPollTimeUnit(TimeUnit resultPollTimeUnit) { - this.resultPollTimeUnit = resultPollTimeUnit; - } - - public int getConcurrency() { - return concurrency; - } - - public void setConcurrency(int concurrency) { - this.concurrency = concurrency; - } - - public long getStatusUpdateInterval() { - return statusUpdateInterval; - } - - public long getStatusUpdateIntervalMillis() { - return statusUpdateTimeUnit.toMillis(statusUpdateInterval); - } - - public void setStatusUpdateInterval(long statusUpdateInterval) { - this.statusUpdateInterval = statusUpdateInterval; - } - - public TimeUnit getStatusUpdateTimeUnit() { - return statusUpdateTimeUnit; - } - - public void setStatusUpdateTimeUnit(TimeUnit statusUpdateTimeUnit) { - this.statusUpdateTimeUnit = statusUpdateTimeUnit; - } - - public long getMaxResultsTimeout() { - return maxResultsTimeout; - } - - public long getMaxResultsTimeoutMillis() { - return maxResultsTimeUnit.toMillis(maxResultsTimeout); - } - - public void setMaxResultsTimeout(long maxResultsTimeout) { - this.maxResultsTimeout = maxResultsTimeout; - } - - public TimeUnit getMaxResultsTimeUnit() { - return maxResultsTimeUnit; - } - - public void setMaxResultsTimeUnit(TimeUnit maxResultsTimeUnit) { - this.maxResultsTimeUnit = maxResultsTimeUnit; - } - - public ThreadPoolTaskExecutorProperties getExecutor() { - return executor; - } - - public void setExecutor(ThreadPoolTaskExecutorProperties executor) { - this.executor = executor; - } -} diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java deleted file mode 100644 index 79847694..00000000 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryProperties.java +++ /dev/null @@ -1,150 +0,0 @@ -package datawave.microservice.query.config; - -import org.springframework.validation.annotation.Validated; - -import javax.validation.constraints.NotEmpty; -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Positive; -import javax.validation.constraints.PositiveOrZero; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.TimeUnit; - -import static datawave.microservice.query.QueryParameters.QUERY_EXPIRATION; -import static datawave.microservice.query.QueryParameters.QUERY_MAX_RESULTS_OVERRIDE; -import static datawave.microservice.query.QueryParameters.QUERY_PAGESIZE; -import static datawave.microservice.query.QueryParameters.QUERY_PAGETIMEOUT; - -@Validated -public class QueryProperties { - @NotEmpty - private String privilegedRole = "PrivilegedUser"; - // The amount of time to wait for the lock to be acquired - @PositiveOrZero - private long lockWaitTime = TimeUnit.SECONDS.toMillis(5); - @NotNull - private TimeUnit lockWaitTimeUnit = TimeUnit.MILLISECONDS; - // The amount of time that the lock will be held before being automatically released - @Positive - private long lockLeaseTime = TimeUnit.SECONDS.toMillis(30); - @NotNull - private TimeUnit lockLeaseTimeUnit = TimeUnit.MILLISECONDS; - @NotEmpty - private String queryServiceName = "query"; - @NotEmpty - private String executorServiceName = "executor"; - // These are the only parameters that can be updated for a running query - private List updatableParams = Arrays.asList(QUERY_EXPIRATION, QUERY_PAGESIZE, QUERY_PAGETIMEOUT, QUERY_MAX_RESULTS_OVERRIDE); - // Whether or not to wait for an executor create response before returning to the caller - private boolean awaitExecutorCreateResponse = true; - - private QueryExpirationProperties expiration = new QueryExpirationProperties(); - private NextCallProperties nextCall = new NextCallProperties(); - private DefaultParameterProperties defaultParams = new DefaultParameterProperties(); - - public String getPrivilegedRole() { - return privilegedRole; - } - - public void setPrivilegedRole(String privilegedRole) { - this.privilegedRole = privilegedRole; - } - - public long getLockWaitTime() { - return lockWaitTime; - } - - public long getLockWaitTimeMillis() { - return lockWaitTimeUnit.toMillis(lockWaitTime); - } - - public void setLockWaitTime(long lockWaitTime) { - this.lockWaitTime = lockWaitTime; - } - - public TimeUnit getLockWaitTimeUnit() { - return lockWaitTimeUnit; - } - - public QueryProperties setLockWaitTimeUnit(TimeUnit lockWaitTimeUnit) { - this.lockWaitTimeUnit = lockWaitTimeUnit; - return this; - } - - public long getLockLeaseTime() { - return lockLeaseTime; - } - - public long getLockLeaseTimeMillis() { - return lockLeaseTimeUnit.toMillis(lockLeaseTime); - } - - public void setLockLeaseTime(long lockLeaseTime) { - this.lockLeaseTime = lockLeaseTime; - } - - public TimeUnit getLockLeaseTimeUnit() { - return lockLeaseTimeUnit; - } - - public QueryProperties setLockLeaseTimeUnit(TimeUnit lockLeaseTimeUnit) { - this.lockLeaseTimeUnit = lockLeaseTimeUnit; - return this; - } - - public String getQueryServiceName() { - return queryServiceName; - } - - public void setQueryServiceName(String queryServiceName) { - this.queryServiceName = queryServiceName; - } - - public String getExecutorServiceName() { - return executorServiceName; - } - - public void setExecutorServiceName(String executorServiceName) { - this.executorServiceName = executorServiceName; - } - - public List getUpdatableParams() { - return updatableParams; - } - - public void setUpdatableParams(List updatableParams) { - this.updatableParams = updatableParams; - } - - public boolean isAwaitExecutorCreateResponse() { - return awaitExecutorCreateResponse; - } - - public void setAwaitExecutorCreateResponse(boolean awaitExecutorCreateResponse) { - this.awaitExecutorCreateResponse = awaitExecutorCreateResponse; - } - - public QueryExpirationProperties getExpiration() { - return expiration; - } - - public void setExpiration(QueryExpirationProperties expiration) { - this.expiration = expiration; - } - - public NextCallProperties getNextCall() { - return nextCall; - } - - public void setNextCall(NextCallProperties nextCall) { - this.nextCall = nextCall; - } - - public DefaultParameterProperties getDefaultParams() { - return defaultParams; - } - - public void setDefaultParams(DefaultParameterProperties defaultParams) { - this.defaultParams = defaultParams; - } -} diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/ThreadPoolTaskExecutorProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/ThreadPoolTaskExecutorProperties.java deleted file mode 100644 index 23b4aa74..00000000 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/ThreadPoolTaskExecutorProperties.java +++ /dev/null @@ -1,62 +0,0 @@ -package datawave.microservice.query.config; - -import org.springframework.validation.annotation.Validated; - -import javax.validation.constraints.NotNull; -import javax.validation.constraints.Positive; -import javax.validation.constraints.PositiveOrZero; - -@Validated -public class ThreadPoolTaskExecutorProperties { - @PositiveOrZero - private int corePoolSize = 0; - @Positive - private int maxPoolSize = 5; - @PositiveOrZero - private int queueCapacity = 0; - @NotNull - private String threadNamePrefix = ""; - - public ThreadPoolTaskExecutorProperties() { - - } - - public ThreadPoolTaskExecutorProperties(int corePoolSize, int maxPoolSize, int queueCapacity, String threadNamePrefix) { - this.corePoolSize = corePoolSize; - this.maxPoolSize = maxPoolSize; - this.queueCapacity = queueCapacity; - this.threadNamePrefix = threadNamePrefix; - } - - public int getCorePoolSize() { - return corePoolSize; - } - - public void setCorePoolSize(int corePoolSize) { - this.corePoolSize = corePoolSize; - } - - public int getMaxPoolSize() { - return maxPoolSize; - } - - public void setMaxPoolSize(int maxPoolSize) { - this.maxPoolSize = maxPoolSize; - } - - public int getQueueCapacity() { - return queueCapacity; - } - - public void setQueueCapacity(int queueCapacity) { - this.queueCapacity = queueCapacity; - } - - public String getThreadNamePrefix() { - return threadNamePrefix; - } - - public void setThreadNamePrefix(String threadNamePrefix) { - this.threadNamePrefix = threadNamePrefix; - } -} diff --git a/query-microservices/query-service/service/pom.xml b/query-microservices/query-service/service/pom.xml index 719f755e..2478a77d 100644 --- a/query-microservices/query-service/service/pom.xml +++ b/query-microservices/query-service/service/pom.xml @@ -119,6 +119,24 @@ org.apache.maven.plugins maven-dependency-plugin + + unpack-test-resources + process-test-resources + + unpack + + + + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-query + test-jar + **/application-*.yml + ${project.build.testOutputDirectory} + + + + unpack package diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 43e3b511..302680b8 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -2,6 +2,7 @@ import com.codahale.metrics.annotation.Timed; import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.query.uuid.LookupUUIDService; import datawave.microservice.query.web.annotation.EnrichQueryMetrics; import datawave.webservice.query.exception.QueryException; import datawave.webservice.result.BaseQueryResponse; @@ -24,9 +25,11 @@ @RequestMapping(path = "/v1", produces = MediaType.APPLICATION_JSON_VALUE) public class QueryController { private final QueryManagementService queryManagementService; + private final LookupUUIDService lookupUUIDService; - public QueryController(QueryManagementService queryManagementService) { + public QueryController(QueryManagementService queryManagementService, LookupUUIDService lookupUUIDService) { this.queryManagementService = queryManagementService; + this.lookupUUIDService = lookupUUIDService; } @Timed(name = "dw.query.listQueryLogic", absolute = true) @@ -70,6 +73,38 @@ public GenericResponse predict(@PathVariable String queryLogic, @Request return queryManagementService.predict(queryLogic, parameters, currentUser); } + @Timed(name = "dw.query.lookupUUID", absolute = true) + @RequestMapping(path = "lookupUUID/{uuidType}/{uuid}", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public BaseQueryResponse lookupUUID(@PathVariable(required = false) String uuidType, @PathVariable(required = false) String uuid, + @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return lookupUUIDService.lookupUUID(uuidType, uuid, parameters, currentUser); + } + + @Timed(name = "dw.query.lookupUUIDBatch", absolute = true) + @RequestMapping(path = "lookupUUID", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public BaseQueryResponse lookupUUIDBatch(@PathVariable(required = false) String uuidType, @PathVariable(required = false) String uuid, + @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return lookupUUIDService.lookupUUID(parameters, currentUser); + } + + @Timed(name = "dw.query.lookupContentUUID", absolute = true) + @RequestMapping(path = "lookupContentUUID/{uuidType}/{uuid}", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public BaseQueryResponse lookupContentUUID(@PathVariable(required = false) String uuidType, @PathVariable(required = false) String uuid, + @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return lookupUUIDService.lookupContentUUID(uuidType, uuid, parameters, currentUser); + } + + @Timed(name = "dw.query.lookupContentUUIDBatch", absolute = true) + @RequestMapping(path = "lookupContentUUID", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public BaseQueryResponse lookupContentUUIDBatch(@PathVariable(required = false) String uuidType, @PathVariable(required = false) String uuid, + @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return lookupUUIDService.lookupContentUUID(parameters, currentUser); + } + @Timed(name = "dw.query.createAndNext", absolute = true) @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE_AND_NEXT) @RequestMapping(path = "{queryLogic}/createAndNext", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java index 88f85a6d..d2bc8b5d 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java @@ -8,7 +8,6 @@ import datawave.microservice.querymetric.QueryMetricFactory; import datawave.microservice.querymetric.QueryMetricFactoryImpl; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -43,13 +42,6 @@ public BaseQueryMetric baseQueryMetric() { return queryMetricFactory().createMetric(); } - @Bean - @ConditionalOnMissingBean(QueryProperties.class) - @ConfigurationProperties("datawave.query") - public QueryProperties queryProperties() { - return new QueryProperties(); - } - @Bean @ConditionalOnMissingBean(type = "QueryMetricFactory") public QueryMetricFactory queryMetricFactory() { diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/uuid/LookupUUIDService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/uuid/LookupUUIDService.java new file mode 100644 index 00000000..4afed6c2 --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/uuid/LookupUUIDService.java @@ -0,0 +1,274 @@ +package datawave.microservice.query.uuid; + +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.authorization.util.AuthorizationsUtil; +import datawave.microservice.query.DefaultQueryParameters; +import datawave.microservice.query.QueryManagementService; +import datawave.microservice.query.QueryParameters; +import datawave.microservice.query.QueryPersistence; +import datawave.microservice.query.config.QueryProperties; +import datawave.query.data.UUIDType; +import datawave.security.util.ProxiedEntityUtils; +import datawave.webservice.query.exception.BadRequestQueryException; +import datawave.webservice.query.exception.DatawaveErrorCode; +import datawave.webservice.query.exception.QueryException; +import datawave.webservice.query.result.event.ResponseObjectFactory; +import datawave.webservice.result.BaseQueryResponse; +import org.apache.commons.lang.time.DateUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +import java.text.ParseException; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; +import static datawave.microservice.query.QueryParameters.QUERY_STRING; +import static datawave.query.QueryParameters.QUERY_SYNTAX; + +@Service +public class LookupUUIDService { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + + public static final String LOOKUP_UUID_PAIRS = "uuidPairs"; + public static final String LUCENE_UUID_SYNTAX = "LUCENE-UUID"; + + protected static final String UUID_TERM_DELIMITER = ":"; + + private final QueryProperties queryProperties; + private final LookupUUIDProperties uuidProperties; + + private final ResponseObjectFactory responseObjectFactory; + private final QueryManagementService queryManagementService; + + public LookupUUIDService(QueryProperties queryProperties, LookupUUIDProperties uuidProperties, ResponseObjectFactory responseObjectFactory, + QueryManagementService queryManagementService) { + this.queryProperties = queryProperties; + this.uuidProperties = uuidProperties; + this.responseObjectFactory = responseObjectFactory; + this.queryManagementService = queryManagementService; + } + + public BaseQueryResponse lookupUUID(String uuidType, String uuid, MultiValueMap parameters, ProxiedUserDetails currentUser) + throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: lookupUUID/{}/{} from {} with params: {}", uuidType, uuid, user, parameters); + } else { + log.info("Request: lookupUUID/{}/{} from {}", uuidType, uuid, user); + } + + parameters.add(LOOKUP_UUID_PAIRS, String.join(UUID_TERM_DELIMITER, uuidType, uuid)); + + try { + return lookup(parameters, currentUser, false); + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error looking up UUID", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error looking up UUID."); + } + } + + public BaseQueryResponse lookupUUID(MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: lookupUUID (batch) from {} with params: {}", user, parameters); + } else { + log.info("Request: lookupUUID (batch) from {}", user); + } + + try { + return lookup(parameters, currentUser, false); + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error looking up UUID", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error looking up UUID."); + } + } + + public BaseQueryResponse lookupContentUUID(String uuidType, String uuid, MultiValueMap parameters, ProxiedUserDetails currentUser) + throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: lookupContentUUID/{}/{} from {} with params: {}", uuidType, uuid, user, parameters); + } else { + log.info("Request: lookupContentUUID/{}/{} from {}", uuidType, uuid, user); + } + + parameters.add(LOOKUP_UUID_PAIRS, String.join(UUID_TERM_DELIMITER, uuidType, uuid)); + + try { + return lookup(parameters, currentUser, true); + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error looking up UUID", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error looking up UUID."); + } + } + + public BaseQueryResponse lookupContentUUID(MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: lookupContentUUID (batch) from {} with params: {}", user, parameters); + } else { + log.info("Request: lookupContentUUID (batch) from {}", user); + } + + try { + return lookup(parameters, currentUser, true); + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error looking up UUID", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error looking up UUID."); + } + } + + private BaseQueryResponse lookup(MultiValueMap parameters, ProxiedUserDetails currentUser, boolean isContentLookup) throws QueryException { + String queryId = null; + + try { + // make sure the UUID lookup is valid + validateLookupParameters(parameters, isContentLookup); + + // update the parameters for query + updateQueryParameters(parameters, currentUser); + + // run the query + BaseQueryResponse nextResponse = queryManagementService.createAndNext(parameters.getFirst(QUERY_LOGIC_NAME), parameters, currentUser); + + // save the query id + queryId = nextResponse.getQueryId(); + + return nextResponse; + } finally { + // close the query if applicable + if (queryId != null) { + queryManagementService.close(queryId, currentUser); + } + } + } + + protected void validateLookupParameters(MultiValueMap parameters, boolean isContentLookup) throws BadRequestQueryException { + // get the requested uuids + List lookupUUIDPairs = parameters.get(LOOKUP_UUID_PAIRS); + if (lookupUUIDPairs != null && !lookupUUIDPairs.isEmpty()) { + + // make sure there aren't too many terms to lookup + if (uuidProperties.getBatchLookupLimit() > 0 && lookupUUIDPairs.size() <= uuidProperties.getBatchLookupLimit()) { + + // map the uuid pairs by key and value + MultiValueMap lookupUUIDMap = new LinkedMultiValueMap<>(lookupUUIDPairs.size()); + + String queryLogic = null; + + // validate each of the uuid pairs + for (String uuidPair : lookupUUIDPairs) { + String[] fieldValue = uuidPair.split(UUID_TERM_DELIMITER); + + // there should be a field and value present - no more, no less + if (fieldValue.length == 2) { + String field = fieldValue[0]; + String value = fieldValue[1]; + + // neither the field or value should be empty + if (!field.isEmpty() && !value.isEmpty()) { + + // is this a supported uuid type/field? + UUIDType uuidType = uuidProperties.getTypes().get(field.toUpperCase()); + if (uuidType != null) { + if (queryLogic == null) { + queryLogic = uuidType.getQueryLogic(); + } + // if we are mixing and matching query logics + else if (!queryLogic.equals(uuidType.getQueryLogic())) { + // TODO: This is a deal breaker + } + } + // if uuid type is null + else { + // TODO: This is a deal breaker + } + + lookupUUIDMap.add(field, value); + } + // if the field or value is empty + else { + // TODO: This is a deal breaker + } + } + // if there isn't a field AND a value + else { + // TODO: This is a deal breaker + } + } + + // let's create the query + String query = lookupUUIDMap.entrySet().stream() + .flatMap(entry -> entry.getValue().stream().map(value -> String.join(UUID_TERM_DELIMITER, entry.getKey(), value))) + .collect(Collectors.joining(" OR ")); + + // add the query logic name and query string to our parameters + parameters.put(QUERY_LOGIC_NAME, Collections.singletonList(queryLogic)); + parameters.put(QUERY_STRING, Collections.singletonList(query)); + } + // too many terms to lookup + else { + // TODO: This is a deal breaker + } + } + // if there are no lookup uuid pairs + else { + // TODO: This is a deal breaker + } + } + + protected void updateQueryParameters(MultiValueMap parameters, ProxiedUserDetails currentUser) { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + + if (uuidProperties.getColumnVisibility() != null) { + parameters.put(QueryParameters.QUERY_VISIBILITY, Collections.singletonList(uuidProperties.getColumnVisibility())); + } + + parameters.put(QUERY_SYNTAX, Collections.singletonList(LUCENE_UUID_SYNTAX)); + + // Override the extraneous query details + String userAuths; + if (parameters.containsKey(QueryParameters.QUERY_AUTHORIZATIONS)) { + userAuths = AuthorizationsUtil.downgradeUserAuths(currentUser, parameters.getFirst(QueryParameters.QUERY_AUTHORIZATIONS)); + } else { + userAuths = AuthorizationsUtil.buildUserAuthorizationString(currentUser); + } + parameters.put(QueryParameters.QUERY_AUTHORIZATIONS, Collections.singletonList(userAuths)); + + final String queryName = user + "-" + UUID.randomUUID(); + parameters.put(QueryParameters.QUERY_NAME, Collections.singletonList(queryName)); + + parameters.put(QueryParameters.QUERY_BEGIN, Collections.singletonList(uuidProperties.getBeginDate())); + + final Date endDate = DateUtils.addDays(new Date(), 2); + try { + parameters.put(QueryParameters.QUERY_END, Collections.singletonList(DefaultQueryParameters.formatDate(endDate))); + } catch (ParseException e) { + throw new RuntimeException("Unable to format new query end date: " + endDate); + } + + final Date expireDate = new Date(endDate.getTime() + 1000 * 60 * 60); + try { + parameters.put(QueryParameters.QUERY_EXPIRATION, Collections.singletonList(DefaultQueryParameters.formatDate(expireDate))); + } catch (ParseException e) { + throw new RuntimeException("Unable to format new query expr date: " + expireDate); + } + parameters.put(QueryParameters.QUERY_PERSISTENCE, Collections.singletonList(QueryPersistence.TRANSIENT.name())); + parameters.put(QueryParameters.QUERY_TRACE, Collections.singletonList("false")); + } +} diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java index d03d7ea0..1efc7b9b 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java @@ -31,7 +31,7 @@ @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceCancelTest extends AbstractQueryServiceTest { @Before public void setup() { diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java index 38bf637a..b02fec65 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java @@ -33,7 +33,7 @@ @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceCloseTest extends AbstractQueryServiceTest { @Before public void setup() { diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java index 66274a98..cce2ef89 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java @@ -37,7 +37,7 @@ @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceCreateTest extends AbstractQueryServiceTest { @Before public void setup() { diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java index bc75e46e..e0ac00a6 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java @@ -36,7 +36,7 @@ @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceDefineTest extends AbstractQueryServiceTest { @Before public void setup() { diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java index 18c40803..0c55606b 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java @@ -35,7 +35,7 @@ @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceDuplicateTest extends AbstractQueryServiceTest { @Before public void setup() { diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java index 31dca9f2..ee95da3e 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java @@ -35,7 +35,7 @@ @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceListTest extends AbstractQueryServiceTest { @Before public void setup() { @@ -419,10 +419,14 @@ public void testListQueryLogicSuccess() throws Exception { QueryLogicResponse qlResponse = response.getBody(); - Assert.assertEquals(2, qlResponse.getQueryLogicList().size()); + String[] expectedQueryLogics = new String[] {"AltEventQuery", "EventQuery", "LuceneUUIDEventQuery", "DiscoveryQuery"}; + + Assert.assertEquals(expectedQueryLogics.length, qlResponse.getQueryLogicList().size()); List qlNames = qlResponse.getQueryLogicList().stream().map(QueryLogicDescription::getName).sorted().collect(Collectors.toList()); - Assert.assertEquals(Arrays.asList("AltEventQuery", "EventQuery"), qlNames); + qlNames.removeAll(Arrays.asList(expectedQueryLogics)); + + Assert.assertTrue(qlNames.isEmpty()); } } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java index bc3dd7df..71214b6f 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java @@ -29,7 +29,7 @@ @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceNextTest extends AbstractQueryServiceTest { @Before public void setup() { diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java index 145ca515..39dc9433 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java @@ -32,7 +32,7 @@ @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServicePlanTest extends AbstractQueryServiceTest { @Autowired diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java index 94c9fa6a..fbb1c396 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java @@ -28,7 +28,7 @@ @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceRemoveTest extends AbstractQueryServiceTest { @Before public void setup() { diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java index 7247500d..8387feea 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java @@ -28,7 +28,7 @@ @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceResetTest extends AbstractQueryServiceTest { @Before public void setup() { diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java index 919317d3..927daffa 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java @@ -37,7 +37,7 @@ @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@ActiveProfiles({"QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceUpdateTest extends AbstractQueryServiceTest { @Before public void setup() { diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/uuid/QueryServiceLookupUUIDTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/uuid/QueryServiceLookupUUIDTest.java new file mode 100644 index 00000000..4b05baf0 --- /dev/null +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/uuid/QueryServiceLookupUUIDTest.java @@ -0,0 +1,99 @@ +package datawave.microservice.query.uuid; + +import datawave.marking.ColumnVisibilitySecurityMarking; +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.query.AbstractQueryServiceTest; +import datawave.microservice.query.DefaultQueryParameters; +import datawave.webservice.result.BaseQueryResponse; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; + +import static datawave.microservice.query.QueryParameters.QUERY_MAX_CONCURRENT_TASKS; +import static datawave.microservice.query.uuid.LookupUUIDService.LOOKUP_UUID_PAIRS; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +public class QueryServiceLookupUUIDTest extends AbstractQueryServiceTest { + @Before + public void setup() { + super.setup(); + } + + @After + public void teardown() throws Exception { + super.teardown(); + } + + @Ignore + @Test + public void testLookupUUIDSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + BaseQueryResponse response = lookupUUID(authUser, createUUIDParams(), "PAGE_TITLE", "anarchy"); + + } + + @Ignore + @Test + public void testBatchLookupUUIDSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + MultiValueMap params = createUUIDParams(); + params.add(LOOKUP_UUID_PAIRS, "PAGE_TITLE:anarchy"); + params.add(LOOKUP_UUID_PAIRS, "PAGE_TITLE:accessiblecomputing"); + + // create a valid query + long currentTimeMillis = System.currentTimeMillis(); + BaseQueryResponse response = batchLookupUUID(authUser, params); + + } + + protected MultiValueMap createUUIDParams() { + MultiValueMap map = new LinkedMultiValueMap<>(); + map.set(DefaultQueryParameters.QUERY_NAME, TEST_QUERY_NAME); + map.set(DefaultQueryParameters.QUERY_AUTHORIZATIONS, TEST_QUERY_AUTHORIZATIONS); + map.set(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, TEST_VISIBILITY_MARKING); + map.set(QUERY_MAX_CONCURRENT_TASKS, Integer.toString(1)); + return map; + } + + protected BaseQueryResponse batchLookupUUID(ProxiedUserDetails authUser, MultiValueMap map) { + UriComponents uri = createUri("lookupUUID"); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseQueryResponse.class); + + return resp.getBody(); + } + + protected BaseQueryResponse lookupUUID(ProxiedUserDetails authUser, MultiValueMap map, String uuidType, String uuid) { + UriComponents uri = createUri("lookupUUID/" + uuidType + "/" + uuid); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.GET, uri); + ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseQueryResponse.class); + + return resp.getBody(); + } +} diff --git a/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml b/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml deleted file mode 100644 index 6e915fee..00000000 --- a/query-microservices/query-service/service/src/test/resources/MyTestQueryLogicFactory.xml +++ /dev/null @@ -1,826 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 2611 - - - - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/query-microservices/query-service/service/src/test/resources/TestQueryLogicFactory.xml b/query-microservices/query-service/service/src/test/resources/TestQueryLogicFactory.xml new file mode 100644 index 00000000..86c1bc97 --- /dev/null +++ b/query-microservices/query-service/service/src/test/resources/TestQueryLogicFactory.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + diff --git a/query-microservices/query-service/service/src/test/resources/config/application-QueryStarterOverrides.yml b/query-microservices/query-service/service/src/test/resources/config/application-QueryStarterOverrides.yml new file mode 100644 index 00000000..28f61b5c --- /dev/null +++ b/query-microservices/query-service/service/src/test/resources/config/application-QueryStarterOverrides.yml @@ -0,0 +1,20 @@ +# Use this file to override properties from the QueryStarterDefaults profile (found in the datawave query starter). +# Make sure that you add the QueryStarterOverrides profile to your tests after the QueryStarterDefaults profile. + +datawave: + query: + awaitExecutorCreateResponse: false + nextCall: + resultPollInterval: 500 + statusUpdateInterval: 500 + expiration: + callTimeout: 5 + callTimeUnit: SECONDS + logic: + factory: + # Uncomment the following line to override the query logic beans to load + xmlBeansPath: "classpath:TestQueryLogicFactory.xml" + logics: + BaseEventQuery: + maxResults: 369 + auditType: "ACTIVE" diff --git a/query-microservices/query-service/service/src/test/resources/config/application.yml b/query-microservices/query-service/service/src/test/resources/config/application.yml index 1ba5fd54..b2dd5d8c 100644 --- a/query-microservices/query-service/service/src/test/resources/config/application.yml +++ b/query-microservices/query-service/service/src/test/resources/config/application.yml @@ -44,212 +44,3 @@ management: logging: level: root: FATAL - datawave.microservice.query: FATAL - -warehouse: - accumulo: - zookeepers: 'localhost:2181' - instanceName: 'my-instance-01' - username: 'root' - password: 'password' - statsd: - host: localhost - port: 8125 - tables: - shard: - name: 'shard' - index: - name: 'shardIndex' - reverseIndex: - name: 'shardReverseIndex' - dateIndex: - name: 'dateIndex' - metadata: - name: 'DatawaveMetadata' - model: - name: 'DatawaveMetadata' - defaults: - queryThreads: 100 - indexLookupThreads: 100 - dateIndexThreads: 20 - fullTableScanEnabled: false - baseIteratorPriority: 100 - maxIndexScanTimeMillis: 31536000000 - eventPerDayThreshold: 40000 - shardsPerDayThreshold: 20 - maxTermThreshold: 2000 - maxDepthThreshold: 2000 - maxUnfieldedExpansionThreshold: 50 - maxValueExpansionThreshold: 50 - maxOrExpansionThreshold: 500 - maxOrRangeThreshold: 10 - maxOrExpansionFstThreshold: 750 - maxFieldIndexRangeSplit: 16 - maxIvaratorSources: 20 - maxEvaluationPipelines: 16 - maxPipelineCachedResults: 16 - hdfsSiteConfigURLs: 'file:///etc/hadoop/conf/core-site.xml,file:///etc/hadoop/conf/hdfs-site.xml' - ivaratorFstHdfsBaseURIs: - - basePathURI: 'hdfs:///IvaratorCache' - ivaratorCacheBufferSize: 10000 - ivaratorMaxOpenFiles: 100 - ivaratorCacheScanPersistThreshold: 100000 - ivaratorCacheScanTimeoutMinutes: 60 - modelName: 'DATAWAVE' - -datawave: - metadata: - all-metadata-auths: - - PRIVATE,PUBLIC - type-substitutions: - "[datawave.data.type.DateType]": "datawave.data.type.RawDateType" - - query: - awaitExecutorCreateResponse: false - nextCall: - resultPollInterval: 500 - statusUpdateInterval: 500 - expiration: - callTimeout: 5 - callTimeUnit: SECONDS - parser: - skipTokenizeUnfieldedFields: - - "DOMETA" - tokenizedFields: - - "CONTENT" - uuidTypes: - - fieldName: "EVENT_ID" - definedView: "LuceneUUIDEventQuery" - allowedWildcardAfter: 28 - - fieldName: "UUID" - definedView: "LuceneUUIDEventQuery" - - fieldName: "PARENT_UUID" - definedView: "LuceneUUIDEventQuery" - - fieldName: "PAGE_ID" - definedView: "LuceneUUIDEventQuery" - - fieldName: "PAGE_TITLE" - definedView: "LuceneUUIDEventQuery" - logic: - factory: - enabled: true - # Uncomment the following line to override the query logic beans to load - xmlBeansPath: "classpath:MyTestQueryLogicFactory.xml" - - # The max page size that a user can request. 0 turns off this feature - maxPageSize: 10000 - - # The number of bytes at which a page will be returned, event if the pagesize has not been reached. 0 turns off this feature - pageByteTrigger: 0 - logics: - BaseEventQuery: - checkpointable: true - accumuloPassword: ${warehouse.accumulo.password} - tableName: ${warehouse.tables.shard.name} - dateIndexTableName: ${warehouse.tables.dateIndex.name} - defaultDateTypeName: "EVENT" - metadataTableName: ${warehouse.tables.metadata.name} - indexTableName: ${warehouse.tables.index.name} - reverseIndexTableName: ${warehouse.tables.reverseIndex.name} - maxResults: 369 - queryThreads: ${warehouse.defaults.queryThreads} - indexLookupThreads: ${warehouse.defaults.indexLookupThreads} - dateIndexThreads: ${warehouse.defaults.dateIndexThreads} - fullTableScanEnabled: ${warehouse.defaults.fullTableScanEnabled} - includeDataTypeAsField: false - disableIndexOnlyDocuments: false - indexOnlyFilterFunctionsEnabled: false - includeHierarchyFields: false - hierarchyFieldOptions: - "FOO": "BAR" - baseIteratorPriority: ${warehouse.defaults.baseIteratorPriority} - maxIndexScanTimeMillis: ${warehouse.defaults.maxIndexScanTimeMillis} - collapseUids: false - collapseUidsThreshold: -1 - useEnrichers: true - contentFieldNames: - - 'CONTENT' - realmSuffixExclusionPatterns: - - '<.*>$' - minimumSelectivity: .2 - enricherClassNames: - - 'datawave.query.enrich.DatawaveTermFrequencyEnricher' - useFilters: false - filterClassNames: - - 'foo.bar' - filterOptions: - 'bar': "foo" - auditType: "ACTIVE" - logicDescription: "Retrieve sharded events/documents, leveraging the global index tables as needed" - eventPerDayThreshold: ${warehouse.defaults.eventPerDayThreshold} - shardsPerDayThreshold: ${warehouse.defaults.shardsPerDayThreshold} - maxTermThreshold: ${warehouse.defaults.maxTermThreshold} - maxDepthThreshold: ${warehouse.defaults.maxDepthThreshold} - maxUnfieldedExpansionThreshold: ${warehouse.defaults.maxUnfieldedExpansionThreshold} - maxValueExpansionThreshold: ${warehouse.defaults.maxValueExpansionThreshold} - maxOrExpansionThreshold: ${warehouse.defaults.maxOrExpansionThreshold} - maxOrRangeThreshold: ${warehouse.defaults.maxOrRangeThreshold} - maxOrExpansionFstThreshold: ${warehouse.defaults.maxOrExpansionFstThreshold} - maxFieldIndexRangeSplit: ${warehouse.defaults.maxFieldIndexRangeSplit} - maxIvaratorSources: ${warehouse.defaults.maxIvaratorSources} - maxEvaluationPipelines: ${warehouse.defaults.maxEvaluationPipelines} - maxPipelineCachedResults: ${warehouse.defaults.maxPipelineCachedResults} - hdfsSiteConfigURLs: ${warehouse.defaults.hdfsSiteConfigURLs} - zookeeperConfig: ${warehouse.accumulo.zookeepers} - ivaratorCacheDirConfigs: - - basePathURI: "hdfs:///IvaratorCache" - ivaratorFstHdfsBaseURIs: ${warehouse.defaults.ivaratorFstHdfsBaseURIs} - ivaratorCacheBufferSize: ${warehouse.defaults.ivaratorCacheBufferSize} - ivaratorMaxOpenFiles: ${warehouse.defaults.ivaratorMaxOpenFiles} - ivaratorCacheScanPersistThreshold: ${warehouse.defaults.ivaratorCacheScanPersistThreshold} - ivaratorCacheScanTimeoutMinutes: ${warehouse.defaults.ivaratorCacheScanTimeoutMinutes} - eventQueryDataDecoratorTransformer: - requestedDecorators: - - "CSV" - - "WIKIPEDIA" - dataDecorators: - "CSV": - "EVENT_ID": "https://localhost:8443/DataWave/Query/lookupUUID/EVENT_ID?uuid=@field_value@&parameters=data.decorators:CSV" - "UUID": "https://localhost:8443/DataWave/Query/lookupUUID/UUID?uuid=@field_value@&parameters=data.decorators:CSV" - "PARENT_UUID": "https://localhost:8443/DataWave/Query/lookupUUID/PARENT_UUID?uuid=@field_value@&parameters=data.decorators:CSV" - "WIKIPEDIA": - "PAGE_ID": "https://localhost:8443/DataWave/Query/lookupUUID/PAGE_ID?uuid=@field_value@&parameters=data.decorators:WIKIPEDIA" - "PAGE_TITLE": "https://localhost:8443/DataWave/Query/lookupUUID/PAGE_TITLE?uuid=@field_value@&parameters=data.decorators:WIKIPEDIA" - modelTableName: ${warehouse.tables.model.name} - modelName: ${warehouse.defaults.modelName} - querySyntaxParsers: - JEXL: "" - LUCENE: "LuceneToJexlQueryParser" - LUCENE-UUID: "LuceneToJexlUUIDQueryParser" - TOKENIZED-LUCENE: "TokenizedLuceneToJexlQueryParser" - sendTimingToStatsd: false - collectQueryMetrics: true - logTimingDetails: false - statsdHost: ${warehouse.statsd.host} - statsdPort: ${warehouse.statsd.port} - evaluationOnlyFields: "" - maxConcurrentTasks: 10 - requiredRoles: - - "AuthorizedUser" - - DiscoveryQuery: - checkpointable: true - tableName: ${warehouse.tables.shard.name} - indexTableName: ${warehouse.tables.index.name} - reverseIndexTableName: ${warehouse.tables.reverseIndex.name} - metadataTableName: ${warehouse.tables.metadata.name} - modelTableName: ${warehouse.tables.model.name} - modelName: ${warehouse.defaults.modelName} - fullTableScanEnabled: ${warehouse.defaults.fullTableScanEnabled} - allowLeadingWildcard: true - auditType: "NONE" - maxResults: -1 - -query: - messaging: - claimCheck: - enabled: false - -audit-client: - discovery: - enabled: false - uri: '${AUDIT_SERVER_URL:http://localhost:11111/audit}' From c4d90457d9a74d2f5a79f9f2c654b801ce5cf35f Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 17 Dec 2021 15:22:24 -0500 Subject: [PATCH 113/218] Got content lookups working. Updated base64 decoding logic to avoid the use of deprecated infinispan library. --- .../microservice/query/QueryController.java | 16 +- .../query/QueryManagementService.java | 3 +- .../query/uuid/LookupService.java | 434 ++++++++++++++++++ .../query/uuid/LookupUUIDService.java | 274 ----------- .../query/QueryServiceListTest.java | 2 +- .../uuid/QueryServiceLookupUUIDTest.java | 2 +- 6 files changed, 446 insertions(+), 285 deletions(-) create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/uuid/LookupService.java delete mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/uuid/LookupUUIDService.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 302680b8..a05abd8a 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -2,7 +2,7 @@ import com.codahale.metrics.annotation.Timed; import datawave.microservice.authorization.user.ProxiedUserDetails; -import datawave.microservice.query.uuid.LookupUUIDService; +import datawave.microservice.query.uuid.LookupService; import datawave.microservice.query.web.annotation.EnrichQueryMetrics; import datawave.webservice.query.exception.QueryException; import datawave.webservice.result.BaseQueryResponse; @@ -25,11 +25,11 @@ @RequestMapping(path = "/v1", produces = MediaType.APPLICATION_JSON_VALUE) public class QueryController { private final QueryManagementService queryManagementService; - private final LookupUUIDService lookupUUIDService; + private final LookupService lookupService; - public QueryController(QueryManagementService queryManagementService, LookupUUIDService lookupUUIDService) { + public QueryController(QueryManagementService queryManagementService, LookupService lookupService) { this.queryManagementService = queryManagementService; - this.lookupUUIDService = lookupUUIDService; + this.lookupService = lookupService; } @Timed(name = "dw.query.listQueryLogic", absolute = true) @@ -78,7 +78,7 @@ public GenericResponse predict(@PathVariable String queryLogic, @Request "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public BaseQueryResponse lookupUUID(@PathVariable(required = false) String uuidType, @PathVariable(required = false) String uuid, @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { - return lookupUUIDService.lookupUUID(uuidType, uuid, parameters, currentUser); + return lookupService.lookupUUID(uuidType, uuid, parameters, currentUser); } @Timed(name = "dw.query.lookupUUIDBatch", absolute = true) @@ -86,7 +86,7 @@ public BaseQueryResponse lookupUUID(@PathVariable(required = false) String uuidT "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public BaseQueryResponse lookupUUIDBatch(@PathVariable(required = false) String uuidType, @PathVariable(required = false) String uuid, @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { - return lookupUUIDService.lookupUUID(parameters, currentUser); + return lookupService.lookupUUID(parameters, currentUser); } @Timed(name = "dw.query.lookupContentUUID", absolute = true) @@ -94,7 +94,7 @@ public BaseQueryResponse lookupUUIDBatch(@PathVariable(required = false) String "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public BaseQueryResponse lookupContentUUID(@PathVariable(required = false) String uuidType, @PathVariable(required = false) String uuid, @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { - return lookupUUIDService.lookupContentUUID(uuidType, uuid, parameters, currentUser); + return lookupService.lookupContentUUID(uuidType, uuid, parameters, currentUser); } @Timed(name = "dw.query.lookupContentUUIDBatch", absolute = true) @@ -102,7 +102,7 @@ public BaseQueryResponse lookupContentUUID(@PathVariable(required = false) Strin "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public BaseQueryResponse lookupContentUUIDBatch(@PathVariable(required = false) String uuidType, @PathVariable(required = false) String uuid, @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { - return lookupUUIDService.lookupContentUUID(parameters, currentUser); + return lookupService.lookupContentUUID(parameters, currentUser); } @Timed(name = "dw.query.createAndNext", absolute = true) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index d4647414..fd645286 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -2240,8 +2240,9 @@ protected void validateParameters(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser) + throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: lookupUUID/{}/{} from {} with params: {}", uuidType, uuid, user, parameters); + } else { + log.info("Request: lookupUUID/{}/{} from {}", uuidType, uuid, user); + } + + parameters.add(LOOKUP_UUID_PAIRS, String.join(LOOKUP_KEY_VALUE_DELIMITER, uuidType, uuid)); + + try { + return lookup(parameters, currentUser, false); + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error looking up UUID", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error looking up UUID."); + } + } + + public BaseQueryResponse lookupUUID(MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: lookupUUID (batch) from {} with params: {}", user, parameters); + } else { + log.info("Request: lookupUUID (batch) from {}", user); + } + + try { + return lookup(parameters, currentUser, false); + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error looking up UUID", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error looking up UUID."); + } + } + + public BaseQueryResponse lookupContentUUID(String uuidType, String uuid, MultiValueMap parameters, ProxiedUserDetails currentUser) + throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: lookupContentUUID/{}/{} from {} with params: {}", uuidType, uuid, user, parameters); + } else { + log.info("Request: lookupContentUUID/{}/{} from {}", uuidType, uuid, user); + } + + parameters.add(LOOKUP_UUID_PAIRS, String.join(LOOKUP_KEY_VALUE_DELIMITER, uuidType, uuid)); + + try { + // first lookup the UUIDs, then get the content for each UUID + return lookup(parameters, currentUser, true); + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error looking up UUID content", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error looking up UUID content."); + } + } + + public BaseQueryResponse lookupContentUUID(MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: lookupContentUUID (batch) from {} with params: {}", user, parameters); + } else { + log.info("Request: lookupContentUUID (batch) from {}", user); + } + + try { + // first lookup the UUIDs, then get the content for each UUID + return lookup(parameters, currentUser, true); + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error looking up UUID content", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error looking up UUID content."); + } + } + + private BaseQueryResponse lookup(MultiValueMap parameters, ProxiedUserDetails currentUser, boolean isContentLookup) throws QueryException { + List lookupTerms = parameters.get(LOOKUP_UUID_PAIRS); + if (lookupTerms == null || lookupTerms.isEmpty()) { + log.error("Unable to validate lookupUUID parameters: No UUID Pairs"); + throw new BadRequestQueryException(DatawaveErrorCode.MISSING_REQUIRED_PARAMETER); + } + + MultiValueMap lookupTermMap = new LinkedMultiValueMap<>(); + + // validate the lookup terms + LookupQueryLogic lookupQueryLogic = validateLookupTerms(lookupTerms, lookupTermMap); + + BaseQueryResponse response = null; + + boolean isEventLookupRequired = lookupQueryLogic.isEventLookupRequired(lookupTermMap); + + // do the event lookup if necessary + if (!isContentLookup || isEventLookupRequired) { + response = lookupEvents(lookupTermMap, lookupQueryLogic, new LinkedMultiValueMap<>(parameters), currentUser); + } + + // perform the content lookup if necessary + if (isContentLookup) { + Set contentLookupTerms; + if (!isEventLookupRequired) { + contentLookupTerms = lookupQueryLogic.getContentLookupTerms(lookupTermMap); + } else { + contentLookupTerms = getContentLookupTerms(response); + } + + response = lookupContent(contentLookupTerms, new LinkedMultiValueMap<>(parameters), currentUser); + } + + return response; + } + + private BaseQueryResponse lookupEvents(MultiValueMap lookupTermMap, LookupQueryLogic lookupQueryLogic, + MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + String queryId = null; + try { + // add the query logic name and query string to our parameters + parameters.put(QUERY_LOGIC_NAME, Collections.singletonList(lookupQueryLogic.getLogicName())); + parameters.put(QUERY_STRING, Collections.singletonList(lookupQueryLogic.createQueryFromLookupTerms(lookupTermMap))); + + // update the parameters for query + updateParametersForEventQuery(parameters, currentUser); + + // run the query + BaseQueryResponse nextResponse = queryManagementService.createAndNext(parameters.getFirst(QUERY_LOGIC_NAME), parameters, currentUser); + + // save the query id + queryId = nextResponse.getQueryId(); + + return nextResponse; + } finally { + // close the query if applicable + if (queryId != null) { + queryManagementService.close(queryId, currentUser); + } + } + } + + protected LookupQueryLogic validateLookupTerms(List lookupUUIDPairs, MultiValueMap lookupUUIDMap) throws QueryException { + String queryLogicName = null; + + // make sure there aren't too many terms to lookup + if (uuidProperties.getBatchLookupLimit() > 0 && lookupUUIDPairs.size() <= uuidProperties.getBatchLookupLimit()) { + + // validate each of the uuid pairs + for (String uuidPair : lookupUUIDPairs) { + String[] fieldValue = uuidPair.split(LOOKUP_KEY_VALUE_DELIMITER); + + // there should be a field and value present - no more, no less + if (fieldValue.length == 2) { + String field = fieldValue[0]; + String value = fieldValue[1]; + + // neither the field or value should be empty + if (!field.isEmpty() && !value.isEmpty()) { + + // is this a supported uuid type/field? + UUIDType uuidType = uuidProperties.getTypes().get(field.toUpperCase()); + if (uuidType != null) { + if (queryLogicName == null) { + queryLogicName = uuidType.getQueryLogic(); + } + // if we are mixing and matching query logics + else if (!queryLogicName.equals(uuidType.getQueryLogic())) { + // TODO: This is a deal breaker + } + } + // if uuid type is null + else { + // TODO: This is a deal breaker + } + + lookupUUIDMap.add(field, value); + } + // if the field or value is empty + else { + // TODO: This is a deal breaker + } + } + // if there isn't a field AND a value + else { + // TODO: This is a deal breaker + } + } + } + // too many terms to lookup + else { + // TODO: This is a deal breaker + } + + try { + QueryLogic queryLogic = queryLogicFactory.getQueryLogic(queryLogicName); + + if (queryLogic instanceof LookupQueryLogic) { + return (LookupQueryLogic) queryLogic; + } else { + log.error("Lookup UUID can only be run with a LookupQueryLogic"); + throw new BadRequestQueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, "Lookup UUID can only be run with a LookupQueryLogic"); + } + } catch (CloneNotSupportedException e) { + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unable to create instance of the requested query logic " + queryLogicName); + } + } + + protected void updateParametersForEventQuery(MultiValueMap parameters, ProxiedUserDetails currentUser) { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + + if (uuidProperties.getColumnVisibility() != null) { + parameters.set(QueryParameters.QUERY_VISIBILITY, uuidProperties.getColumnVisibility()); + } + + parameters.set(QUERY_SYNTAX, LUCENE_UUID_SYNTAX); + + // Override the extraneous query details + String userAuths; + if (parameters.containsKey(QueryParameters.QUERY_AUTHORIZATIONS)) { + userAuths = AuthorizationsUtil.downgradeUserAuths(currentUser, parameters.getFirst(QueryParameters.QUERY_AUTHORIZATIONS)); + } else { + userAuths = AuthorizationsUtil.buildUserAuthorizationString(currentUser); + } + parameters.set(QueryParameters.QUERY_AUTHORIZATIONS, userAuths); + + final String queryName = user + "-" + UUID.randomUUID(); + parameters.set(QueryParameters.QUERY_NAME, queryName); + + parameters.set(QueryParameters.QUERY_BEGIN, uuidProperties.getBeginDate()); + + final Date endDate = DateUtils.addDays(new Date(), 2); + try { + parameters.set(QueryParameters.QUERY_END, DefaultQueryParameters.formatDate(endDate)); + } catch (ParseException e) { + throw new RuntimeException("Unable to format new query end date: " + endDate); + } + + // TODO: This may not be needed? + final Date expireDate = new Date(endDate.getTime() + 1000 * 60 * 60); + try { + parameters.set(QueryParameters.QUERY_EXPIRATION, DefaultQueryParameters.formatDate(expireDate)); + } catch (ParseException e) { + throw new RuntimeException("Unable to format new query expr date: " + expireDate); + } + } + + private Set getContentLookupTerms(BaseQueryResponse response) { + Set contentQueries = new HashSet<>(); + + if (response instanceof EventQueryResponseBase) { + ((EventQueryResponseBase) response).getEvents().forEach(e -> contentQueries.add(createContentLookupTerm(e.getMetadata()))); + } + + return contentQueries; + } + + private String createContentLookupTerm(Metadata eventMetadata) { + return DOCUMENT_FIELD_PREFIX + + String.join(CONTENT_QUERY_VALUE_DELIMITER, eventMetadata.getRow(), eventMetadata.getDataType(), eventMetadata.getInternalId()); + } + + private BaseQueryResponse lookupContent(Set contentLookupTerms, MultiValueMap parameters, ProxiedUserDetails currentUser) + throws QueryException { + // create queries from the content lookup terms + List contentQueries = createContentQueries(contentLookupTerms); + + EventQueryResponseBase mergedResponse = null; + for (String contentQuery : contentQueries) { + // set the content query string + parameters.put(QUERY_STRING, Collections.singletonList(contentQuery)); + + // update parameters for the query + updateParametersForContentQuery(parameters, currentUser); + + // run the query + EventQueryResponseBase contentQueryResponse = runContentQuery(parameters, currentUser); + + if (contentQueryResponse != null) { + if (mergedResponse == null) { + mergedResponse = contentQueryResponse; + } else { + mergedResponse.merge(contentQueryResponse); + } + } + + } + return mergedResponse; + } + + private List createContentQueries(Set contentLookupTerms) { + List contentQueries = new ArrayList<>(); + + Iterables.partition(contentLookupTerms, uuidProperties.getBatchLookupLimit()) + .forEach(termBatch -> contentQueries.add(String.join(CONTENT_QUERY_TERM_SEPARATOR, termBatch))); + + return contentQueries; + } + + protected void updateParametersForContentQuery(MultiValueMap parameters, ProxiedUserDetails currentUser) { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + + // all content queries use the same query logic + parameters.put(QUERY_LOGIC_NAME, Collections.singletonList(uuidProperties.getContentQueryLogicName())); + + parameters.set(QueryParameters.QUERY_NAME, user + '-' + UUID.randomUUID()); + + parameters.set(QueryParameters.QUERY_BEGIN, uuidProperties.getBeginDate()); + + final Date endDate = new Date(); + try { + parameters.set(QueryParameters.QUERY_END, DefaultQueryParameters.formatDate(endDate)); + } catch (ParseException e1) { + throw new RuntimeException("Error formatting end date: " + endDate); + } + + final String userAuths = AuthorizationsUtil.buildUserAuthorizationString(currentUser); + parameters.set(QueryParameters.QUERY_AUTHORIZATIONS, userAuths); + + final Date expireDate = new Date(endDate.getTime() + 1000 * 60 * 60); + try { + parameters.set(QueryParameters.QUERY_EXPIRATION, DefaultQueryParameters.formatDate(expireDate)); + } catch (ParseException e1) { + throw new RuntimeException("Error formatting expr date: " + expireDate); + } + } + + protected EventQueryResponseBase runContentQuery(MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + EventQueryResponseBase mergedResponse = null; + String queryId = null; + boolean isQueryFinished = false; + + do { + BaseQueryResponse nextResponse = null; + try { + if (queryId == null) { + nextResponse = queryManagementService.createAndNext(parameters.getFirst(QUERY_LOGIC_NAME), parameters, currentUser); + queryId = nextResponse.getQueryId(); + } else { + nextResponse = queryManagementService.next(queryId, currentUser); + } + } catch (NoResultsQueryException e) { + log.info("No results found for content query '{}'", parameters.getFirst(QUERY_STRING)); + } catch (QueryException e) { + log.info("Encountered error while getting results for content query '{}'", parameters.getFirst(QUERY_STRING)); + } + + if (nextResponse instanceof EventQueryResponseBase) { + EventQueryResponseBase nextEventQueryResponse = (EventQueryResponseBase) nextResponse; + + // Prevent NPE due to attempted merge when total events is null + if (nextEventQueryResponse.getTotalEvents() == null) { + final Long totalEvents = nextEventQueryResponse.getReturnedEvents(); + nextEventQueryResponse.setTotalEvents((totalEvents != null) ? totalEvents : 0L); + } + + // save or update the merged response + if (mergedResponse == null) { + mergedResponse = nextEventQueryResponse; + } else { + mergedResponse.merge(nextEventQueryResponse); + } + } else { + isQueryFinished = true; + } + } while (!isQueryFinished); + + return mergedResponse; + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/uuid/LookupUUIDService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/uuid/LookupUUIDService.java deleted file mode 100644 index 4afed6c2..00000000 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/uuid/LookupUUIDService.java +++ /dev/null @@ -1,274 +0,0 @@ -package datawave.microservice.query.uuid; - -import datawave.microservice.authorization.user.ProxiedUserDetails; -import datawave.microservice.authorization.util.AuthorizationsUtil; -import datawave.microservice.query.DefaultQueryParameters; -import datawave.microservice.query.QueryManagementService; -import datawave.microservice.query.QueryParameters; -import datawave.microservice.query.QueryPersistence; -import datawave.microservice.query.config.QueryProperties; -import datawave.query.data.UUIDType; -import datawave.security.util.ProxiedEntityUtils; -import datawave.webservice.query.exception.BadRequestQueryException; -import datawave.webservice.query.exception.DatawaveErrorCode; -import datawave.webservice.query.exception.QueryException; -import datawave.webservice.query.result.event.ResponseObjectFactory; -import datawave.webservice.result.BaseQueryResponse; -import org.apache.commons.lang.time.DateUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; - -import java.text.ParseException; -import java.util.Collections; -import java.util.Date; -import java.util.List; -import java.util.UUID; -import java.util.stream.Collectors; - -import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; -import static datawave.microservice.query.QueryParameters.QUERY_STRING; -import static datawave.query.QueryParameters.QUERY_SYNTAX; - -@Service -public class LookupUUIDService { - private final Logger log = LoggerFactory.getLogger(this.getClass()); - - public static final String LOOKUP_UUID_PAIRS = "uuidPairs"; - public static final String LUCENE_UUID_SYNTAX = "LUCENE-UUID"; - - protected static final String UUID_TERM_DELIMITER = ":"; - - private final QueryProperties queryProperties; - private final LookupUUIDProperties uuidProperties; - - private final ResponseObjectFactory responseObjectFactory; - private final QueryManagementService queryManagementService; - - public LookupUUIDService(QueryProperties queryProperties, LookupUUIDProperties uuidProperties, ResponseObjectFactory responseObjectFactory, - QueryManagementService queryManagementService) { - this.queryProperties = queryProperties; - this.uuidProperties = uuidProperties; - this.responseObjectFactory = responseObjectFactory; - this.queryManagementService = queryManagementService; - } - - public BaseQueryResponse lookupUUID(String uuidType, String uuid, MultiValueMap parameters, ProxiedUserDetails currentUser) - throws QueryException { - String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); - if (log.isDebugEnabled()) { - log.info("Request: lookupUUID/{}/{} from {} with params: {}", uuidType, uuid, user, parameters); - } else { - log.info("Request: lookupUUID/{}/{} from {}", uuidType, uuid, user); - } - - parameters.add(LOOKUP_UUID_PAIRS, String.join(UUID_TERM_DELIMITER, uuidType, uuid)); - - try { - return lookup(parameters, currentUser, false); - } catch (QueryException e) { - throw e; - } catch (Exception e) { - log.error("Unknown error looking up UUID", e); - throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error looking up UUID."); - } - } - - public BaseQueryResponse lookupUUID(MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { - String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); - if (log.isDebugEnabled()) { - log.info("Request: lookupUUID (batch) from {} with params: {}", user, parameters); - } else { - log.info("Request: lookupUUID (batch) from {}", user); - } - - try { - return lookup(parameters, currentUser, false); - } catch (QueryException e) { - throw e; - } catch (Exception e) { - log.error("Unknown error looking up UUID", e); - throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error looking up UUID."); - } - } - - public BaseQueryResponse lookupContentUUID(String uuidType, String uuid, MultiValueMap parameters, ProxiedUserDetails currentUser) - throws QueryException { - String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); - if (log.isDebugEnabled()) { - log.info("Request: lookupContentUUID/{}/{} from {} with params: {}", uuidType, uuid, user, parameters); - } else { - log.info("Request: lookupContentUUID/{}/{} from {}", uuidType, uuid, user); - } - - parameters.add(LOOKUP_UUID_PAIRS, String.join(UUID_TERM_DELIMITER, uuidType, uuid)); - - try { - return lookup(parameters, currentUser, true); - } catch (QueryException e) { - throw e; - } catch (Exception e) { - log.error("Unknown error looking up UUID", e); - throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error looking up UUID."); - } - } - - public BaseQueryResponse lookupContentUUID(MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { - String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); - if (log.isDebugEnabled()) { - log.info("Request: lookupContentUUID (batch) from {} with params: {}", user, parameters); - } else { - log.info("Request: lookupContentUUID (batch) from {}", user); - } - - try { - return lookup(parameters, currentUser, true); - } catch (QueryException e) { - throw e; - } catch (Exception e) { - log.error("Unknown error looking up UUID", e); - throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error looking up UUID."); - } - } - - private BaseQueryResponse lookup(MultiValueMap parameters, ProxiedUserDetails currentUser, boolean isContentLookup) throws QueryException { - String queryId = null; - - try { - // make sure the UUID lookup is valid - validateLookupParameters(parameters, isContentLookup); - - // update the parameters for query - updateQueryParameters(parameters, currentUser); - - // run the query - BaseQueryResponse nextResponse = queryManagementService.createAndNext(parameters.getFirst(QUERY_LOGIC_NAME), parameters, currentUser); - - // save the query id - queryId = nextResponse.getQueryId(); - - return nextResponse; - } finally { - // close the query if applicable - if (queryId != null) { - queryManagementService.close(queryId, currentUser); - } - } - } - - protected void validateLookupParameters(MultiValueMap parameters, boolean isContentLookup) throws BadRequestQueryException { - // get the requested uuids - List lookupUUIDPairs = parameters.get(LOOKUP_UUID_PAIRS); - if (lookupUUIDPairs != null && !lookupUUIDPairs.isEmpty()) { - - // make sure there aren't too many terms to lookup - if (uuidProperties.getBatchLookupLimit() > 0 && lookupUUIDPairs.size() <= uuidProperties.getBatchLookupLimit()) { - - // map the uuid pairs by key and value - MultiValueMap lookupUUIDMap = new LinkedMultiValueMap<>(lookupUUIDPairs.size()); - - String queryLogic = null; - - // validate each of the uuid pairs - for (String uuidPair : lookupUUIDPairs) { - String[] fieldValue = uuidPair.split(UUID_TERM_DELIMITER); - - // there should be a field and value present - no more, no less - if (fieldValue.length == 2) { - String field = fieldValue[0]; - String value = fieldValue[1]; - - // neither the field or value should be empty - if (!field.isEmpty() && !value.isEmpty()) { - - // is this a supported uuid type/field? - UUIDType uuidType = uuidProperties.getTypes().get(field.toUpperCase()); - if (uuidType != null) { - if (queryLogic == null) { - queryLogic = uuidType.getQueryLogic(); - } - // if we are mixing and matching query logics - else if (!queryLogic.equals(uuidType.getQueryLogic())) { - // TODO: This is a deal breaker - } - } - // if uuid type is null - else { - // TODO: This is a deal breaker - } - - lookupUUIDMap.add(field, value); - } - // if the field or value is empty - else { - // TODO: This is a deal breaker - } - } - // if there isn't a field AND a value - else { - // TODO: This is a deal breaker - } - } - - // let's create the query - String query = lookupUUIDMap.entrySet().stream() - .flatMap(entry -> entry.getValue().stream().map(value -> String.join(UUID_TERM_DELIMITER, entry.getKey(), value))) - .collect(Collectors.joining(" OR ")); - - // add the query logic name and query string to our parameters - parameters.put(QUERY_LOGIC_NAME, Collections.singletonList(queryLogic)); - parameters.put(QUERY_STRING, Collections.singletonList(query)); - } - // too many terms to lookup - else { - // TODO: This is a deal breaker - } - } - // if there are no lookup uuid pairs - else { - // TODO: This is a deal breaker - } - } - - protected void updateQueryParameters(MultiValueMap parameters, ProxiedUserDetails currentUser) { - String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); - - if (uuidProperties.getColumnVisibility() != null) { - parameters.put(QueryParameters.QUERY_VISIBILITY, Collections.singletonList(uuidProperties.getColumnVisibility())); - } - - parameters.put(QUERY_SYNTAX, Collections.singletonList(LUCENE_UUID_SYNTAX)); - - // Override the extraneous query details - String userAuths; - if (parameters.containsKey(QueryParameters.QUERY_AUTHORIZATIONS)) { - userAuths = AuthorizationsUtil.downgradeUserAuths(currentUser, parameters.getFirst(QueryParameters.QUERY_AUTHORIZATIONS)); - } else { - userAuths = AuthorizationsUtil.buildUserAuthorizationString(currentUser); - } - parameters.put(QueryParameters.QUERY_AUTHORIZATIONS, Collections.singletonList(userAuths)); - - final String queryName = user + "-" + UUID.randomUUID(); - parameters.put(QueryParameters.QUERY_NAME, Collections.singletonList(queryName)); - - parameters.put(QueryParameters.QUERY_BEGIN, Collections.singletonList(uuidProperties.getBeginDate())); - - final Date endDate = DateUtils.addDays(new Date(), 2); - try { - parameters.put(QueryParameters.QUERY_END, Collections.singletonList(DefaultQueryParameters.formatDate(endDate))); - } catch (ParseException e) { - throw new RuntimeException("Unable to format new query end date: " + endDate); - } - - final Date expireDate = new Date(endDate.getTime() + 1000 * 60 * 60); - try { - parameters.put(QueryParameters.QUERY_EXPIRATION, Collections.singletonList(DefaultQueryParameters.formatDate(expireDate))); - } catch (ParseException e) { - throw new RuntimeException("Unable to format new query expr date: " + expireDate); - } - parameters.put(QueryParameters.QUERY_PERSISTENCE, Collections.singletonList(QueryPersistence.TRANSIENT.name())); - parameters.put(QueryParameters.QUERY_TRACE, Collections.singletonList("false")); - } -} diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java index ee95da3e..227a1297 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java @@ -419,7 +419,7 @@ public void testListQueryLogicSuccess() throws Exception { QueryLogicResponse qlResponse = response.getBody(); - String[] expectedQueryLogics = new String[] {"AltEventQuery", "EventQuery", "LuceneUUIDEventQuery", "DiscoveryQuery"}; + String[] expectedQueryLogics = new String[] {"AltEventQuery", "EventQuery", "LuceneUUIDEventQuery", "DiscoveryQuery", "ContentQuery"}; Assert.assertEquals(expectedQueryLogics.length, qlResponse.getQueryLogicList().size()); diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/uuid/QueryServiceLookupUUIDTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/uuid/QueryServiceLookupUUIDTest.java index 4b05baf0..4f2695f3 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/uuid/QueryServiceLookupUUIDTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/uuid/QueryServiceLookupUUIDTest.java @@ -22,7 +22,7 @@ import org.springframework.web.util.UriComponents; import static datawave.microservice.query.QueryParameters.QUERY_MAX_CONCURRENT_TASKS; -import static datawave.microservice.query.uuid.LookupUUIDService.LOOKUP_UUID_PAIRS; +import static datawave.microservice.query.uuid.LookupService.LOOKUP_UUID_PAIRS; @RunWith(SpringRunner.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) From d8c3892b1beff4d8ea1d287cac11411171beeb7a Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 17 Dec 2021 15:45:25 -0500 Subject: [PATCH 114/218] Removed placeholder predict response. --- .../datawave/microservice/query/QueryManagementService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index fd645286..8fd5b3a9 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -433,7 +433,7 @@ public GenericResponse predict(String queryLogicName, MultiValueMap response = new GenericResponse<>(); response.setResult(queryPrediction); From f8def3744fdee229da8863214d0133451f59118b Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 29 Dec 2021 11:20:02 -0500 Subject: [PATCH 115/218] Added tests for LookupService --- .../microservice/query/QueryController.java | 2 +- .../query/{uuid => lookup}/LookupService.java | 229 ++++- .../query/lookup/LookupServiceTest.java | 804 ++++++++++++++++++ .../uuid/QueryServiceLookupUUIDTest.java | 99 --- .../application-QueryStarterOverrides.yml | 21 + 5 files changed, 1015 insertions(+), 140 deletions(-) rename query-microservices/query-service/service/src/main/java/datawave/microservice/query/{uuid => lookup}/LookupService.java (63%) create mode 100644 query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java delete mode 100644 query-microservices/query-service/service/src/test/java/datawave/microservice/query/uuid/QueryServiceLookupUUIDTest.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index a05abd8a..1235f2b6 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -2,7 +2,7 @@ import com.codahale.metrics.annotation.Timed; import datawave.microservice.authorization.user.ProxiedUserDetails; -import datawave.microservice.query.uuid.LookupService; +import datawave.microservice.query.lookup.LookupService; import datawave.microservice.query.web.annotation.EnrichQueryMetrics; import datawave.webservice.query.exception.QueryException; import datawave.webservice.result.BaseQueryResponse; diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/uuid/LookupService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/lookup/LookupService.java similarity index 63% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/uuid/LookupService.java rename to query-microservices/query-service/service/src/main/java/datawave/microservice/query/lookup/LookupService.java index 5155928f..0a38693d 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/uuid/LookupService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/lookup/LookupService.java @@ -1,4 +1,4 @@ -package datawave.microservice.query.uuid; +package datawave.microservice.query.lookup; import com.google.common.collect.Iterables; import datawave.microservice.authorization.user.ProxiedUserDetails; @@ -15,10 +15,13 @@ import datawave.webservice.query.exception.DatawaveErrorCode; import datawave.webservice.query.exception.NoResultsQueryException; import datawave.webservice.query.exception.QueryException; +import datawave.webservice.query.exception.TimeoutQueryException; +import datawave.webservice.query.exception.UnauthorizedQueryException; import datawave.webservice.query.result.event.Metadata; import datawave.webservice.result.BaseQueryResponse; import datawave.webservice.result.EventQueryResponseBase; import org.apache.commons.lang.time.DateUtils; +import org.apache.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; @@ -51,17 +54,54 @@ public class LookupService { private static final String CONTENT_QUERY_TERM_SEPARATOR = " "; private static final String DOCUMENT_FIELD_PREFIX = "DOCUMENT" + CONTENT_QUERY_TERM_DELIMITER; - private final LookupUUIDProperties uuidProperties; + private final LookupProperties lookupProperties; private final QueryLogicFactory queryLogicFactory; private final QueryManagementService queryManagementService; - public LookupService(LookupUUIDProperties uuidProperties, QueryLogicFactory queryLogicFactory, QueryManagementService queryManagementService) { - this.uuidProperties = uuidProperties; + public LookupService(LookupProperties lookupProperties, QueryLogicFactory queryLogicFactory, QueryManagementService queryManagementService) { + this.lookupProperties = lookupProperties; this.queryLogicFactory = queryLogicFactory; this.queryManagementService = queryManagementService; } + /** + * Creates an event lookup query using the query logic associated with the given uuid type and parameters, and returns the first page of results. + *

+ * Lookup queries will start running immediately.
+ * Auditing is performed before the query is started.
+ * After the first page is returned, the query will be closed. + * + * @param uuidType + * the uuid type, not null + * @param uuid + * the uuid, not null + * @param parameters + * the query parameters, not null + * @param currentUser + * the user who called this method, not null + * @return a base query response containing the first page of results + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws TimeoutQueryException + * if the next call times out + * @throws NoResultsQueryException + * if no query results are found + * @throws QueryException + * if this next task is rejected by the executor + * @throws QueryException + * if there is an unknown error + */ public BaseQueryResponse lookupUUID(String uuidType, String uuid, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); @@ -83,6 +123,40 @@ public BaseQueryResponse lookupUUID(String uuidType, String uuid, MultiValueMap< } } + /** + * Creates a batch event lookup query using the query logic associated with the given uuid type(s) and parameters, and returns the first page of results. + *

+ * Lookup queries will start running immediately.
+ * Auditing is performed before the query is started.
+ * Each of the uuid pairs must map to the same query logic.
+ * After the first page is returned, the query will be closed. + * + * @param parameters + * the query parameters, not null + * @param currentUser + * the user who called this method, not null + * @return a base query response containing the first page of results + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws TimeoutQueryException + * if the next call times out + * @throws NoResultsQueryException + * if no query results are found + * @throws QueryException + * if this next task is rejected by the executor + * @throws QueryException + * if there is an unknown error + */ public BaseQueryResponse lookupUUID(MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); if (log.isDebugEnabled()) { @@ -101,6 +175,43 @@ public BaseQueryResponse lookupUUID(MultiValueMap parameters, Pro } } + /** + * Creates a content lookup query using the query logic associated with the given uuid type and parameters, and returns the first page of results. + *

+ * Lookup queries will start running immediately.
+ * Auditing is performed before the query is started.
+ * After the first page is returned, the query will be closed. + * + * @param uuidType + * the uuid type, not null + * @param uuid + * the uuid, not null + * @param parameters + * the query parameters, not null + * @param currentUser + * the user who called this method, not null + * @return a base query response containing the first page of results + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws TimeoutQueryException + * if the next call times out + * @throws NoResultsQueryException + * if no query results are found + * @throws QueryException + * if this next task is rejected by the executor + * @throws QueryException + * if there is an unknown error + */ public BaseQueryResponse lookupContentUUID(String uuidType, String uuid, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); @@ -123,6 +234,40 @@ public BaseQueryResponse lookupContentUUID(String uuidType, String uuid, MultiVa } } + /** + * Creates a batch content lookup query using the query logic associated with the given uuid type(s) and parameters, and returns the first page of results. + *

+ * Lookup queries will start running immediately.
+ * Auditing is performed before the query is started.
+ * Each of the uuid pairs must map to the same query logic.
+ * After the first page is returned, the query will be closed. + * + * @param parameters + * the query parameters, not null + * @param currentUser + * the user who called this method, not null + * @return a base query response containing the first page of results + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws TimeoutQueryException + * if the next call times out + * @throws NoResultsQueryException + * if no query results are found + * @throws QueryException + * if this next task is rejected by the executor + * @throws QueryException + * if there is an unknown error + */ public BaseQueryResponse lookupContentUUID(MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); if (log.isDebugEnabled()) { @@ -187,7 +332,7 @@ private BaseQueryResponse lookupEvents(MultiValueMap lookupTermMa parameters.put(QUERY_STRING, Collections.singletonList(lookupQueryLogic.createQueryFromLookupTerms(lookupTermMap))); // update the parameters for query - updateParametersForEventQuery(parameters, currentUser); + setEventQueryParameters(parameters, currentUser); // run the query BaseQueryResponse nextResponse = queryManagementService.createAndNext(parameters.getFirst(QUERY_LOGIC_NAME), parameters, currentUser); @@ -208,7 +353,7 @@ protected LookupQueryLogic validateLookupTerms(List lookupUUIDPairs, String queryLogicName = null; // make sure there aren't too many terms to lookup - if (uuidProperties.getBatchLookupLimit() > 0 && lookupUUIDPairs.size() <= uuidProperties.getBatchLookupLimit()) { + if (lookupProperties.getBatchLookupLimit() > 0 && lookupUUIDPairs.size() <= lookupProperties.getBatchLookupLimit()) { // validate each of the uuid pairs for (String uuidPair : lookupUUIDPairs) { @@ -223,37 +368,50 @@ protected LookupQueryLogic validateLookupTerms(List lookupUUIDPairs, if (!field.isEmpty() && !value.isEmpty()) { // is this a supported uuid type/field? - UUIDType uuidType = uuidProperties.getTypes().get(field.toUpperCase()); + UUIDType uuidType = lookupProperties.getTypes().get(field.toUpperCase()); if (uuidType != null) { if (queryLogicName == null) { queryLogicName = uuidType.getQueryLogic(); } // if we are mixing and matching query logics else if (!queryLogicName.equals(uuidType.getQueryLogic())) { - // TODO: This is a deal breaker + String message = "Multiple UUID types '" + queryLogicName + "' and '" + uuidType.getQueryLogic() + + "' not supported within the same lookup request"; + log.error(message); + throw new BadRequestQueryException(new IllegalArgumentException(message), HttpStatus.SC_BAD_REQUEST + "-1"); } } // if uuid type is null else { - // TODO: This is a deal breaker + String message = "Invalid type '" + field.toUpperCase() + "' for UUID " + value + + " not supported with the LuceneToJexlUUIDQueryParser"; + log.error(message); + throw new BadRequestQueryException(new IllegalArgumentException(message), HttpStatus.SC_BAD_REQUEST + "-1"); } lookupUUIDMap.add(field, value); } // if the field or value is empty else { - // TODO: This is a deal breaker + String message = "Empty UUID type or value extracted from uuidPair " + uuidPair; + log.error(message); + throw new BadRequestQueryException(new IllegalArgumentException(message), HttpStatus.SC_BAD_REQUEST + "-1"); } } // if there isn't a field AND a value else { - // TODO: This is a deal breaker + String message = "Unable to determine UUID type and value from uuidPair " + uuidPair; + log.error(message); + throw new BadRequestQueryException(new IllegalArgumentException(message), HttpStatus.SC_BAD_REQUEST + "-1"); } } } // too many terms to lookup else { - // TODO: This is a deal breaker + String message = "The " + lookupUUIDPairs.size() + " specified UUIDs exceed the maximum number of " + lookupProperties.getBatchLookupLimit() + + " allowed for a given lookup request"; + log.error(message); + throw new BadRequestQueryException(new IllegalArgumentException(message), HttpStatus.SC_BAD_REQUEST + "-1"); } try { @@ -270,12 +428,11 @@ else if (!queryLogicName.equals(uuidType.getQueryLogic())) { } } - protected void updateParametersForEventQuery(MultiValueMap parameters, ProxiedUserDetails currentUser) { + @SuppressWarnings("ConstantConditions") + protected void setEventQueryParameters(MultiValueMap parameters, ProxiedUserDetails currentUser) { String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); - if (uuidProperties.getColumnVisibility() != null) { - parameters.set(QueryParameters.QUERY_VISIBILITY, uuidProperties.getColumnVisibility()); - } + setOptionalQueryParameters(parameters); parameters.set(QUERY_SYNTAX, LUCENE_UUID_SYNTAX); @@ -291,7 +448,7 @@ protected void updateParametersForEventQuery(MultiValueMap parame final String queryName = user + "-" + UUID.randomUUID(); parameters.set(QueryParameters.QUERY_NAME, queryName); - parameters.set(QueryParameters.QUERY_BEGIN, uuidProperties.getBeginDate()); + parameters.set(QueryParameters.QUERY_BEGIN, lookupProperties.getBeginDate()); final Date endDate = DateUtils.addDays(new Date(), 2); try { @@ -299,13 +456,11 @@ protected void updateParametersForEventQuery(MultiValueMap parame } catch (ParseException e) { throw new RuntimeException("Unable to format new query end date: " + endDate); } - - // TODO: This may not be needed? - final Date expireDate = new Date(endDate.getTime() + 1000 * 60 * 60); - try { - parameters.set(QueryParameters.QUERY_EXPIRATION, DefaultQueryParameters.formatDate(expireDate)); - } catch (ParseException e) { - throw new RuntimeException("Unable to format new query expr date: " + expireDate); + } + + protected void setOptionalQueryParameters(MultiValueMap parameters) { + if (lookupProperties.getColumnVisibility() != null) { + parameters.set(QueryParameters.QUERY_VISIBILITY, lookupProperties.getColumnVisibility()); } } @@ -324,8 +479,7 @@ private String createContentLookupTerm(Metadata eventMetadata) { + String.join(CONTENT_QUERY_VALUE_DELIMITER, eventMetadata.getRow(), eventMetadata.getDataType(), eventMetadata.getInternalId()); } - private BaseQueryResponse lookupContent(Set contentLookupTerms, MultiValueMap parameters, ProxiedUserDetails currentUser) - throws QueryException { + private BaseQueryResponse lookupContent(Set contentLookupTerms, MultiValueMap parameters, ProxiedUserDetails currentUser) { // create queries from the content lookup terms List contentQueries = createContentQueries(contentLookupTerms); @@ -335,7 +489,7 @@ private BaseQueryResponse lookupContent(Set contentLookupTerms, MultiVal parameters.put(QUERY_STRING, Collections.singletonList(contentQuery)); // update parameters for the query - updateParametersForContentQuery(parameters, currentUser); + setContentQueryParameters(parameters, currentUser); // run the query EventQueryResponseBase contentQueryResponse = runContentQuery(parameters, currentUser); @@ -355,21 +509,23 @@ private BaseQueryResponse lookupContent(Set contentLookupTerms, MultiVal private List createContentQueries(Set contentLookupTerms) { List contentQueries = new ArrayList<>(); - Iterables.partition(contentLookupTerms, uuidProperties.getBatchLookupLimit()) + Iterables.partition(contentLookupTerms, lookupProperties.getBatchLookupLimit()) .forEach(termBatch -> contentQueries.add(String.join(CONTENT_QUERY_TERM_SEPARATOR, termBatch))); return contentQueries; } - protected void updateParametersForContentQuery(MultiValueMap parameters, ProxiedUserDetails currentUser) { + protected void setContentQueryParameters(MultiValueMap parameters, ProxiedUserDetails currentUser) { String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + setOptionalQueryParameters(parameters); + // all content queries use the same query logic - parameters.put(QUERY_LOGIC_NAME, Collections.singletonList(uuidProperties.getContentQueryLogicName())); + parameters.put(QUERY_LOGIC_NAME, Collections.singletonList(lookupProperties.getContentQueryLogicName())); parameters.set(QueryParameters.QUERY_NAME, user + '-' + UUID.randomUUID()); - parameters.set(QueryParameters.QUERY_BEGIN, uuidProperties.getBeginDate()); + parameters.set(QueryParameters.QUERY_BEGIN, lookupProperties.getBeginDate()); final Date endDate = new Date(); try { @@ -380,16 +536,9 @@ protected void updateParametersForContentQuery(MultiValueMap para final String userAuths = AuthorizationsUtil.buildUserAuthorizationString(currentUser); parameters.set(QueryParameters.QUERY_AUTHORIZATIONS, userAuths); - - final Date expireDate = new Date(endDate.getTime() + 1000 * 60 * 60); - try { - parameters.set(QueryParameters.QUERY_EXPIRATION, DefaultQueryParameters.formatDate(expireDate)); - } catch (ParseException e1) { - throw new RuntimeException("Error formatting expr date: " + expireDate); - } } - protected EventQueryResponseBase runContentQuery(MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + protected EventQueryResponseBase runContentQuery(MultiValueMap parameters, ProxiedUserDetails currentUser) { EventQueryResponseBase mergedResponse = null; String queryId = null; boolean isQueryFinished = false; @@ -404,7 +553,7 @@ protected EventQueryResponseBase runContentQuery(MultiValueMap pa nextResponse = queryManagementService.next(queryId, currentUser); } } catch (NoResultsQueryException e) { - log.info("No results found for content query '{}'", parameters.getFirst(QUERY_STRING)); + log.debug("No results found for content query '{}'", parameters.getFirst(QUERY_STRING)); } catch (QueryException e) { log.info("Encountered error while getting results for content query '{}'", parameters.getFirst(QUERY_STRING)); } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java new file mode 100644 index 00000000..3aa72408 --- /dev/null +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java @@ -0,0 +1,804 @@ +package datawave.microservice.query.lookup; + +import com.google.common.collect.Iterables; +import datawave.marking.ColumnVisibilitySecurityMarking; +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.query.AbstractQueryServiceTest; +import datawave.microservice.query.DefaultQueryParameters; +import datawave.microservice.query.messaging.Result; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.services.query.logic.QueryKey; +import datawave.webservice.query.result.event.DefaultEvent; +import datawave.webservice.query.result.event.DefaultField; +import datawave.webservice.query.result.event.Metadata; +import datawave.webservice.result.BaseQueryResponse; +import datawave.webservice.result.DefaultEventQueryResponse; +import datawave.webservice.result.VoidResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpMethod; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.stream.Collectors; + +import static datawave.microservice.query.QueryParameters.QUERY_MAX_CONCURRENT_TASKS; +import static datawave.microservice.query.lookup.LookupService.LOOKUP_UUID_PAIRS; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +public class LookupServiceTest extends AbstractQueryServiceTest { + + @Autowired + public LookupProperties lookupProperties; + + @Before + public void setup() { + super.setup(); + } + + @After + public void teardown() throws Exception { + super.teardown(); + } + + @Test + public void testLookupUUIDSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + MultiValueMap uuidParams = createUUIDParams(); + + String uuidType = "PAGE_TITLE"; + String uuid = "anarchy"; + + Future> future = lookupUUID(authUser, uuidParams, uuidType, uuid); + + String queryId = null; + + // get the lookup query id + long startTime = System.currentTimeMillis(); + while ((System.currentTimeMillis() - startTime) < 5000 && queryId == null) { + List queryStatuses = queryStorageCache.getQueryStatus(); + if (queryStatuses.size() > 0) { + queryId = queryStatuses.get(0).getQueryKey().getQueryId(); + } + } + + // pump enough results into the queue to trigger a complete page + int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add(uuidType, uuid); + + // @formatter:off + publishEventsToQueue( + queryId, + pageSize, + fieldValues, + "ALL"); + // @formatter:on + + ResponseEntity response = future.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // verify some headers + Assert.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + + DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); + + // verify the query response + // @formatter:off + assertQueryResponse( + queryId, + "LuceneUUIDEventQuery", + 1, + false, + Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-OperationTimeInMS")))), + 1, + Collections.singletonList(uuidType), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Collections.singletonList(uuidType), + Collections.singletonList(uuid), + event); + // @formatter:on + + // verify that the correct events were published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testBatchLookupUUIDSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + MultiValueMap uuidParams = createUUIDParams(); + uuidParams.add(LOOKUP_UUID_PAIRS, "PAGE_TITLE:anarchy"); + uuidParams.add(LOOKUP_UUID_PAIRS, "PAGE_TITLE:accessiblecomputing"); + + Future> future = batchLookupUUID(authUser, uuidParams); + + String queryId = null; + + // get the lookup query id + long startTime = System.currentTimeMillis(); + while ((System.currentTimeMillis() - startTime) < 5000 && queryId == null) { + List queryStatuses = queryStorageCache.getQueryStatus(); + if (queryStatuses.size() > 0) { + queryId = queryStatuses.get(0).getQueryKey().getQueryId(); + } + } + + // pump enough results into the queue to trigger a complete page + int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("PAGE_TITLE", "anarchy"); + fieldValues.add("PAGE_TITLE", "accessiblecomputing"); + + // @formatter:off + publishEventsToQueue( + queryId, + pageSize, + fieldValues, + "ALL"); + // @formatter:on + + ResponseEntity response = future.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // verify some headers + Assert.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + + DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); + + // verify the query response + // @formatter:off + assertQueryResponse( + queryId, + "LuceneUUIDEventQuery", + 1, + false, + Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-OperationTimeInMS")))), + 1, + Collections.singletonList("PAGE_TITLE"), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Arrays.asList("PAGE_TITLE", "PAGE_TITLE"), + Arrays.asList("anarchy", "accessiblecomputing"), + event); + // @formatter:on + + // verify that the correct events were published + Assert.assertEquals(3, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testLookupContentUUIDSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + MultiValueMap uuidParams = createUUIDParams(); + + String uuidType = "PAGE_TITLE"; + String uuid = "anarchy"; + + Future> future = lookupContentUUID(authUser, uuidParams, uuidType, uuid); + + String queryId = null; + + // get the lookup query id + long startTime = System.currentTimeMillis(); + while ((System.currentTimeMillis() - startTime) < 5000 && queryId == null) { + List queryStatuses = queryStorageCache.getQueryStatus(); + if (queryStatuses.size() > 0) { + queryId = queryStatuses.get(0).getQueryKey().getQueryId(); + } + } + + // pump enough results into the queue to trigger a complete page + int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add(uuidType, uuid); + + // @formatter:off + publishEventsToQueue( + queryId, + pageSize, + fieldValues, + "ALL"); + // @formatter:on + + Set contentQueryIds = null; + // wait for the initial event query to be closed + startTime = System.currentTimeMillis(); + while ((System.currentTimeMillis() - startTime) < 5000 && contentQueryIds == null) { + final String eventQueryId = queryId; + List queryStatuses = queryStorageCache.getQueryStatus(); + if (queryStatuses.size() == 1 + Math.ceil((double) pageSize / lookupProperties.getBatchLookupLimit())) { + contentQueryIds = queryStatuses.stream().map(QueryStatus::getQueryKey).map(QueryKey::getQueryId) + .filter(contentQueryId -> !contentQueryId.equals(eventQueryId)).collect(Collectors.toSet()); + } + } + + Assert.assertNotNull(contentQueryIds); + for (String contentQueryId : contentQueryIds) { + MultiValueMap contentFieldValues = new LinkedMultiValueMap<>(); + contentFieldValues.add("CONTENT", "look I made you some content!"); + + // @formatter:off + publishEventsToQueue( + contentQueryId, + pageSize, + contentFieldValues, + "ALL"); + // @formatter:on + } + + ResponseEntity response = future.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // verify some headers + Assert.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + + DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); + + String responseQueryId = queryResponse.getQueryId(); + + Assert.assertTrue(contentQueryIds.contains(responseQueryId)); + + // verify the query response + // @formatter:off + assertContentQueryResponse( + responseQueryId, + "ContentQuery", + 1, + false, + Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-OperationTimeInMS")))), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Collections.singletonList("CONTENT"), + Collections.singletonList("look I made you some content!"), + event); + // @formatter:on + + // verify that the correct events were published + Assert.assertEquals(7, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + responseQueryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + responseQueryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + responseQueryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + responseQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testBatchLookupContentUUIDSuccess() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + MultiValueMap uuidParams = createUUIDParams(); + uuidParams.add(LOOKUP_UUID_PAIRS, "PAGE_TITLE:anarchy"); + uuidParams.add(LOOKUP_UUID_PAIRS, "PAGE_TITLE:accessiblecomputing"); + + Future> future = batchLookupContentUUID(authUser, uuidParams); + + String queryId = null; + + // get the lookup query id + long startTime = System.currentTimeMillis(); + while ((System.currentTimeMillis() - startTime) < 5000 && queryId == null) { + List queryStatuses = queryStorageCache.getQueryStatus(); + if (queryStatuses.size() > 0) { + queryId = queryStatuses.get(0).getQueryKey().getQueryId(); + } + } + + // pump enough results into the queue to trigger a complete page + int pageSize = queryStorageCache.getQueryStatus(queryId).getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("PAGE_TITLE", "anarchy"); + fieldValues.add("PAGE_TITLE", "accessiblecomputing"); + + // @formatter:off + publishEventsToQueue( + queryId, + pageSize, + fieldValues, + "ALL"); + // @formatter:on + + Set contentQueryIds = null; + // wait for the initial event query to be closed + startTime = System.currentTimeMillis(); + while ((System.currentTimeMillis() - startTime) < 5000 && contentQueryIds == null) { + final String eventQueryId = queryId; + List queryStatuses = queryStorageCache.getQueryStatus(); + if (queryStatuses.size() == 1 + Math.ceil((double) pageSize / lookupProperties.getBatchLookupLimit())) { + contentQueryIds = queryStatuses.stream().map(QueryStatus::getQueryKey).map(QueryKey::getQueryId) + .filter(contentQueryId -> !contentQueryId.equals(eventQueryId)).collect(Collectors.toSet()); + } + } + + Assert.assertNotNull(contentQueryIds); + for (String contentQueryId : contentQueryIds) { + MultiValueMap contentFieldValues = new LinkedMultiValueMap<>(); + contentFieldValues.add("CONTENT", "look I made you some content!"); + + // @formatter:off + publishEventsToQueue( + contentQueryId, + pageSize, + contentFieldValues, + "ALL"); + // @formatter:on + } + + ResponseEntity response = future.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // verify some headers + Assert.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + + DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); + + String responseQueryId = queryResponse.getQueryId(); + + Assert.assertTrue(contentQueryIds.contains(responseQueryId)); + + // verify the query response + // @formatter:off + assertContentQueryResponse( + responseQueryId, + "ContentQuery", + 1, + false, + Long.parseLong(Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-OperationTimeInMS")))), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Collections.singletonList("CONTENT"), + Collections.singletonList("look I made you some content!"), + event); + // @formatter:on + + // verify that the correct events were published + Assert.assertEquals(7, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + responseQueryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + responseQueryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + responseQueryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + responseQueryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testBatchLookupUUIDFailure_noLookupUUIDPairs() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + MultiValueMap uuidParams = createUUIDParams(); + + UriComponents uri = createUri("lookupUUID"); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, uuidParams, null, HttpMethod.POST, uri); + + ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Missing required parameter.", + "Exception with no cause caught", + "400-40", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + } + + @Test + public void testBatchLookupUUIDFailure_mixedQueryLogics() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + MultiValueMap uuidParams = createUUIDParams(); + uuidParams.add(LOOKUP_UUID_PAIRS, "PAGE_TITLE:anarchy"); + uuidParams.add(LOOKUP_UUID_PAIRS, "PAGE_NUMBER:accessiblecomputing"); + + UriComponents uri = createUri("lookupUUID"); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, uuidParams, null, HttpMethod.POST, uri); + + ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Multiple UUID types 'LuceneUUIDEventQuery' and 'EventQuery' not supported within the same lookup request", + "java.lang.IllegalArgumentException: Multiple UUID types 'LuceneUUIDEventQuery' and 'EventQuery' not supported within the same lookup request", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + } + + @Test + public void testBatchLookupUUIDFailure_nullUUIDType() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + MultiValueMap uuidParams = createUUIDParams(); + uuidParams.add(LOOKUP_UUID_PAIRS, "PAGE:anarchy"); + + UriComponents uri = createUri("lookupUUID"); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, uuidParams, null, HttpMethod.POST, uri); + + ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Invalid type 'PAGE' for UUID anarchy not supported with the LuceneToJexlUUIDQueryParser", + "java.lang.IllegalArgumentException: Invalid type 'PAGE' for UUID anarchy not supported with the LuceneToJexlUUIDQueryParser", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + } + + @Test + public void testBatchLookupUUIDFailure_emptyUUIDFieldValue() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + MultiValueMap uuidParams = createUUIDParams(); + uuidParams.add(LOOKUP_UUID_PAIRS, ":anarchy"); + + UriComponents uri = createUri("lookupUUID"); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, uuidParams, null, HttpMethod.POST, uri); + + ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Empty UUID type or value extracted from uuidPair :anarchy", + "java.lang.IllegalArgumentException: Empty UUID type or value extracted from uuidPair :anarchy", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + } + + @Test + public void testBatchLookupUUIDFailure_invalidUUIDPair() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + MultiValueMap uuidParams = createUUIDParams(); + uuidParams.add(LOOKUP_UUID_PAIRS, ":"); + + UriComponents uri = createUri("lookupUUID"); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, uuidParams, null, HttpMethod.POST, uri); + + ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Unable to determine UUID type and value from uuidPair :", + "java.lang.IllegalArgumentException: Unable to determine UUID type and value from uuidPair :", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + } + + @Test + public void testBatchLookupUUIDFailure_tooManyTerms() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + MultiValueMap uuidParams = createUUIDParams(); + + for (int i = 0; i < lookupProperties.getBatchLookupLimit() + 1; i++) { + uuidParams.add(LOOKUP_UUID_PAIRS, "PAGE_TITLE:anarchy-" + i); + } + + UriComponents uri = createUri("lookupUUID"); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, uuidParams, null, HttpMethod.POST, uri); + + ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); + + Assert.assertEquals(400, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "The " + (lookupProperties.getBatchLookupLimit() + 1) + " specified UUIDs exceed the maximum number of " + lookupProperties.getBatchLookupLimit() + " allowed for a given lookup request", + "java.lang.IllegalArgumentException: The " + (lookupProperties.getBatchLookupLimit() + 1) + " specified UUIDs exceed the maximum number of " + lookupProperties.getBatchLookupLimit() + " allowed for a given lookup request", + "400-1", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + } + + @Test + public void testBatchLookupUUIDFailure_nonLookupQueryLogic() throws Exception { + ProxiedUserDetails authUser = createUserDetails(); + + MultiValueMap uuidParams = createUUIDParams(); + uuidParams.add(LOOKUP_UUID_PAIRS, "PAGE_NUMBER:accessiblecomputing"); + + UriComponents uri = createUri("lookupUUID"); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, uuidParams, null, HttpMethod.POST, uri); + + ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); + + Assert.assertEquals(500, response.getStatusCodeValue()); + + // @formatter:off + assertQueryException( + "Error setting up query. Lookup UUID can only be run with a LookupQueryLogic", + "Exception with no cause caught", + "500-66", + Iterables.getOnlyElement(response.getBody().getExceptions())); + // @formatter:on + } + + protected MultiValueMap createUUIDParams() { + MultiValueMap map = new LinkedMultiValueMap<>(); + map.set(DefaultQueryParameters.QUERY_NAME, TEST_QUERY_NAME); + map.set(DefaultQueryParameters.QUERY_AUTHORIZATIONS, TEST_QUERY_AUTHORIZATIONS); + map.set(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, TEST_VISIBILITY_MARKING); + map.set(QUERY_MAX_CONCURRENT_TASKS, Integer.toString(1)); + return map; + } + + protected Future> batchLookupUUID(ProxiedUserDetails authUser, MultiValueMap map) { + UriComponents uri = createUri("lookupUUID"); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, BaseQueryResponse.class)); + } + + protected Future> lookupUUID(ProxiedUserDetails authUser, MultiValueMap map, String uuidType, + String uuid) { + UriComponents uri = createUri("lookupUUID/" + uuidType + "/" + uuid); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.GET, uri); + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, BaseQueryResponse.class)); + } + + protected Future> batchLookupContentUUID(ProxiedUserDetails authUser, MultiValueMap map) { + UriComponents uri = createUri("lookupContentUUID"); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, BaseQueryResponse.class)); + } + + protected Future> lookupContentUUID(ProxiedUserDetails authUser, MultiValueMap map, String uuidType, + String uuid) { + UriComponents uri = createUri("lookupContentUUID/" + uuidType + "/" + uuid); + + // not testing audit with this method + auditIgnoreSetup(); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.GET, uri); + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, BaseQueryResponse.class)); + } + + protected void publishEventsToQueue(String queryId, int numEvents, MultiValueMap fieldValues, String visibility) throws Exception { + for (int resultId = 0; resultId < numEvents; resultId++) { + DefaultEvent event = new DefaultEvent(); + long currentTime = System.currentTimeMillis(); + List fields = new ArrayList<>(); + for (Map.Entry> entry : fieldValues.entrySet()) { + for (String value : entry.getValue()) { + fields.add(new DefaultField(entry.getKey(), visibility, currentTime, value)); + } + } + event.setFields(fields); + + Metadata metadata = new Metadata(); + // tonight i'm gonna party like it's + metadata.setRow("19991231_0"); + metadata.setDataType("prince"); + metadata.setInternalId(UUID.randomUUID().toString()); + event.setMetadata(metadata); + queryQueueManager.createPublisher(queryId).publish(new Result(Integer.toString(resultId), event)); + } + } + + protected void assertContentQueryResponse(String queryId, String logicName, long pageNumber, boolean partialResults, long operationTimeInMS, int numEvents, + DefaultEventQueryResponse queryResponse) { + Assert.assertEquals(queryId, queryResponse.getQueryId()); + Assert.assertEquals(logicName, queryResponse.getLogicName()); + Assert.assertEquals(pageNumber, queryResponse.getPageNumber()); + Assert.assertEquals(partialResults, queryResponse.isPartialResults()); + Assert.assertEquals(operationTimeInMS, queryResponse.getOperationTimeMS()); + Assert.assertEquals(numEvents, queryResponse.getEvents().size()); + } +} diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/uuid/QueryServiceLookupUUIDTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/uuid/QueryServiceLookupUUIDTest.java deleted file mode 100644 index 4f2695f3..00000000 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/uuid/QueryServiceLookupUUIDTest.java +++ /dev/null @@ -1,99 +0,0 @@ -package datawave.microservice.query.uuid; - -import datawave.marking.ColumnVisibilitySecurityMarking; -import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; -import datawave.microservice.authorization.user.ProxiedUserDetails; -import datawave.microservice.query.AbstractQueryServiceTest; -import datawave.microservice.query.DefaultQueryParameters; -import datawave.webservice.result.BaseQueryResponse; -import org.junit.After; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.http.HttpMethod; -import org.springframework.http.RequestEntity; -import org.springframework.http.ResponseEntity; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; -import org.springframework.web.util.UriComponents; - -import static datawave.microservice.query.QueryParameters.QUERY_MAX_CONCURRENT_TASKS; -import static datawave.microservice.query.uuid.LookupService.LOOKUP_UUID_PAIRS; - -@RunWith(SpringRunner.class) -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) -public class QueryServiceLookupUUIDTest extends AbstractQueryServiceTest { - @Before - public void setup() { - super.setup(); - } - - @After - public void teardown() throws Exception { - super.teardown(); - } - - @Ignore - @Test - public void testLookupUUIDSuccess() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - // create a valid query - long currentTimeMillis = System.currentTimeMillis(); - BaseQueryResponse response = lookupUUID(authUser, createUUIDParams(), "PAGE_TITLE", "anarchy"); - - } - - @Ignore - @Test - public void testBatchLookupUUIDSuccess() throws Exception { - ProxiedUserDetails authUser = createUserDetails(); - - MultiValueMap params = createUUIDParams(); - params.add(LOOKUP_UUID_PAIRS, "PAGE_TITLE:anarchy"); - params.add(LOOKUP_UUID_PAIRS, "PAGE_TITLE:accessiblecomputing"); - - // create a valid query - long currentTimeMillis = System.currentTimeMillis(); - BaseQueryResponse response = batchLookupUUID(authUser, params); - - } - - protected MultiValueMap createUUIDParams() { - MultiValueMap map = new LinkedMultiValueMap<>(); - map.set(DefaultQueryParameters.QUERY_NAME, TEST_QUERY_NAME); - map.set(DefaultQueryParameters.QUERY_AUTHORIZATIONS, TEST_QUERY_AUTHORIZATIONS); - map.set(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, TEST_VISIBILITY_MARKING); - map.set(QUERY_MAX_CONCURRENT_TASKS, Integer.toString(1)); - return map; - } - - protected BaseQueryResponse batchLookupUUID(ProxiedUserDetails authUser, MultiValueMap map) { - UriComponents uri = createUri("lookupUUID"); - - // not testing audit with this method - auditIgnoreSetup(); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.POST, uri); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseQueryResponse.class); - - return resp.getBody(); - } - - protected BaseQueryResponse lookupUUID(ProxiedUserDetails authUser, MultiValueMap map, String uuidType, String uuid) { - UriComponents uri = createUri("lookupUUID/" + uuidType + "/" + uuid); - - // not testing audit with this method - auditIgnoreSetup(); - - RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, null, HttpMethod.GET, uri); - ResponseEntity resp = jwtRestTemplate.exchange(requestEntity, BaseQueryResponse.class); - - return resp.getBody(); - } -} diff --git a/query-microservices/query-service/service/src/test/resources/config/application-QueryStarterOverrides.yml b/query-microservices/query-service/service/src/test/resources/config/application-QueryStarterOverrides.yml index 28f61b5c..c4d7e5e9 100644 --- a/query-microservices/query-service/service/src/test/resources/config/application-QueryStarterOverrides.yml +++ b/query-microservices/query-service/service/src/test/resources/config/application-QueryStarterOverrides.yml @@ -18,3 +18,24 @@ datawave: BaseEventQuery: maxResults: 369 auditType: "ACTIVE" + lookup: + types: + 'EVENT_ID': + fieldName: 'EVENT_ID' + queryLogic: 'LuceneUUIDEventQuery' + allowedWildcardAfter: 28 + 'UUID': + fieldName: 'UUID' + queryLogic: 'LuceneUUIDEventQuery' + 'PARENT_UUID': + fieldName: 'PARENT_UUID' + queryLogic: 'LuceneUUIDEventQuery' + 'PAGE_ID': + fieldName: 'PAGE_ID' + queryLogic: 'LuceneUUIDEventQuery' + 'PAGE_TITLE': + fieldName: 'PAGE_TITLE' + queryLogic: 'LuceneUUIDEventQuery' + 'PAGE_NUMBER': + fieldName: 'PAGE_NUMBER' + queryLogic: 'EventQuery' \ No newline at end of file From e1647007f0ff347ed54fac11263cffb4bf9a3eed Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 29 Dec 2021 13:38:20 -0500 Subject: [PATCH 116/218] Updated the edge xmlBeansPath to pull from a unique property instead of using the one for the query logic factory. --- .../java/datawave/microservice/query/QueryServiceListTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java index 227a1297..f517d063 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java @@ -419,7 +419,7 @@ public void testListQueryLogicSuccess() throws Exception { QueryLogicResponse qlResponse = response.getBody(); - String[] expectedQueryLogics = new String[] {"AltEventQuery", "EventQuery", "LuceneUUIDEventQuery", "DiscoveryQuery", "ContentQuery"}; + String[] expectedQueryLogics = new String[] {"AltEventQuery", "EventQuery", "LuceneUUIDEventQuery", "DiscoveryQuery", "ContentQuery", "EdgeQuery"}; Assert.assertEquals(expectedQueryLogics.length, qlResponse.getQueryLogicList().size()); From 43c3e97afe39b5d5831ff30467469bd496f5e0c7 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 7 Jan 2022 12:56:29 -0500 Subject: [PATCH 117/218] Added a streaming service which can be used to stream query results. --- .../microservice/query/QueryController.java | 42 +++- .../query/QueryManagementService.java | 68 ++++++- .../query/config/QueryServiceConfig.java | 14 ++ .../microservice/query/runner/NextCall.java | 4 + .../query/stream/StreamingService.java | 38 ++++ .../CountingResponseBodyEmitterListener.java | 41 ++++ .../listener/StreamingResponseListener.java | 14 ++ .../query/stream/runner/StreamingCall.java | 189 ++++++++++++++++++ .../web/filter/BaseMethodStatsFilter.java | 28 ++- .../filter/CountingResponseBodyEmitter.java | 15 ++ .../QueryMetricsEnrichmentFilterAdvice.java | 2 +- 11 files changed, 438 insertions(+), 17 deletions(-) create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/StreamingService.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 1235f2b6..77983fc6 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -3,21 +3,29 @@ import com.codahale.metrics.annotation.Timed; import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.query.lookup.LookupService; +import datawave.microservice.query.stream.StreamingService; +import datawave.microservice.query.stream.listener.CountingResponseBodyEmitterListener; import datawave.microservice.query.web.annotation.EnrichQueryMetrics; +import datawave.microservice.query.web.filter.BaseMethodStatsFilter; +import datawave.microservice.query.web.filter.CountingResponseBodyEmitter; +import datawave.microservice.query.web.filter.QueryMetricsEnrichmentFilterAdvice; import datawave.webservice.query.exception.QueryException; import datawave.webservice.result.BaseQueryResponse; import datawave.webservice.result.GenericResponse; import datawave.webservice.result.QueryImplListResponse; import datawave.webservice.result.QueryLogicResponse; import datawave.webservice.result.VoidResponse; +import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.util.MultiValueMap; import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyEmitter; import javax.annotation.security.RolesAllowed; @@ -26,10 +34,21 @@ public class QueryController { private final QueryManagementService queryManagementService; private final LookupService lookupService; + private final StreamingService streamingService; - public QueryController(QueryManagementService queryManagementService, LookupService lookupService) { + // Note: baseMethodStatsContext needs to be request scoped + private final BaseMethodStatsFilter.BaseMethodStatsContext baseMethodStatsContext; + // Note: queryMetricsEnrichmentContest needs to be request scoped + private final QueryMetricsEnrichmentFilterAdvice.QueryMetricsEnrichmentContext queryMetricsEnrichmentContext; + + public QueryController(QueryManagementService queryManagementService, LookupService lookupService, StreamingService streamingService, + BaseMethodStatsFilter.BaseMethodStatsContext baseMethodStatsContext, + QueryMetricsEnrichmentFilterAdvice.QueryMetricsEnrichmentContext queryMetricsEnrichmentContext) { this.queryManagementService = queryManagementService; this.lookupService = lookupService; + this.streamingService = streamingService; + this.baseMethodStatsContext = baseMethodStatsContext; + this.queryMetricsEnrichmentContext = queryMetricsEnrichmentContext; } @Timed(name = "dw.query.listQueryLogic", absolute = true) @@ -109,7 +128,7 @@ public BaseQueryResponse lookupContentUUIDBatch(@PathVariable(required = false) @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE_AND_NEXT) @RequestMapping(path = "{queryLogic}/createAndNext", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public BaseQueryResponse createQueryAndNext(@PathVariable String queryLogic, @RequestParam MultiValueMap parameters, + public BaseQueryResponse createAndNext(@PathVariable String queryLogic, @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.createAndNext(queryLogic, parameters, currentUser); } @@ -251,4 +270,23 @@ public VoidResponse adminCloseAll(@AuthenticationPrincipal ProxiedUserDetails cu public VoidResponse adminRemoveAll(@AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.adminRemoveAll(currentUser); } + + @Timed(name = "dw.query.executeQuery", absolute = true) + @RequestMapping(path = "{queryLogic}/execute", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public ResponseBodyEmitter execute(@PathVariable String queryLogic, @RequestParam MultiValueMap parameters, + @AuthenticationPrincipal ProxiedUserDetails currentUser, @RequestHeader HttpHeaders headers) throws QueryException { + String queryId = queryManagementService.create(queryLogic, parameters, currentUser).getResult(); + + // unfortunately this needs to be set manually. ResponseBodyAdvice does not run for streaming endpoints + queryMetricsEnrichmentContext.setMethodType(EnrichQueryMetrics.MethodType.CREATE); + queryMetricsEnrichmentContext.setQueryId(queryId); + + CountingResponseBodyEmitter emitter = baseMethodStatsContext.getCountingResponseBodyEmitter(); + + // bytesWritten is updated as streaming data is written to the response output stream + streamingService.execute(queryId, parameters, currentUser, new CountingResponseBodyEmitterListener(emitter, headers.getAccept())); + + return emitter; + } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 8fd5b3a9..7e1a5828 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -117,6 +117,13 @@ public class QueryManagementService implements QueryRequestHandler { private final String selfDestination; + // Note: for requests which don't originate with a rest call, provide ThreadLocal queryParameters + private final ThreadLocal queryParametersOverride; + // Note: for requests which don't originate with a rest call, provide ThreadLocal securityMarkings + private final ThreadLocal securityMarkingOverride; + // Note: for requests which don't originate with a rest call, provide ThreadLocal baseQueryMetric + private final ThreadLocal baseQueryMetricOverride; + private final Map queryLatchMap = new ConcurrentHashMap<>(); public QueryManagementService(QueryProperties queryProperties, ApplicationEventPublisher eventPublisher, BusProperties busProperties, @@ -138,6 +145,9 @@ public QueryManagementService(QueryProperties queryProperties, ApplicationEventP this.nextCallExecutor = nextCallExecutor; this.queryStatusUpdateUtil = new QueryStatusUpdateUtil(this.queryProperties, this.queryStorageCache); this.selfDestination = getSelfDestination(); + this.queryParametersOverride = new ThreadLocal<>(); + this.securityMarkingOverride = new ThreadLocal<>(); + this.baseQueryMetricOverride = new ThreadLocal<>(); } /** @@ -501,6 +511,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p // TODO: Downgrade the auths before or after auditing??? // downgrade the auths + QueryParameters queryParameters = getQueryParameters(); Set downgradedAuthorizations; try { downgradedAuthorizations = AuthorizationsUtil.getDowngradedAuthorizations(queryParameters.getAuths(), currentUser); @@ -560,6 +571,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p } // update the query metric + BaseQueryMetric baseQueryMetric = getBaseQueryMetric(); if (queryType == DEFINED || queryType == CREATED) { baseQueryMetric.setQueryId(taskKey.getQueryId()); baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.DEFINED); @@ -668,7 +680,7 @@ public BaseQueryResponse createAndNext(String queryLogicName, MultiValueMap userRoles) thr QueryLogic queryLogic = queryLogicFactory.getQueryLogic(queryStatus.getQuery().getQueryLogicName(), userRoles); // update query metrics + BaseQueryMetric baseQueryMetric = getBaseQueryMetric(); baseQueryMetric.setQueryId(queryId); baseQueryMetric.setQueryLogic(queryLogicName); @@ -1057,6 +1070,7 @@ public void cancel(String queryId, boolean publishEvent) throws InterruptedExcep publishExecutorEvent(cancelRequest, queryStatus.getQueryKey().getQueryPool()); // update query metrics + BaseQueryMetric baseQueryMetric = getBaseQueryMetric(); baseQueryMetric.setQueryId(queryId); baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.CANCELLED); baseQueryMetric.setLastUpdated(new Date()); @@ -1255,6 +1269,7 @@ public void close(String queryId) throws InterruptedException, QueryException { publishExecutorEvent(QueryRequest.close(queryId), queryStatus.getQueryKey().getQueryPool()); // update query metrics + BaseQueryMetric baseQueryMetric = getBaseQueryMetric(); baseQueryMetric.setQueryId(queryId); baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.CLOSED); baseQueryMetric.setLastUpdated(new Date()); @@ -2040,7 +2055,7 @@ protected void audit(Query query, QueryLogic queryLogic, MultiValueMap queryLogic, MultiValueMap queryLogic) { // if there's an override, use it + QueryParameters queryParameters = getQueryParameters(); if (queryParameters.isMaxConcurrentTasksOverridden()) { return queryParameters.getMaxConcurrentTasks(); } @@ -2166,6 +2183,9 @@ else if (queryLogic.getMaxConcurrentTasks() > 0) { * @return an instantiated query object */ protected Query createQuery(String queryLogicName, MultiValueMap parameters, String userDn, List dnList, String queryId) { + QueryParameters queryParameters = getQueryParameters(); + SecurityMarking securityMarking = getSecurityMarking(); + Query q = responseObjectFactory.getQueryImpl(); q.initialize(userDn, dnList, queryLogicName, queryParameters, queryParameters.getUnknownParameters(parameters)); q.setColumnVisibility(securityMarking.toColumnVisibilityString()); @@ -2249,6 +2269,7 @@ protected void validateParameters(String queryLogicName, MultiValueMap queryLogic, MultiValueMap 0 && queryParameters.getPagesize() > queryLogic.getMaxPageSize()) { log.error("Invalid page size: {} vs {}", queryParameters.getPagesize(), queryLogic.getMaxPageSize()); throw new BadRequestQueryException(DatawaveErrorCode.PAGE_SIZE_TOO_LARGE, MessageFormat.format("Max = {0}.", queryLogic.getMaxPageSize())); @@ -2379,7 +2401,7 @@ protected void validateQueryLogic(QueryLogic queryLogic, MultiValueMap parameters) throws BadRequestQueryException { try { - securityMarking.validate(parameters); + getSecurityMarking().validate(parameters); } catch (IllegalArgumentException e) { log.error("Failed security markings validation", e); throw new BadRequestQueryException(DatawaveErrorCode.SECURITY_MARKING_CHECK_ERROR, e); @@ -2403,7 +2425,7 @@ protected void setInternalAuditParameters(String queryLogicName, String userDn, // These are parameters that aren't passed in by the user, but rather are computed from other sources. PrivateAuditConstants.stripPrivateParameters(parameters); parameters.add(PrivateAuditConstants.LOGIC_CLASS, queryLogicName); - parameters.set(PrivateAuditConstants.COLUMN_VISIBILITY, securityMarking.toColumnVisibilityString()); + parameters.set(PrivateAuditConstants.COLUMN_VISIBILITY, getSecurityMarking().toColumnVisibilityString()); parameters.add(PrivateAuditConstants.USER_DN, userDn); } @@ -2416,4 +2438,40 @@ private String writeValueAsString(Object object) { } return stringValue; } + + public QueryParameters getQueryParameters() { + if (queryParametersOverride.get() != null) { + return queryParametersOverride.get(); + } else { + return queryParameters; + } + } + + public SecurityMarking getSecurityMarking() { + if (securityMarkingOverride.get() != null) { + return securityMarkingOverride.get(); + } else { + return securityMarking; + } + } + + public BaseQueryMetric getBaseQueryMetric() { + if (baseQueryMetricOverride.get() != null) { + return baseQueryMetricOverride.get(); + } else { + return baseQueryMetric; + } + } + + public ThreadLocal getQueryParametersOverride() { + return queryParametersOverride; + } + + public ThreadLocal getSecurityMarkingOverride() { + return securityMarkingOverride; + } + + public ThreadLocal getBaseQueryMetricOverride() { + return baseQueryMetricOverride; + } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java index d2bc8b5d..3562bee2 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java @@ -4,6 +4,7 @@ import datawave.marking.SecurityMarking; import datawave.microservice.query.DefaultQueryParameters; import datawave.microservice.query.QueryParameters; +import datawave.microservice.query.stream.StreamingProperties; import datawave.microservice.querymetric.BaseQueryMetric; import datawave.microservice.querymetric.QueryMetricFactory; import datawave.microservice.querymetric.QueryMetricFactoryImpl; @@ -60,4 +61,17 @@ public ThreadPoolTaskExecutor nextCallExecutor(QueryProperties queryProperties) executor.initialize(); return executor; } + + @RefreshScope + @Bean + public ThreadPoolTaskExecutor streamingExecutor(StreamingProperties streamingProperties) { + ThreadPoolTaskExecutorProperties executorProperties = streamingProperties.getExecutor(); + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(executorProperties.getCorePoolSize()); + executor.setMaxPoolSize(executorProperties.getMaxPoolSize()); + executor.setQueueCapacity(executorProperties.getQueueCapacity()); + executor.setThreadNamePrefix(executorProperties.getThreadNamePrefix()); + executor.initialize(); + return executor; + } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 0b228a55..20eca070 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -396,6 +396,10 @@ public void setFuture(Future> future) { this.future = future; } + public BaseQueryMetric.Lifecycle getLifecycle() { + return lifecycle; + } + public static class Builder { private NextCallProperties nextCallProperties; private QueryExpirationProperties expirationProperties; diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/StreamingService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/StreamingService.java new file mode 100644 index 00000000..e50c4b66 --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/StreamingService.java @@ -0,0 +1,38 @@ +package datawave.microservice.query.stream; + +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.query.QueryManagementService; +import datawave.microservice.query.stream.listener.StreamingResponseListener; +import datawave.microservice.query.stream.runner.StreamingCall; +import datawave.microservice.querymetric.QueryMetricClient; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.stereotype.Service; +import org.springframework.util.MultiValueMap; + +@Service +public class StreamingService { + private final QueryManagementService queryManagementService; + private final QueryMetricClient queryMetricClient; + + private final ThreadPoolTaskExecutor streamingExecutor; + + public StreamingService(QueryManagementService queryManagementService, QueryMetricClient queryMetricClient, ThreadPoolTaskExecutor streamingExecutor) { + this.queryManagementService = queryManagementService; + this.queryMetricClient = queryMetricClient; + this.streamingExecutor = streamingExecutor; + } + + public void execute(String queryId, MultiValueMap parameters, ProxiedUserDetails currentUser, StreamingResponseListener listener) { + // @formatter:off + streamingExecutor.submit( + new StreamingCall.Builder() + .setQueryManagementService(queryManagementService) + .setQueryMetricClient(queryMetricClient) + .setParameters(parameters) + .setCurrentUser(currentUser) + .setQueryId(queryId) + .setListener(listener) + .build()); + // @formatter:on + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java new file mode 100644 index 00000000..d4826303 --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java @@ -0,0 +1,41 @@ +package datawave.microservice.query.stream.listener; + +import datawave.microservice.query.web.filter.CountingResponseBodyEmitter; +import datawave.webservice.result.BaseQueryResponse; +import org.springframework.http.MediaType; + +import java.io.IOException; +import java.util.List; + +public class CountingResponseBodyEmitterListener implements StreamingResponseListener { + private final CountingResponseBodyEmitter countingEmitter; + private final MediaType mediaType; + + public CountingResponseBodyEmitterListener(CountingResponseBodyEmitter countingEmitter, List mediaTypes) { + this.countingEmitter = countingEmitter; + this.mediaType = determineMediaType(mediaTypes); + } + + @Override + public void onResponse(BaseQueryResponse response) throws IOException { + countingEmitter.send(response, mediaType); + } + + @Override + public void cleanup() { + countingEmitter.complete(); + } + + public long getBytesWritten() { + return (countingEmitter != null) ? countingEmitter.getBytesWritten() : 0L; + } + + private MediaType determineMediaType(List acceptedMediaTypes) { + MediaType mediaType = null; + if (acceptedMediaTypes != null && !acceptedMediaTypes.isEmpty()) { + MediaType.sortBySpecificityAndQuality(acceptedMediaTypes); + mediaType = acceptedMediaTypes.get(0); + } + return mediaType; + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java new file mode 100644 index 00000000..f80b4d6e --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java @@ -0,0 +1,14 @@ +package datawave.microservice.query.stream.listener; + +import datawave.webservice.result.BaseQueryResponse; + +import java.io.IOException; + +public interface StreamingResponseListener { + + void onResponse(BaseQueryResponse response) throws IOException; + + default void cleanup() { + // do nothing + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java new file mode 100644 index 00000000..7f4c925b --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java @@ -0,0 +1,189 @@ +package datawave.microservice.query.stream.runner; + +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.query.QueryManagementService; +import datawave.microservice.query.stream.listener.CountingResponseBodyEmitterListener; +import datawave.microservice.query.stream.listener.StreamingResponseListener; +import datawave.microservice.querymetric.BaseQueryMetric; +import datawave.microservice.querymetric.QueryMetricClient; +import datawave.microservice.querymetric.QueryMetricType; +import datawave.webservice.query.exception.DatawaveErrorCode; +import datawave.webservice.query.exception.NoResultsQueryException; +import datawave.webservice.query.exception.QueryException; +import datawave.webservice.result.BaseQueryResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.MultiValueMap; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.Callable; + +import static datawave.microservice.query.QueryParameters.QUERY_STRING; + +public class StreamingCall implements Callable { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + + final private QueryManagementService queryManagementService; + final private QueryMetricClient queryMetricClient; + final private BaseQueryMetric baseQueryMetric; + + final private MultiValueMap parameters; + final private ProxiedUserDetails currentUser; + final private String queryId; + + final private StreamingResponseListener listener; + + private StreamingCall(Builder builder) { + this.queryManagementService = builder.queryManagementService; + this.queryMetricClient = builder.queryMetricClient; + this.baseQueryMetric = builder.queryManagementService.getBaseQueryMetric().duplicate(); + + this.parameters = builder.parameters; + this.currentUser = builder.currentUser; + this.queryId = builder.queryId; + + this.listener = builder.listener; + } + + @Override + public Void call() throws Exception { + // since this is running in a separate thread, we need to set and use the thread-local baseQueryMetric + ThreadLocal baseQueryMetricOverride = queryManagementService.getBaseQueryMetricOverride(); + baseQueryMetricOverride.set(baseQueryMetric); + + try { + boolean isFinished = false; + do { + final BaseQueryResponse nextResponse = next(queryId, currentUser); + if (nextResponse != null) { + onResponse(nextResponse); + updateMetrics(); + } else { + isFinished = true; + } + } while (!isFinished); + + return null; + } catch (Exception e) { + log.error("Error encountered while processing streaming results for query {}", queryId, e); + throw e; + } finally { + baseQueryMetricOverride.remove(); + listener.cleanup(); + } + } + + private BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) { + BaseQueryResponse nextResponse = null; + try { + long startTimeMillis = System.currentTimeMillis(); + nextResponse = queryManagementService.next(queryId, currentUser); + long nextCallTimeMillis = System.currentTimeMillis() - startTimeMillis; + + BaseQueryMetric.PageMetric lastPageMetric = getLastPageMetric(); + if (lastPageMetric != null) { + lastPageMetric.setCallTime(nextCallTimeMillis); + } + } catch (NoResultsQueryException e) { + log.debug("No results found for query '{}'", parameters.getFirst(QUERY_STRING)); + } catch (QueryException e) { + log.info("Encountered error while getting results for query '{}'", parameters.getFirst(QUERY_STRING)); + } + return nextResponse; + } + + private void onResponse(BaseQueryResponse nextResponse) throws QueryException { + try { + long startBytesWritten = getBytesWritten(); + long startTimeMillis = System.currentTimeMillis(); + listener.onResponse(nextResponse); + long serializationTimeMillis = System.currentTimeMillis() - startTimeMillis; + + BaseQueryMetric.PageMetric lastPageMetric = getLastPageMetric(); + if (lastPageMetric != null) { + lastPageMetric.setSerializationTime(serializationTimeMillis); + lastPageMetric.setBytesWritten(getBytesWritten() - startBytesWritten); + } + } catch (IOException e) { + throw new QueryException(DatawaveErrorCode.UNKNOWN_SERVER_ERROR, e, "Unknown error sending next page for query " + queryId); + } + } + + private long getBytesWritten() { + long bytesWritten = 0L; + if (listener instanceof CountingResponseBodyEmitterListener) { + bytesWritten = ((CountingResponseBodyEmitterListener) listener).getBytesWritten(); + } + return bytesWritten; + } + + private void updateMetrics() { + // send out the metrics + try { + // @formatter:off + queryMetricClient.submit( + new QueryMetricClient.Request.Builder() + .withMetric(baseQueryMetric.duplicate()) + .withMetricType(QueryMetricType.DISTRIBUTED) + .build()); + // @formatter:on + } catch (Exception e) { + log.error("Error updating query metric", e); + } + } + + private BaseQueryMetric.PageMetric getLastPageMetric() { + BaseQueryMetric.PageMetric pageMetric = null; + List pageTimes = baseQueryMetric.getPageTimes(); + if (!pageTimes.isEmpty()) { + pageMetric = pageTimes.get(pageTimes.size() - 1); + } + return pageMetric; + } + + public static class Builder { + private QueryManagementService queryManagementService; + private QueryMetricClient queryMetricClient; + + private MultiValueMap parameters; + private ProxiedUserDetails currentUser; + private String queryId; + + private StreamingResponseListener listener; + + public Builder setQueryManagementService(QueryManagementService queryManagementService) { + this.queryManagementService = queryManagementService; + return this; + } + + public Builder setQueryMetricClient(QueryMetricClient queryMetricClient) { + this.queryMetricClient = queryMetricClient; + return this; + } + + public Builder setParameters(MultiValueMap parameters) { + this.parameters = parameters; + return this; + } + + public Builder setCurrentUser(ProxiedUserDetails currentUser) { + this.currentUser = currentUser; + return this; + } + + public Builder setQueryId(String queryId) { + this.queryId = queryId; + return this; + } + + public Builder setListener(StreamingResponseListener listener) { + this.listener = listener; + return this; + } + + public StreamingCall build() { + return new StreamingCall(this); + } + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java index 99e7eb0d..748c0c4c 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java @@ -100,18 +100,19 @@ public long getBytesWritten() { public MultiValueMap getResponseHeaders() { return responseHeaders; } - } @Override public void doFilterInternal(@NonNull HttpServletRequest request, @NonNull HttpServletResponse response, @NonNull FilterChain chain) throws IOException, ServletException { - preProcess(request, response); if (!(response instanceof CountingHttpServletResponseWrapper)) { response = new CountingHttpServletResponseWrapper(response); } + if (baseMethodStatsContext.getCountingResponseBodyEmitter() == null) { + baseMethodStatsContext.setCountingResponseBodyEmitter(new CountingResponseBodyEmitter((CountingHttpServletResponseWrapper) response)); + } chain.doFilter(request, response); postProcess(request, response); @@ -200,7 +201,7 @@ private ResponseMethodStats createResponseMethodStats(HttpServletRequest request responseStats.callTime = TimeUnit.NANOSECONDS.toMillis(stop - baseMethodStatsContext.getRequestStats().getCallStartTime()); if (response instanceof CountingHttpServletResponseWrapper) { - responseStats.bytesWritten = ((CountingHttpServletResponseWrapper) response).getByteCount(); + responseStats.bytesWritten = ((CountingHttpServletResponseWrapper) response).getBytesWritten(); } for (String header : response.getHeaderNames()) { @@ -210,7 +211,7 @@ private ResponseMethodStats createResponseMethodStats(HttpServletRequest request return responseStats; } - private static class CountingHttpServletResponseWrapper extends HttpServletResponseWrapper { + static class CountingHttpServletResponseWrapper extends HttpServletResponseWrapper { private final ServletResponse response; private CountingServletOutputStream cos; @@ -235,8 +236,8 @@ public ServletOutputStream getOutputStream() throws IOException { return cos; } - public long getByteCount() { - return cos.getByteCount(); + public long getBytesWritten() { + return cos != null ? cos.getBytesWritten() : 0L; } } @@ -267,17 +268,18 @@ public void write(int b) throws IOException { @Override public void write(byte[] b, int off, int len) throws IOException { outputStream.write(b, off, len); - this.count += (long) len; + count += len; } - public long getByteCount() { + public long getBytesWritten() { return count; } } - private static class BaseMethodStatsContext { + public static class BaseMethodStatsContext { private RequestMethodStats requestStats; private ResponseMethodStats responseStats; + private CountingResponseBodyEmitter countingResponseBodyEmitter; public RequestMethodStats getRequestStats() { return requestStats; @@ -294,6 +296,14 @@ public ResponseMethodStats getResponseStats() { public void setResponseStats(ResponseMethodStats responseStats) { this.responseStats = responseStats; } + + public CountingResponseBodyEmitter getCountingResponseBodyEmitter() { + return countingResponseBodyEmitter; + } + + public void setCountingResponseBodyEmitter(CountingResponseBodyEmitter countingResponseBodyEmitter) { + this.countingResponseBodyEmitter = countingResponseBodyEmitter; + } } @Configuration diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java new file mode 100644 index 00000000..fb2ecc7b --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java @@ -0,0 +1,15 @@ +package datawave.microservice.query.web.filter; + +import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyEmitter; + +public class CountingResponseBodyEmitter extends ResponseBodyEmitter { + private final BaseMethodStatsFilter.CountingHttpServletResponseWrapper countingResponse; + + CountingResponseBodyEmitter(BaseMethodStatsFilter.CountingHttpServletResponseWrapper countingResponse) { + this.countingResponse = countingResponse; + } + + public long getBytesWritten() { + return (countingResponse != null) ? countingResponse.getBytesWritten() : 0L; + } +} diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java index e8c33c39..2a061a23 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java @@ -168,7 +168,7 @@ public void postProcess(ResponseMethodStats responseStats) { } } - private static class QueryMetricsEnrichmentContext { + public static class QueryMetricsEnrichmentContext { private String queryId; private EnrichQueryMetrics.MethodType methodType; From 3ac3d04581abbf56f5b959095b810dc7cd263072 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 7 Jan 2022 14:24:48 -0500 Subject: [PATCH 118/218] Added javadoc and removed unnecessary parameters for streaming service. --- .../microservice/query/QueryController.java | 5 +---- .../query/stream/StreamingService.java | 18 +++++++++++++++--- .../CountingResponseBodyEmitterListener.java | 2 +- .../listener/StreamingResponseListener.java | 2 +- .../query/stream/runner/StreamingCall.java | 17 +++-------------- 5 files changed, 21 insertions(+), 23 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 77983fc6..60fc3800 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -283,10 +283,7 @@ public ResponseBodyEmitter execute(@PathVariable String queryLogic, @RequestPara queryMetricsEnrichmentContext.setQueryId(queryId); CountingResponseBodyEmitter emitter = baseMethodStatsContext.getCountingResponseBodyEmitter(); - - // bytesWritten is updated as streaming data is written to the response output stream - streamingService.execute(queryId, parameters, currentUser, new CountingResponseBodyEmitterListener(emitter, headers.getAccept())); - + streamingService.execute(queryId, currentUser, new CountingResponseBodyEmitterListener(emitter, headers.getAccept())); return emitter; } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/StreamingService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/StreamingService.java index e50c4b66..7681e52f 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/StreamingService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/StreamingService.java @@ -7,7 +7,6 @@ import datawave.microservice.querymetric.QueryMetricClient; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.stereotype.Service; -import org.springframework.util.MultiValueMap; @Service public class StreamingService { @@ -22,13 +21,26 @@ public StreamingService(QueryManagementService queryManagementService, QueryMetr this.streamingExecutor = streamingExecutor; } - public void execute(String queryId, MultiValueMap parameters, ProxiedUserDetails currentUser, StreamingResponseListener listener) { + /** + * Gets all pages of results for the given query and streams them to the configured listener. + *

+ * Execute can only be called on a running query.
+ * Execute is a non-blocking call, and will return immediately.
+ * Only the query owner can call execute on the specified query. + * + * @param queryId + * the query id, not null + * @param currentUser + * the user who called this method, not null + * @param listener + * the listener which will handle the result pages, not null + */ + public void execute(String queryId, ProxiedUserDetails currentUser, StreamingResponseListener listener) { // @formatter:off streamingExecutor.submit( new StreamingCall.Builder() .setQueryManagementService(queryManagementService) .setQueryMetricClient(queryMetricClient) - .setParameters(parameters) .setCurrentUser(currentUser) .setQueryId(queryId) .setListener(listener) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java index d4826303..3358b591 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java @@ -22,7 +22,7 @@ public void onResponse(BaseQueryResponse response) throws IOException { } @Override - public void cleanup() { + public void close() { countingEmitter.complete(); } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java index f80b4d6e..2829a54c 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java @@ -8,7 +8,7 @@ public interface StreamingResponseListener { void onResponse(BaseQueryResponse response) throws IOException; - default void cleanup() { + default void close() { // do nothing } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java index 7f4c925b..f0983777 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java @@ -13,14 +13,11 @@ import datawave.webservice.result.BaseQueryResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.util.MultiValueMap; import java.io.IOException; import java.util.List; import java.util.concurrent.Callable; -import static datawave.microservice.query.QueryParameters.QUERY_STRING; - public class StreamingCall implements Callable { private final Logger log = LoggerFactory.getLogger(this.getClass()); @@ -28,7 +25,6 @@ public class StreamingCall implements Callable { final private QueryMetricClient queryMetricClient; final private BaseQueryMetric baseQueryMetric; - final private MultiValueMap parameters; final private ProxiedUserDetails currentUser; final private String queryId; @@ -39,7 +35,6 @@ private StreamingCall(Builder builder) { this.queryMetricClient = builder.queryMetricClient; this.baseQueryMetric = builder.queryManagementService.getBaseQueryMetric().duplicate(); - this.parameters = builder.parameters; this.currentUser = builder.currentUser; this.queryId = builder.queryId; @@ -70,7 +65,7 @@ public Void call() throws Exception { throw e; } finally { baseQueryMetricOverride.remove(); - listener.cleanup(); + listener.close(); } } @@ -86,9 +81,9 @@ private BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) { lastPageMetric.setCallTime(nextCallTimeMillis); } } catch (NoResultsQueryException e) { - log.debug("No results found for query '{}'", parameters.getFirst(QUERY_STRING)); + log.debug("No results found for query '{}'", queryId); } catch (QueryException e) { - log.info("Encountered error while getting results for query '{}'", parameters.getFirst(QUERY_STRING)); + log.info("Encountered error while getting results for query '{}'", queryId); } return nextResponse; } @@ -146,7 +141,6 @@ public static class Builder { private QueryManagementService queryManagementService; private QueryMetricClient queryMetricClient; - private MultiValueMap parameters; private ProxiedUserDetails currentUser; private String queryId; @@ -162,11 +156,6 @@ public Builder setQueryMetricClient(QueryMetricClient queryMetricClient) { return this; } - public Builder setParameters(MultiValueMap parameters) { - this.parameters = parameters; - return this; - } - public Builder setCurrentUser(ProxiedUserDetails currentUser) { this.currentUser = currentUser; return this; From ae64095c0795728616767b80db99642353cf1f3f Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 14 Jan 2022 13:33:23 -0500 Subject: [PATCH 119/218] Added a streaming query test. --- .../microservice/query/QueryController.java | 58 ++- .../query/stream/StreamingService.java | 66 +++- .../CountingResponseBodyEmitterListener.java | 25 +- .../listener/StreamingResponseListener.java | 4 + .../query/stream/runner/StreamingCall.java | 3 +- .../web/filter/BaseMethodStatsFilter.java | 18 +- .../filter/CountingResponseBodyEmitter.java | 3 +- .../query/AbstractQueryServiceTest.java | 9 +- .../query/stream/StreamingServiceTest.java | 369 ++++++++++++++++++ 9 files changed, 522 insertions(+), 33 deletions(-) create mode 100644 query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 60fc3800..d6824812 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -3,6 +3,7 @@ import com.codahale.metrics.annotation.Timed; import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.query.lookup.LookupService; +import datawave.microservice.query.stream.StreamingProperties; import datawave.microservice.query.stream.StreamingService; import datawave.microservice.query.stream.listener.CountingResponseBodyEmitterListener; import datawave.microservice.query.web.annotation.EnrichQueryMetrics; @@ -16,7 +17,9 @@ import datawave.webservice.result.QueryLogicResponse; import datawave.webservice.result.VoidResponse; import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.util.MultiValueMap; import org.springframework.web.bind.annotation.PathVariable; @@ -28,6 +31,7 @@ import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyEmitter; import javax.annotation.security.RolesAllowed; +import java.util.List; @RestController @RequestMapping(path = "/v1", produces = MediaType.APPLICATION_JSON_VALUE) @@ -36,17 +40,20 @@ public class QueryController { private final LookupService lookupService; private final StreamingService streamingService; + private final StreamingProperties streamingProperties; + // Note: baseMethodStatsContext needs to be request scoped private final BaseMethodStatsFilter.BaseMethodStatsContext baseMethodStatsContext; // Note: queryMetricsEnrichmentContest needs to be request scoped private final QueryMetricsEnrichmentFilterAdvice.QueryMetricsEnrichmentContext queryMetricsEnrichmentContext; public QueryController(QueryManagementService queryManagementService, LookupService lookupService, StreamingService streamingService, - BaseMethodStatsFilter.BaseMethodStatsContext baseMethodStatsContext, + StreamingProperties streamingProperties, BaseMethodStatsFilter.BaseMethodStatsContext baseMethodStatsContext, QueryMetricsEnrichmentFilterAdvice.QueryMetricsEnrichmentContext queryMetricsEnrichmentContext) { this.queryManagementService = queryManagementService; this.lookupService = lookupService; this.streamingService = streamingService; + this.streamingProperties = streamingProperties; this.baseMethodStatsContext = baseMethodStatsContext; this.queryMetricsEnrichmentContext = queryMetricsEnrichmentContext; } @@ -271,19 +278,52 @@ public VoidResponse adminRemoveAll(@AuthenticationPrincipal ProxiedUserDetails c return queryManagementService.adminRemoveAll(currentUser); } - @Timed(name = "dw.query.executeQuery", absolute = true) - @RequestMapping(path = "{queryLogic}/execute", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", - "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public ResponseBodyEmitter execute(@PathVariable String queryLogic, @RequestParam MultiValueMap parameters, + @Timed(name = "dw.query.createAndExecuteQuery", absolute = true) + @RequestMapping(path = "{queryLogic}/createAndExecute", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", + "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public ResponseEntity createAndExecute(@PathVariable String queryLogic, @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser, @RequestHeader HttpHeaders headers) throws QueryException { - String queryId = queryManagementService.create(queryLogic, parameters, currentUser).getResult(); + MediaType contentType = determineContentType(headers.getAccept(), MediaType.parseMediaType(streamingProperties.getDefaultContentType())); + CountingResponseBodyEmitter emitter = baseMethodStatsContext.createCountingResponseBodyEmitter(streamingProperties.getCallTimeoutMillis()); + String queryId = streamingService.createAndExecute(queryLogic, parameters, currentUser, new CountingResponseBodyEmitterListener(emitter, contentType)); // unfortunately this needs to be set manually. ResponseBodyAdvice does not run for streaming endpoints queryMetricsEnrichmentContext.setMethodType(EnrichQueryMetrics.MethodType.CREATE); queryMetricsEnrichmentContext.setQueryId(queryId); - CountingResponseBodyEmitter emitter = baseMethodStatsContext.getCountingResponseBodyEmitter(); - streamingService.execute(queryId, currentUser, new CountingResponseBodyEmitterListener(emitter, headers.getAccept())); - return emitter; + return createStreamingResponse(emitter, contentType); + } + + @Timed(name = "dw.query.executeQuery", absolute = true) + @RequestMapping(path = "{queryId}/execute", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public ResponseEntity execute(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser, + @RequestHeader HttpHeaders headers) { + MediaType contentType = determineContentType(headers.getAccept(), MediaType.parseMediaType(streamingProperties.getDefaultContentType())); + CountingResponseBodyEmitter emitter = baseMethodStatsContext.createCountingResponseBodyEmitter(streamingProperties.getCallTimeoutMillis()); + streamingService.execute(queryId, currentUser, new CountingResponseBodyEmitterListener(emitter, contentType)); + + return createStreamingResponse(emitter, contentType); + } + + private MediaType determineContentType(List acceptedMediaTypes, MediaType defaultMediaType) { + MediaType mediaType = null; + + if (acceptedMediaTypes != null && !acceptedMediaTypes.isEmpty()) { + MediaType.sortBySpecificityAndQuality(acceptedMediaTypes); + mediaType = acceptedMediaTypes.get(0); + } + + if (mediaType == null || MediaType.ALL.equals(mediaType)) { + mediaType = defaultMediaType; + } + + return mediaType; + } + + private ResponseEntity createStreamingResponse(ResponseBodyEmitter emitter, MediaType contentType) { + HttpHeaders responseHeaders = new HttpHeaders(); + responseHeaders.setContentType(contentType); + return new ResponseEntity<>(emitter, responseHeaders, HttpStatus.OK); } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/StreamingService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/StreamingService.java index 7681e52f..edc708de 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/StreamingService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/StreamingService.java @@ -5,11 +5,21 @@ import datawave.microservice.query.stream.listener.StreamingResponseListener; import datawave.microservice.query.stream.runner.StreamingCall; import datawave.microservice.querymetric.QueryMetricClient; +import datawave.security.util.ProxiedEntityUtils; +import datawave.webservice.query.exception.BadRequestQueryException; +import datawave.webservice.query.exception.NoResultsQueryException; +import datawave.webservice.query.exception.QueryException; +import datawave.webservice.query.exception.UnauthorizedQueryException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.stereotype.Service; +import org.springframework.util.MultiValueMap; @Service public class StreamingService { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + private final QueryManagementService queryManagementService; private final QueryMetricClient queryMetricClient; @@ -21,6 +31,54 @@ public StreamingService(QueryManagementService queryManagementService, QueryMetr this.streamingExecutor = streamingExecutor; } + /** + * Creates a query using the given query logic and parameters, and streams all pages of results to the configured listener. + *

+ * Created queries will start running immediately.
+ * Auditing is performed before the query is started.
+ * Stop a running query gracefully using {@link QueryManagementService#close} or forcefully using {@link QueryManagementService#cancel}.
+ * Stop, and restart a running query using {@link QueryManagementService#reset}.
+ * Create a copy of a running query using {@link QueryManagementService#duplicate}.
+ * Aside from a limited set of admin actions, only the query owner can act on a running query. + * + * @param queryLogicName + * the requested query logic, not null + * @param parameters + * the query parameters, not null + * @param currentUser + * the user who called this method, not null + * @return the query id + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws NoResultsQueryException + * if no query results are found + * @throws QueryException + * if there is an unknown error + */ + public String createAndExecute(String queryLogicName, MultiValueMap parameters, ProxiedUserDetails currentUser, + StreamingResponseListener listener) throws QueryException { + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + if (log.isDebugEnabled()) { + log.info("Request: {}/createAndExecute from {} with params: {}", queryLogicName, user, parameters); + } else { + log.info("Request: {}/createAndExecute from {}", queryLogicName, user); + } + + String queryId = queryManagementService.create(queryLogicName, parameters, currentUser).getResult(); + submitStreamingCall(queryId, currentUser, listener); + return queryId; + } + /** * Gets all pages of results for the given query and streams them to the configured listener. *

@@ -36,13 +94,19 @@ public StreamingService(QueryManagementService queryManagementService, QueryMetr * the listener which will handle the result pages, not null */ public void execute(String queryId, ProxiedUserDetails currentUser, StreamingResponseListener listener) { + log.info("Request: {}/execute from {}", queryId, ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName())); + + submitStreamingCall(queryId, currentUser, listener); + } + + private void submitStreamingCall(String queryId, ProxiedUserDetails currentUser, StreamingResponseListener listener) { // @formatter:off streamingExecutor.submit( new StreamingCall.Builder() .setQueryManagementService(queryManagementService) .setQueryMetricClient(queryMetricClient) - .setCurrentUser(currentUser) .setQueryId(queryId) + .setCurrentUser(currentUser) .setListener(listener) .build()); // @formatter:on diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java index 3358b591..9cc8c486 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java @@ -5,15 +5,14 @@ import org.springframework.http.MediaType; import java.io.IOException; -import java.util.List; public class CountingResponseBodyEmitterListener implements StreamingResponseListener { private final CountingResponseBodyEmitter countingEmitter; private final MediaType mediaType; - public CountingResponseBodyEmitterListener(CountingResponseBodyEmitter countingEmitter, List mediaTypes) { + public CountingResponseBodyEmitterListener(CountingResponseBodyEmitter countingEmitter, MediaType mediaType) { this.countingEmitter = countingEmitter; - this.mediaType = determineMediaType(mediaTypes); + this.mediaType = mediaType; } @Override @@ -26,16 +25,20 @@ public void close() { countingEmitter.complete(); } - public long getBytesWritten() { - return (countingEmitter != null) ? countingEmitter.getBytesWritten() : 0L; + @Override + public void closeWithError(Throwable t) { + countingEmitter.completeWithError(t); + } + + public CountingResponseBodyEmitter getCountingEmitter() { + return countingEmitter; } - private MediaType determineMediaType(List acceptedMediaTypes) { - MediaType mediaType = null; - if (acceptedMediaTypes != null && !acceptedMediaTypes.isEmpty()) { - MediaType.sortBySpecificityAndQuality(acceptedMediaTypes); - mediaType = acceptedMediaTypes.get(0); - } + public MediaType getMediaType() { return mediaType; } + + public long getBytesWritten() { + return (countingEmitter != null) ? countingEmitter.getBytesWritten() : 0L; + } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java index 2829a54c..b8d00f26 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java @@ -11,4 +11,8 @@ public interface StreamingResponseListener { default void close() { // do nothing } + + default void closeWithError(Throwable t) { + // do nothing + } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java index f0983777..6ce9370c 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java @@ -59,13 +59,14 @@ public Void call() throws Exception { } } while (!isFinished); + listener.close(); return null; } catch (Exception e) { log.error("Error encountered while processing streaming results for query {}", queryId, e); + listener.closeWithError(e); throw e; } finally { baseQueryMetricOverride.remove(); - listener.close(); } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java index 748c0c4c..c6906756 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java @@ -110,8 +110,8 @@ public void doFilterInternal(@NonNull HttpServletRequest request, @NonNull HttpS if (!(response instanceof CountingHttpServletResponseWrapper)) { response = new CountingHttpServletResponseWrapper(response); } - if (baseMethodStatsContext.getCountingResponseBodyEmitter() == null) { - baseMethodStatsContext.setCountingResponseBodyEmitter(new CountingResponseBodyEmitter((CountingHttpServletResponseWrapper) response)); + if (baseMethodStatsContext.getCountingHttpServletResponseWrapper() == null) { + baseMethodStatsContext.setCountingHttpServletResponseWrapper((CountingHttpServletResponseWrapper) response); } chain.doFilter(request, response); @@ -279,7 +279,7 @@ public long getBytesWritten() { public static class BaseMethodStatsContext { private RequestMethodStats requestStats; private ResponseMethodStats responseStats; - private CountingResponseBodyEmitter countingResponseBodyEmitter; + private CountingHttpServletResponseWrapper countingHttpServletResponseWrapper; public RequestMethodStats getRequestStats() { return requestStats; @@ -297,12 +297,16 @@ public void setResponseStats(ResponseMethodStats responseStats) { this.responseStats = responseStats; } - public CountingResponseBodyEmitter getCountingResponseBodyEmitter() { - return countingResponseBodyEmitter; + CountingHttpServletResponseWrapper getCountingHttpServletResponseWrapper() { + return countingHttpServletResponseWrapper; } - public void setCountingResponseBodyEmitter(CountingResponseBodyEmitter countingResponseBodyEmitter) { - this.countingResponseBodyEmitter = countingResponseBodyEmitter; + public void setCountingHttpServletResponseWrapper(CountingHttpServletResponseWrapper countingHttpServletResponseWrapper) { + this.countingHttpServletResponseWrapper = countingHttpServletResponseWrapper; + } + + public CountingResponseBodyEmitter createCountingResponseBodyEmitter(Long timeout) { + return new CountingResponseBodyEmitter(timeout, countingHttpServletResponseWrapper); } } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java index fb2ecc7b..5bd267c7 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java @@ -5,7 +5,8 @@ public class CountingResponseBodyEmitter extends ResponseBodyEmitter { private final BaseMethodStatsFilter.CountingHttpServletResponseWrapper countingResponse; - CountingResponseBodyEmitter(BaseMethodStatsFilter.CountingHttpServletResponseWrapper countingResponse) { + public CountingResponseBodyEmitter(Long timeout, BaseMethodStatsFilter.CountingHttpServletResponseWrapper countingResponse) { + super(timeout); this.countingResponse = countingResponse; } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java index 43f7fbd9..bb3c4f9b 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java @@ -62,6 +62,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -87,6 +88,8 @@ public abstract class AbstractQueryServiceTest { protected static final String TEST_QUERY_BEGIN = "20000101 000000.000"; protected static final String TEST_QUERY_END = "20500101 000000.000"; protected static final String TEST_VISIBILITY_MARKING = "ALL"; + protected static final long TEST_MAX_RESULTS_OVERRIDE = 369L; + protected static final long TEST_PAGESIZE = 123L; @LocalServerPort protected int webServicePort; @@ -147,7 +150,7 @@ protected void publishEventsToQueue(String queryId, int numEvents, MultiValueMap List fields = new ArrayList<>(); for (Map.Entry> entry : fieldValues.entrySet()) { for (String value : entry.getValue()) { - fields.add(new DefaultField(entry.getKey(), visibility, currentTime, value)); + fields.add(new DefaultField(entry.getKey(), visibility, new HashMap<>(), currentTime, value)); } } event.setFields(fields); @@ -365,8 +368,8 @@ protected MultiValueMap createParams() { map.set(DefaultQueryParameters.QUERY_END, TEST_QUERY_END); map.set(ColumnVisibilitySecurityMarking.VISIBILITY_MARKING, TEST_VISIBILITY_MARKING); map.set(QUERY_MAX_CONCURRENT_TASKS, Integer.toString(1)); - map.set(QUERY_MAX_RESULTS_OVERRIDE, Long.toString(369)); - map.set(QUERY_PAGESIZE, Long.toString(123)); + map.set(QUERY_MAX_RESULTS_OVERRIDE, Long.toString(TEST_MAX_RESULTS_OVERRIDE)); + map.set(QUERY_PAGESIZE, Long.toString(TEST_PAGESIZE)); return map; } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java new file mode 100644 index 00000000..c731ae62 --- /dev/null +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java @@ -0,0 +1,369 @@ +package datawave.microservice.query.stream; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.module.jaxb.JaxbAnnotationModule; +import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.query.AbstractQueryServiceTest; +import datawave.microservice.query.DefaultQueryParameters; +import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; +import datawave.webservice.query.result.event.DefaultEvent; +import datawave.webservice.result.DefaultEventQueryResponse; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.RequestEntity; +import org.springframework.http.ResponseEntity; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.util.UriComponents; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Unmarshaller; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +@RunWith(SpringRunner.class) +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +@ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) +public class StreamingServiceTest extends AbstractQueryServiceTest { + + @Before + public void setup() { + super.setup(); + } + + @After + public void teardown() throws Exception { + super.teardown(); + } + + @Test + public void testExecuteSuccess() throws Throwable { + ProxiedUserDetails authUser = createUserDetails(); + + // create a valid query + String queryId = createQuery(authUser, createParams()); + + // pump enough results into the queue to trigger a complete page + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + int pageSize = queryStatus.getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("LOKI", "ALLIGATOR"); + fieldValues.add("LOKI", "CLASSIC"); + + // @formatter:off + publishEventsToQueue( + queryId, + (int)TEST_MAX_RESULTS_OVERRIDE, + fieldValues, + "ALL"); + // @formatter:on + + // make the execute call asynchronously + Future> future = execute(authUser, queryId); + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // verify some headers + Assert.assertEquals(MediaType.APPLICATION_XML, response.getHeaders().getContentType()); + + int pageNumber = 1; + + List queryResponses = parseXMLBaseQueryResponses(response.getBody()); + for (DefaultEventQueryResponse queryResponse : queryResponses) { + // verify the query response + // @formatter:off + assertQueryResponse( + queryId, + "EventQuery", + pageNumber++, + false, + queryResponse.getOperationTimeMS(), + 1, + Collections.singletonList("LOKI"), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Arrays.asList("LOKI", "LOKI"), + Arrays.asList("ALLIGATOR", "CLASSIC"), + event); + // @formatter:on + } + + // verify that the next event was published + Assert.assertEquals(6, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + } + + @Test + public void testCreateAndExecuteSuccess() throws Throwable { + ProxiedUserDetails authUser = createUserDetails(); + + final String query = TEST_QUERY_STRING + " CREATE_AND_NEXT:TEST"; + + MultiValueMap params = createParams(); + params.set(DefaultQueryParameters.QUERY_STRING, query); + + // make the execute call asynchronously + Future> future = createAndExecute(authUser, params); + + long startTime = System.currentTimeMillis(); + QueryStatus queryStatus = null; + while (queryStatus == null && (System.currentTimeMillis() - startTime) < TimeUnit.SECONDS.toMillis(5)) { + queryStatus = queryStorageCache.getQueryStatus().stream().filter(x -> x.getQuery().getQuery().equals(query)).findAny().orElse(null); + if (queryStatus == null) { + Thread.sleep(500); + } + } + + String queryId = queryStatus.getQueryKey().getQueryId(); + + // pump enough results into the queue to trigger a complete page + int pageSize = queryStatus.getQuery().getPagesize(); + + // test field value pairings + MultiValueMap fieldValues = new LinkedMultiValueMap<>(); + fieldValues.add("LOKI", "ALLIGATOR"); + fieldValues.add("LOKI", "CLASSIC"); + + // @formatter:off + publishEventsToQueue( + queryId, + (int)TEST_MAX_RESULTS_OVERRIDE, + fieldValues, + "ALL"); + // @formatter:on + + // the response should come back right away + ResponseEntity response = future.get(); + + Assert.assertEquals(200, response.getStatusCodeValue()); + + // verify some headers + Assert.assertEquals(MediaType.APPLICATION_XML, response.getHeaders().getContentType()); + + int pageNumber = 1; + + List queryResponses = parseXMLBaseQueryResponses(response.getBody()); + for (DefaultEventQueryResponse queryResponse : queryResponses) { + // verify the query response + // @formatter:off + assertQueryResponse( + queryId, + "EventQuery", + pageNumber++, + false, + queryResponse.getOperationTimeMS(), + 1, + Collections.singletonList("LOKI"), + pageSize, + Objects.requireNonNull(queryResponse)); + // @formatter:on + + // validate one of the events + DefaultEvent event = (DefaultEvent) queryResponse.getEvents().get(0); + // @formatter:off + assertDefaultEvent( + Arrays.asList("LOKI", "LOKI"), + Arrays.asList("ALLIGATOR", "CLASSIC"), + event); + // @formatter:on + } + + // verify that the next event was published + Assert.assertEquals(6, queryRequestEvents.size()); + // @formatter:off + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CREATE, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.NEXT, + queryId, + queryRequestEvents.removeLast()); + assertQueryRequestEvent( + "executor-unassigned:**", + QueryRequest.Method.CLOSE, + queryId, + queryRequestEvents.removeLast()); + // @formatter:on + + } + + @Test + public void testBatchLookupUUIDSuccess() throws Exception {} + + @Test + public void testLookupContentUUIDSuccess() throws Exception {} + + @Test + public void testBatchLookupContentUUIDSuccess() throws Exception {} + + @Test + public void testBatchLookupUUIDFailure_noLookupUUIDPairs() throws Exception {} + + @Test + public void testBatchLookupUUIDFailure_mixedQueryLogics() throws Exception {} + + @Test + public void testBatchLookupUUIDFailure_nullUUIDType() throws Exception {} + + @Test + public void testBatchLookupUUIDFailure_emptyUUIDFieldValue() throws Exception {} + + @Test + public void testBatchLookupUUIDFailure_invalidUUIDPair() throws Exception {} + + @Test + public void testBatchLookupUUIDFailure_tooManyTerms() throws Exception {} + + @Test + public void testBatchLookupUUIDFailure_nonLookupQueryLogic() throws Exception {} + + protected Future> createAndExecute(ProxiedUserDetails authUser, MultiValueMap map) { + UriComponents uri = createUri("EventQuery/createAndExecute"); + + // not testing audit with this method + auditIgnoreSetup(); + + MultiValueMap headers = new LinkedMultiValueMap<>(); + headers.set(HttpHeaders.ACCEPT, MediaType.APPLICATION_XML_VALUE); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, map, headers, HttpMethod.POST, uri); + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + } + + protected Future> execute(ProxiedUserDetails authUser, String queryId) { + UriComponents uri = createUri(queryId + "/execute"); + + MultiValueMap headers = new LinkedMultiValueMap<>(); + headers.set(HttpHeaders.ACCEPT, MediaType.APPLICATION_XML_VALUE); + + RequestEntity> requestEntity = jwtRestTemplate.createRequestEntity(authUser, null, headers, HttpMethod.GET, uri); + return Executors.newSingleThreadExecutor().submit(() -> jwtRestTemplate.exchange(requestEntity, String.class)); + } + + private ObjectMapper createJSONObjectMapper() { + ObjectMapper mapper = new ObjectMapper(); + mapper.registerModule(new JaxbAnnotationModule()); + mapper.configure(MapperFeature.USE_WRAPPER_NAME_AS_PROPERTY_NAME, true); + return mapper; + } + + protected List parseJSONBaseQueryResponses(String responseBody) throws JsonProcessingException { + String delimiter = "}{"; + ObjectMapper mapper = createJSONObjectMapper(); + List baseResponses = new ArrayList<>(); + int start = 0; + int end = responseBody.indexOf(delimiter) + 1; + while (end > start) { + String stringResponse = responseBody.substring(start, end); + baseResponses.add(mapper.readValue(stringResponse, DefaultEventQueryResponse.class)); + start = end; + end = responseBody.indexOf(delimiter, start) + 1; + if (end == 0) { + end = responseBody.length(); + } + } + return baseResponses; + } + + protected List parseXMLBaseQueryResponses(String responseBody) throws JAXBException { + String delimiter = ""; + List baseResponses = new ArrayList<>(); + int start = responseBody.indexOf(delimiter); + int end = responseBody.indexOf(delimiter, start + delimiter.length()); + while (end > start) { + String stringResponse = responseBody.substring(start, end); + + JAXBContext jaxbContext = JAXBContext.newInstance(DefaultEventQueryResponse.class); + Unmarshaller unmarshaller = jaxbContext.createUnmarshaller(); + baseResponses.add((DefaultEventQueryResponse) unmarshaller.unmarshal(new StringReader(stringResponse))); + + start = end; + end = responseBody.indexOf(delimiter, start + delimiter.length()); + if (end == -1) { + end = responseBody.length(); + } + } + return baseResponses; + } +} From 45a78c7bc537239bae573d6985e3bdf243cc343d Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 20 Jan 2022 16:40:37 -0500 Subject: [PATCH 120/218] Updated guava to the latest version across the board to fix a hazelcast-related serialization issue. --- query-microservices/query-service/service/pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/query-microservices/query-service/service/pom.xml b/query-microservices/query-service/service/pom.xml index 2478a77d..a6099a52 100644 --- a/query-microservices/query-service/service/pom.xml +++ b/query-microservices/query-service/service/pom.xml @@ -54,10 +54,6 @@ gov.nsa.datawave.microservice query-api - - gov.nsa.datawave.microservice - spring-boot-starter-datawave - gov.nsa.datawave.microservice spring-boot-starter-datawave-audit From 22c298a44bd1fc2421c1f0b11474472ed098b9d4 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 14 Jan 2022 15:30:44 -0500 Subject: [PATCH 121/218] Updated lookup service to be able to stream results. --- .../microservice/query/QueryController.java | 62 ++++-- .../query/lookup/LookupService.java | 178 ++++++++++-------- .../query/stream/StreamingService.java | 2 + .../query/stream/StreamingServiceTest.java | 31 --- 4 files changed, 156 insertions(+), 117 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index d6824812..766bf3eb 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -33,6 +33,10 @@ import javax.annotation.security.RolesAllowed; import java.util.List; +import static datawave.microservice.query.lookup.LookupService.LOOKUP_STREAMING; +import static datawave.microservice.query.lookup.LookupService.LOOKUP_UUID_PAIRS; +import static datawave.services.query.logic.lookup.LookupQueryLogic.LOOKUP_KEY_VALUE_DELIMITER; + @RestController @RequestMapping(path = "/v1", produces = MediaType.APPLICATION_JSON_VALUE) public class QueryController { @@ -102,33 +106,67 @@ public GenericResponse predict(@PathVariable String queryLogic, @Request @Timed(name = "dw.query.lookupUUID", absolute = true) @RequestMapping(path = "lookupUUID/{uuidType}/{uuid}", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public BaseQueryResponse lookupUUID(@PathVariable(required = false) String uuidType, @PathVariable(required = false) String uuid, - @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { - return lookupService.lookupUUID(uuidType, uuid, parameters, currentUser); + public Object lookupUUID(@PathVariable(required = false) String uuidType, @PathVariable(required = false) String uuid, + @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser, + @RequestHeader HttpHeaders headers) throws QueryException { + parameters.add(LOOKUP_UUID_PAIRS, String.join(LOOKUP_KEY_VALUE_DELIMITER, uuidType, uuid)); + + if (Boolean.parseBoolean(parameters.getFirst(LOOKUP_STREAMING))) { + MediaType contentType = determineContentType(headers.getAccept(), MediaType.parseMediaType(streamingProperties.getDefaultContentType())); + CountingResponseBodyEmitter emitter = baseMethodStatsContext.createCountingResponseBodyEmitter(streamingProperties.getCallTimeoutMillis()); + lookupService.lookupUUID(parameters, currentUser, new CountingResponseBodyEmitterListener(emitter, contentType)); + return emitter; + } else { + return lookupService.lookupUUID(parameters, currentUser); + } } @Timed(name = "dw.query.lookupUUIDBatch", absolute = true) @RequestMapping(path = "lookupUUID", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public BaseQueryResponse lookupUUIDBatch(@PathVariable(required = false) String uuidType, @PathVariable(required = false) String uuid, - @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { - return lookupService.lookupUUID(parameters, currentUser); + public Object lookupUUIDBatch(@RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser, + @RequestHeader HttpHeaders headers) throws QueryException { + if (Boolean.parseBoolean(parameters.getFirst(LOOKUP_STREAMING))) { + MediaType contentType = determineContentType(headers.getAccept(), MediaType.parseMediaType(streamingProperties.getDefaultContentType())); + CountingResponseBodyEmitter emitter = baseMethodStatsContext.createCountingResponseBodyEmitter(streamingProperties.getCallTimeoutMillis()); + lookupService.lookupUUID(parameters, currentUser, new CountingResponseBodyEmitterListener(emitter, contentType)); + return emitter; + } else { + return lookupService.lookupUUID(parameters, currentUser); + } } @Timed(name = "dw.query.lookupContentUUID", absolute = true) @RequestMapping(path = "lookupContentUUID/{uuidType}/{uuid}", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public BaseQueryResponse lookupContentUUID(@PathVariable(required = false) String uuidType, @PathVariable(required = false) String uuid, - @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { - return lookupService.lookupContentUUID(uuidType, uuid, parameters, currentUser); + public Object lookupContentUUID(@PathVariable(required = false) String uuidType, @PathVariable(required = false) String uuid, + @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser, + @RequestHeader HttpHeaders headers) throws QueryException { + parameters.add(LOOKUP_UUID_PAIRS, String.join(LOOKUP_KEY_VALUE_DELIMITER, uuidType, uuid)); + + if (Boolean.parseBoolean(parameters.getFirst(LOOKUP_STREAMING))) { + MediaType contentType = determineContentType(headers.getAccept(), MediaType.parseMediaType(streamingProperties.getDefaultContentType())); + CountingResponseBodyEmitter emitter = baseMethodStatsContext.createCountingResponseBodyEmitter(streamingProperties.getCallTimeoutMillis()); + lookupService.lookupContentUUID(parameters, currentUser, new CountingResponseBodyEmitterListener(emitter, contentType)); + return emitter; + } else { + return lookupService.lookupContentUUID(parameters, currentUser); + } } @Timed(name = "dw.query.lookupContentUUIDBatch", absolute = true) @RequestMapping(path = "lookupContentUUID", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public BaseQueryResponse lookupContentUUIDBatch(@PathVariable(required = false) String uuidType, @PathVariable(required = false) String uuid, - @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { - return lookupService.lookupContentUUID(parameters, currentUser); + public Object lookupContentUUIDBatch(@RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser, + @RequestHeader HttpHeaders headers) throws QueryException { + if (Boolean.parseBoolean(parameters.getFirst(LOOKUP_STREAMING))) { + MediaType contentType = determineContentType(headers.getAccept(), MediaType.parseMediaType(streamingProperties.getDefaultContentType())); + CountingResponseBodyEmitter emitter = baseMethodStatsContext.createCountingResponseBodyEmitter(streamingProperties.getCallTimeoutMillis()); + lookupService.lookupContentUUID(parameters, currentUser, new CountingResponseBodyEmitterListener(emitter, contentType)); + return emitter; + } else { + return lookupService.lookupContentUUID(parameters, currentUser); + } } @Timed(name = "dw.query.createAndNext", absolute = true) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/lookup/LookupService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/lookup/LookupService.java index 0a38693d..0eb2d641 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/lookup/LookupService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/lookup/LookupService.java @@ -6,6 +6,8 @@ import datawave.microservice.query.DefaultQueryParameters; import datawave.microservice.query.QueryManagementService; import datawave.microservice.query.QueryParameters; +import datawave.microservice.query.stream.StreamingService; +import datawave.microservice.query.stream.listener.StreamingResponseListener; import datawave.query.data.UUIDType; import datawave.security.util.ProxiedEntityUtils; import datawave.services.query.logic.QueryLogic; @@ -48,6 +50,7 @@ public class LookupService { public static final String LOOKUP_UUID_PAIRS = "uuidPairs"; public static final String LUCENE_UUID_SYNTAX = "LUCENE-UUID"; + public static final String LOOKUP_STREAMING = "streaming"; private static final String CONTENT_QUERY_TERM_DELIMITER = ":"; private static final String CONTENT_QUERY_VALUE_DELIMITER = "/"; @@ -58,29 +61,30 @@ public class LookupService { private final QueryLogicFactory queryLogicFactory; private final QueryManagementService queryManagementService; + private final StreamingService streamingService; - public LookupService(LookupProperties lookupProperties, QueryLogicFactory queryLogicFactory, QueryManagementService queryManagementService) { + public LookupService(LookupProperties lookupProperties, QueryLogicFactory queryLogicFactory, QueryManagementService queryManagementService, + StreamingService streamingService) { this.lookupProperties = lookupProperties; this.queryLogicFactory = queryLogicFactory; this.queryManagementService = queryManagementService; + this.streamingService = streamingService; } /** - * Creates an event lookup query using the query logic associated with the given uuid type and parameters, and returns the first page of results. + * Creates a batch event lookup query using the query logic associated with the given uuid type(s) and parameters, and returns the first page of results. *

* Lookup queries will start running immediately.
* Auditing is performed before the query is started.
+ * Each of the uuid pairs must map to the same query logic.
* After the first page is returned, the query will be closed. * - * @param uuidType - * the uuid type, not null - * @param uuid - * the uuid, not null * @param parameters * the query parameters, not null * @param currentUser * the user who called this method, not null - * @return a base query response containing the first page of results + * @param listener + * the listener which will handle the result pages, not null * @throws BadRequestQueryException * if parameter validation fails * @throws BadRequestQueryException @@ -102,19 +106,16 @@ public LookupService(LookupProperties lookupProperties, QueryLogicFactory queryL * @throws QueryException * if there is an unknown error */ - public BaseQueryResponse lookupUUID(String uuidType, String uuid, MultiValueMap parameters, ProxiedUserDetails currentUser) - throws QueryException { + public void lookupUUID(MultiValueMap parameters, ProxiedUserDetails currentUser, StreamingResponseListener listener) throws QueryException { String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); if (log.isDebugEnabled()) { - log.info("Request: lookupUUID/{}/{} from {} with params: {}", uuidType, uuid, user, parameters); + log.info("Request: lookupUUID from {} with params: {}", user, parameters); } else { - log.info("Request: lookupUUID/{}/{} from {}", uuidType, uuid, user); + log.info("Request: lookupUUID from {}", user); } - parameters.add(LOOKUP_UUID_PAIRS, String.join(LOOKUP_KEY_VALUE_DELIMITER, uuidType, uuid)); - try { - return lookup(parameters, currentUser, false); + lookup(parameters, currentUser, listener); } catch (QueryException e) { throw e; } catch (Exception e) { @@ -160,13 +161,13 @@ public BaseQueryResponse lookupUUID(String uuidType, String uuid, MultiValueMap< public BaseQueryResponse lookupUUID(MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); if (log.isDebugEnabled()) { - log.info("Request: lookupUUID (batch) from {} with params: {}", user, parameters); + log.info("Request: lookupUUID from {} with params: {}", user, parameters); } else { - log.info("Request: lookupUUID (batch) from {}", user); + log.info("Request: lookupUUID from {}", user); } try { - return lookup(parameters, currentUser, false); + return lookup(parameters, currentUser, null); } catch (QueryException e) { throw e; } catch (Exception e) { @@ -176,20 +177,19 @@ public BaseQueryResponse lookupUUID(MultiValueMap parameters, Pro } /** - * Creates a content lookup query using the query logic associated with the given uuid type and parameters, and returns the first page of results. + * Creates a batch content lookup query using the query logic associated with the given uuid type(s) and parameters, and returns the first page of results. *

* Lookup queries will start running immediately.
* Auditing is performed before the query is started.
+ * Each of the uuid pairs must map to the same query logic.
* After the first page is returned, the query will be closed. * - * @param uuidType - * the uuid type, not null - * @param uuid - * the uuid, not null * @param parameters * the query parameters, not null * @param currentUser * the user who called this method, not null + * @param listener + * the listener which will handle the result pages, not null * @return a base query response containing the first page of results * @throws BadRequestQueryException * if parameter validation fails @@ -212,20 +212,18 @@ public BaseQueryResponse lookupUUID(MultiValueMap parameters, Pro * @throws QueryException * if there is an unknown error */ - public BaseQueryResponse lookupContentUUID(String uuidType, String uuid, MultiValueMap parameters, ProxiedUserDetails currentUser) + public T lookupContentUUID(MultiValueMap parameters, ProxiedUserDetails currentUser, StreamingResponseListener listener) throws QueryException { String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); if (log.isDebugEnabled()) { - log.info("Request: lookupContentUUID/{}/{} from {} with params: {}", uuidType, uuid, user, parameters); + log.info("Request: lookupContentUUID from {} with params: {}", user, parameters); } else { - log.info("Request: lookupContentUUID/{}/{} from {}", uuidType, uuid, user); + log.info("Request: lookupContentUUID from {}", user); } - parameters.add(LOOKUP_UUID_PAIRS, String.join(LOOKUP_KEY_VALUE_DELIMITER, uuidType, uuid)); - try { // first lookup the UUIDs, then get the content for each UUID - return lookup(parameters, currentUser, true); + return lookupContent(parameters, currentUser, listener); } catch (QueryException e) { throw e; } catch (Exception e) { @@ -271,14 +269,14 @@ public BaseQueryResponse lookupContentUUID(String uuidType, String uuid, MultiVa public BaseQueryResponse lookupContentUUID(MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); if (log.isDebugEnabled()) { - log.info("Request: lookupContentUUID (batch) from {} with params: {}", user, parameters); + log.info("Request: lookupContentUUID from {} with params: {}", user, parameters); } else { - log.info("Request: lookupContentUUID (batch) from {}", user); + log.info("Request: lookupContentUUID from {}", user); } try { // first lookup the UUIDs, then get the content for each UUID - return lookup(parameters, currentUser, true); + return lookupContent(parameters, currentUser, null); } catch (QueryException e) { throw e; } catch (Exception e) { @@ -287,7 +285,7 @@ public BaseQueryResponse lookupContentUUID(MultiValueMap paramete } } - private BaseQueryResponse lookup(MultiValueMap parameters, ProxiedUserDetails currentUser, boolean isContentLookup) throws QueryException { + private T lookup(MultiValueMap parameters, ProxiedUserDetails currentUser, StreamingResponseListener listener) throws QueryException { List lookupTerms = parameters.get(LOOKUP_UUID_PAIRS); if (lookupTerms == null || lookupTerms.isEmpty()) { log.error("Unable to validate lookupUUID parameters: No UUID Pairs"); @@ -299,32 +297,17 @@ private BaseQueryResponse lookup(MultiValueMap parameters, Proxie // validate the lookup terms LookupQueryLogic lookupQueryLogic = validateLookupTerms(lookupTerms, lookupTermMap); - BaseQueryResponse response = null; - - boolean isEventLookupRequired = lookupQueryLogic.isEventLookupRequired(lookupTermMap); - - // do the event lookup if necessary - if (!isContentLookup || isEventLookupRequired) { - response = lookupEvents(lookupTermMap, lookupQueryLogic, new LinkedMultiValueMap<>(parameters), currentUser); - } - - // perform the content lookup if necessary - if (isContentLookup) { - Set contentLookupTerms; - if (!isEventLookupRequired) { - contentLookupTerms = lookupQueryLogic.getContentLookupTerms(lookupTermMap); - } else { - contentLookupTerms = getContentLookupTerms(response); - } - - response = lookupContent(contentLookupTerms, new LinkedMultiValueMap<>(parameters), currentUser); - } - - return response; + // perform the event lookup + return lookupEvents(lookupTermMap, lookupQueryLogic, new LinkedMultiValueMap<>(parameters), currentUser, listener); } private BaseQueryResponse lookupEvents(MultiValueMap lookupTermMap, LookupQueryLogic lookupQueryLogic, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + return lookupEvents(lookupTermMap, lookupQueryLogic, parameters, currentUser, null); + } + + private T lookupEvents(MultiValueMap lookupTermMap, LookupQueryLogic lookupQueryLogic, MultiValueMap parameters, + ProxiedUserDetails currentUser, StreamingResponseListener listener) throws QueryException { String queryId = null; try { // add the query logic name and query string to our parameters @@ -334,16 +317,21 @@ private BaseQueryResponse lookupEvents(MultiValueMap lookupTermMa // update the parameters for query setEventQueryParameters(parameters, currentUser); - // run the query - BaseQueryResponse nextResponse = queryManagementService.createAndNext(parameters.getFirst(QUERY_LOGIC_NAME), parameters, currentUser); + // create the query + queryId = queryManagementService.create(parameters.getFirst(QUERY_LOGIC_NAME), parameters, currentUser).getResult(); - // save the query id - queryId = nextResponse.getQueryId(); - - return nextResponse; + if (listener != null) { + // stream results to the listener + streamingService.execute(queryId, currentUser, listener); + return null; + } else { + // get the first page of results + // noinspection unchecked + return (T) queryManagementService.next(queryId, currentUser); + } } finally { // close the query if applicable - if (queryId != null) { + if (listener == null && queryId != null) { queryManagementService.close(queryId, currentUser); } } @@ -464,6 +452,39 @@ protected void setOptionalQueryParameters(MultiValueMap parameter } } + private T lookupContent(MultiValueMap parameters, ProxiedUserDetails currentUser, StreamingResponseListener listener) + throws QueryException { + List lookupTerms = parameters.get(LOOKUP_UUID_PAIRS); + if (lookupTerms == null || lookupTerms.isEmpty()) { + log.error("Unable to validate lookupContentUUID parameters: No UUID Pairs"); + throw new BadRequestQueryException(DatawaveErrorCode.MISSING_REQUIRED_PARAMETER); + } + + MultiValueMap lookupTermMap = new LinkedMultiValueMap<>(); + + // validate the lookup terms + LookupQueryLogic lookupQueryLogic = validateLookupTerms(lookupTerms, lookupTermMap); + + BaseQueryResponse response = null; + + boolean isEventLookupRequired = lookupQueryLogic.isEventLookupRequired(lookupTermMap); + + // do the event lookup if necessary + if (isEventLookupRequired) { + response = lookupEvents(lookupTermMap, lookupQueryLogic, new LinkedMultiValueMap<>(parameters), currentUser); + } + + // perform the content lookup + Set contentLookupTerms; + if (!isEventLookupRequired) { + contentLookupTerms = lookupQueryLogic.getContentLookupTerms(lookupTermMap); + } else { + contentLookupTerms = getContentLookupTerms(response); + } + + return lookupContent(contentLookupTerms, parameters, currentUser, listener); + } + private Set getContentLookupTerms(BaseQueryResponse response) { Set contentQueries = new HashSet<>(); @@ -479,31 +500,40 @@ private String createContentLookupTerm(Metadata eventMetadata) { + String.join(CONTENT_QUERY_VALUE_DELIMITER, eventMetadata.getRow(), eventMetadata.getDataType(), eventMetadata.getInternalId()); } - private BaseQueryResponse lookupContent(Set contentLookupTerms, MultiValueMap parameters, ProxiedUserDetails currentUser) { + private T lookupContent(Set contentLookupTerms, MultiValueMap parameters, ProxiedUserDetails currentUser, + StreamingResponseListener listener) throws QueryException { // create queries from the content lookup terms List contentQueries = createContentQueries(contentLookupTerms); EventQueryResponseBase mergedResponse = null; for (String contentQuery : contentQueries) { + MultiValueMap queryParameters = new LinkedMultiValueMap<>(parameters); + // set the content query string - parameters.put(QUERY_STRING, Collections.singletonList(contentQuery)); + queryParameters.put(QUERY_STRING, Collections.singletonList(contentQuery)); // update parameters for the query - setContentQueryParameters(parameters, currentUser); + setContentQueryParameters(queryParameters, currentUser); - // run the query - EventQueryResponseBase contentQueryResponse = runContentQuery(parameters, currentUser); - - if (contentQueryResponse != null) { - if (mergedResponse == null) { - mergedResponse = contentQueryResponse; - } else { - mergedResponse.merge(contentQueryResponse); + if (listener != null) { + streamingService.createAndExecute(parameters.getFirst(QUERY_LOGIC_NAME), parameters, currentUser, listener); + } else { + // run the query + EventQueryResponseBase contentQueryResponse = runContentQuery(queryParameters, currentUser); + + // merge the response + if (contentQueryResponse != null) { + if (mergedResponse == null) { + mergedResponse = contentQueryResponse; + } else { + mergedResponse.merge(contentQueryResponse); + } } } - } - return mergedResponse; + + // noinspection unchecked + return (T) mergedResponse; } private List createContentQueries(Set contentLookupTerms) { diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/StreamingService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/StreamingService.java index edc708de..fa2b0b9f 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/StreamingService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/StreamingService.java @@ -47,6 +47,8 @@ public StreamingService(QueryManagementService queryManagementService, QueryMetr * the query parameters, not null * @param currentUser * the user who called this method, not null + * @param listener + * the listener which will handle the result pages, not null * @return the query id * @throws BadRequestQueryException * if parameter validation fails diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java index c731ae62..71aa0799 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java @@ -265,39 +265,8 @@ public void testCreateAndExecuteSuccess() throws Throwable { queryId, queryRequestEvents.removeLast()); // @formatter:on - } - @Test - public void testBatchLookupUUIDSuccess() throws Exception {} - - @Test - public void testLookupContentUUIDSuccess() throws Exception {} - - @Test - public void testBatchLookupContentUUIDSuccess() throws Exception {} - - @Test - public void testBatchLookupUUIDFailure_noLookupUUIDPairs() throws Exception {} - - @Test - public void testBatchLookupUUIDFailure_mixedQueryLogics() throws Exception {} - - @Test - public void testBatchLookupUUIDFailure_nullUUIDType() throws Exception {} - - @Test - public void testBatchLookupUUIDFailure_emptyUUIDFieldValue() throws Exception {} - - @Test - public void testBatchLookupUUIDFailure_invalidUUIDPair() throws Exception {} - - @Test - public void testBatchLookupUUIDFailure_tooManyTerms() throws Exception {} - - @Test - public void testBatchLookupUUIDFailure_nonLookupQueryLogic() throws Exception {} - protected Future> createAndExecute(ProxiedUserDetails authUser, MultiValueMap map) { UriComponents uri = createUri("EventQuery/createAndExecute"); From 6f174bf979395d23d9055d285c0081947fd77504 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 21 Jan 2022 14:54:43 -0500 Subject: [PATCH 122/218] Updated lookup to pass the right parameters when running a streaming query. --- .../microservice/query/config/QueryServiceConfig.java | 2 +- .../datawave/microservice/query/lookup/LookupService.java | 2 +- .../microservice/query/stream/StreamingService.java | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java index 3562bee2..f54141f2 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java @@ -64,7 +64,7 @@ public ThreadPoolTaskExecutor nextCallExecutor(QueryProperties queryProperties) @RefreshScope @Bean - public ThreadPoolTaskExecutor streamingExecutor(StreamingProperties streamingProperties) { + public ThreadPoolTaskExecutor streamingCallExecutor(StreamingProperties streamingProperties) { ThreadPoolTaskExecutorProperties executorProperties = streamingProperties.getExecutor(); ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); executor.setCorePoolSize(executorProperties.getCorePoolSize()); diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/lookup/LookupService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/lookup/LookupService.java index 0eb2d641..56af7cd1 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/lookup/LookupService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/lookup/LookupService.java @@ -516,7 +516,7 @@ private T lookupContent(Set contentLookupTerms, MultiValueMap Date: Tue, 25 Jan 2022 16:50:41 -0500 Subject: [PATCH 123/218] Updated the query management service to periodically check for executor service failures when sending a CREATE, PLAN, or PREDICT request. --- .../config/QueryExpirationProperties.java | 24 ++++++++++++ .../query/QueryManagementService.java | 37 +++++++++++++------ 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java index 753765a1..b8a4eb8d 100644 --- a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java +++ b/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java @@ -21,6 +21,10 @@ public class QueryExpirationProperties { @NotNull private TimeUnit callTimeUnit = TimeUnit.MINUTES; @Positive + private long callTimeoutInterval = 1; + @NotNull + private TimeUnit callTimeoutIntervalUnit = TimeUnit.MINUTES; + @Positive private long pageMinTimeout = 1; @NotNull private TimeUnit pageMinTimeUnit = TimeUnit.MINUTES; @@ -97,6 +101,26 @@ public void setCallTimeUnit(TimeUnit callTimeUnit) { this.callTimeUnit = callTimeUnit; } + public long getCallTimeoutInterval() { + return callTimeoutInterval; + } + + public long getCallTimeoutIntervalMillis() { + return callTimeoutIntervalUnit.toMillis(callTimeoutInterval); + } + + public void setCallTimeoutInterval(long callTimeoutInterval) { + this.callTimeoutInterval = callTimeoutInterval; + } + + public TimeUnit getCallTimeoutIntervalUnit() { + return callTimeoutIntervalUnit; + } + + public void setCallTimeoutIntervalUnit(TimeUnit callTimeoutIntervalUnit) { + this.callTimeoutIntervalUnit = callTimeoutIntervalUnit; + } + public long getPageMinTimeout() { return pageMinTimeout; } diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 7e1a5828..4c1634b9 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -80,6 +80,7 @@ import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CLOSED; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CREATED; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.DEFINED; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.FAILED; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.PLANNED; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.PREDICTED; import static datawave.webservice.query.QueryImpl.DN_LIST; @@ -592,23 +593,37 @@ private void sendRequestAwaitResponse(QueryRequest request, boolean isAwaitRespo queryLatchMap.put(request.getQueryId(), new CountDownLatch(1)); } - // publish a plan event to the executor pool + // publish an event to the executor pool publishExecutorEvent(request, getPoolName()); if (isAwaitResponse) { + // TODO: Should we incorporate the call start time into this check? + long startTimeMillis = System.currentTimeMillis(); + log.info("Waiting on query {} response from the executor.", request.getMethod().name()); - // wait for the executor to finish creating the query + try { - // TODO: Should we incorporate the call start time into this check? - if (!queryLatchMap.get(request.getQueryId()).await(queryProperties.getExpiration().getCallTimeout(), - queryProperties.getExpiration().getCallTimeUnit())) { - throw new QueryException(DatawaveErrorCode.QUERY_TIMEOUT, "Timed out waiting for query " + request.getMethod().name() + " to finish."); - } else { - log.info("Received query {} response from the executor.", request.getMethod().name()); + boolean isFinished = false; + while (!isFinished && System.currentTimeMillis() < (startTimeMillis + queryProperties.getExpiration().getCallTimeoutMillis())) { + try { + // wait for the executor response + if (queryLatchMap.get(request.getQueryId()).await(queryProperties.getExpiration().getCallTimeoutInterval(), + queryProperties.getExpiration().getCallTimeoutIntervalUnit())) { + log.info("Received query {} response from the executor.", request.getMethod().name()); + isFinished = true; + } + + // did the request fail? + QueryStatus queryStatus = queryStorageCache.getQueryStatus(request.getQueryId()); + if (queryStorageCache.getQueryStatus(request.getQueryId()).getQueryState() == FAILED) { + log.error("Query {} failed for queryId {}: {}", request.getMethod().name(), request.getQueryId(), queryStatus.getFailureMessage()); + throw new QueryException(queryStatus.getErrorCode(), "Query " + request.getMethod().name() + " failed for queryId " + + request.getQueryId() + ": " + queryStatus.getFailureMessage()); + } + } catch (InterruptedException e) { + log.warn("Interrupted while waiting for query {} latch for queryId {}", request.getMethod().name(), request.getQueryId()); + } } - } catch (InterruptedException e) { - log.warn("Interrupted while waiting for query {} latch for queryId {}", request.getMethod().name(), request.getQueryId()); - throw new QueryException(DatawaveErrorCode.QUERY_TIMEOUT, "Interrupted while waiting for query " + request.getMethod().name() + " to finish."); } finally { queryLatchMap.remove(request.getQueryId()); } From 51bfa412fa7a00a88042fda05076c82b015c6504 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 27 Jan 2022 12:00:51 -0500 Subject: [PATCH 124/218] Updated datawave microservice parent version to 1.9-SNAPSHOT. Updated accumulo-api to 1.3-SNAPSHOT. Added surefire test configuration to datawave services module. --- query-microservices/query-service/api/pom.xml | 2 +- query-microservices/query-service/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/query-microservices/query-service/api/pom.xml b/query-microservices/query-service/api/pom.xml index ba5b7216..9d6f7686 100644 --- a/query-microservices/query-service/api/pom.xml +++ b/query-microservices/query-service/api/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 1.8 + 1.9-SNAPSHOT query-api diff --git a/query-microservices/query-service/pom.xml b/query-microservices/query-service/pom.xml index 792b8307..088b40e1 100644 --- a/query-microservices/query-service/pom.xml +++ b/query-microservices/query-service/pom.xml @@ -4,7 +4,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent - 1.8 + 1.9-SNAPSHOT query-service-parent From 75b4955d47af1f4e52d92ac472f120a1e8ae3d0c Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 28 Jan 2022 12:07:39 -0500 Subject: [PATCH 125/218] Updates for spring boot 2.6.3 --- .../service/src/main/resources/config/bootstrap.yml | 5 +++++ .../service/src/test/resources/config/bootstrap.yml | 5 ++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/query-microservices/query-service/service/src/main/resources/config/bootstrap.yml b/query-microservices/query-service/service/src/main/resources/config/bootstrap.yml index d2e4730f..63351c57 100644 --- a/query-microservices/query-service/service/src/main/resources/config/bootstrap.yml +++ b/query-microservices/query-service/service/src/main/resources/config/bootstrap.yml @@ -15,6 +15,11 @@ spring: retry: max-attempts: 60 uri: '${CONFIG_SERVER_URL:http://configuration:8888/configserver}' + # Starting with spring-boot 2.6, circular references are disabled by default + # This is still needed for the evaluation-only function + main: + allow-circular-references: true + --- # For the dev profile, check localhost for the config server by default diff --git a/query-microservices/query-service/service/src/test/resources/config/bootstrap.yml b/query-microservices/query-service/service/src/test/resources/config/bootstrap.yml index 38f5a1eb..c8d89162 100644 --- a/query-microservices/query-service/service/src/test/resources/config/bootstrap.yml +++ b/query-microservices/query-service/service/src/test/resources/config/bootstrap.yml @@ -9,4 +9,7 @@ spring: discovery: enabled: false main: - banner-mode: "OFF" \ No newline at end of file + banner-mode: "OFF" + # Starting with spring-boot 2.6, circular references are disabled by default + # This is still needed for the evaluation-only function + allow-circular-references: true From 24de9975a7cd596f8f91df4e7885f0088b0fcc6c Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Mon, 31 Jan 2022 17:42:17 +0000 Subject: [PATCH 126/218] Run the query service tests one by one --- query-microservices/query-service/service/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/query-microservices/query-service/service/pom.xml b/query-microservices/query-service/service/pom.xml index a6099a52..8edbd518 100644 --- a/query-microservices/query-service/service/pom.xml +++ b/query-microservices/query-service/service/pom.xml @@ -12,6 +12,7 @@ DATAWAVE Query Microservice datawave.microservice.query.QueryService + 1 1.0-SNAPSHOT 1.3-SNAPSHOT 1.0-SNAPSHOT From 381f550bca088ee7cb87d680a96e37d20e101962 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 2 Feb 2022 10:28:56 -0500 Subject: [PATCH 127/218] Fixed some issues with hanging tests. Updated composite, and lookup query logics to be checkpointable. --- query-microservices/query-service/api/pom.xml | 1 + .../query/AbstractQueryServiceTest.java | 20 ++++++++++++++---- .../query/QueryServiceCancelTest.java | 18 +++------------- .../query/QueryServiceCloseTest.java | 12 ----------- .../query/QueryServiceDefineTest.java | 3 --- .../query/QueryServiceNextTest.java | 5 +++++ .../query/lookup/LookupServiceTest.java | 21 +++++++++++++++++-- .../application-QueryStarterOverrides.yml | 6 +++--- .../src/test/resources/config/application.yml | 4 ++++ 9 files changed, 51 insertions(+), 39 deletions(-) diff --git a/query-microservices/query-service/api/pom.xml b/query-microservices/query-service/api/pom.xml index 9d6f7686..3a27995a 100644 --- a/query-microservices/query-service/api/pom.xml +++ b/query-microservices/query-service/api/pom.xml @@ -73,6 +73,7 @@ org.apache.maven.plugins maven-jar-plugin + 2.4 jboss diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java index bb3c4f9b..8aa28ec8 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java @@ -1,12 +1,16 @@ package datawave.microservice.query; +import com.hazelcast.config.Config; +import com.hazelcast.core.Hazelcast; +import com.hazelcast.core.HazelcastInstance; import datawave.marking.ColumnVisibilitySecurityMarking; import datawave.microservice.audit.AuditClient; import datawave.microservice.authorization.jwt.JWTRestTemplate; import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.query.config.QueryProperties; +import datawave.microservice.query.messaging.QueryResultsManager; +import datawave.microservice.query.messaging.QueryResultsPublisher; import datawave.microservice.query.messaging.Result; -import datawave.microservice.query.messaging.TestQueryResultsManager; import datawave.microservice.query.remote.QueryRequest; import datawave.microservice.query.storage.QueryStatus; import datawave.microservice.query.storage.QueryStorageCache; @@ -66,6 +70,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.stream.Collectors; @@ -109,7 +114,7 @@ public abstract class AbstractQueryServiceTest { protected QueryStorageCache queryStorageCache; @Autowired - protected TestQueryResultsManager queryQueueManager; + protected QueryResultsManager queryQueueManager; @Autowired protected AuditClient auditClient; @@ -140,10 +145,10 @@ public void setup() { public void teardown() throws Exception { queryStorageCache.clear(); queryRequestEvents.clear(); - queryQueueManager.clear(); } protected void publishEventsToQueue(String queryId, int numEvents, MultiValueMap fieldValues, String visibility) throws Exception { + QueryResultsPublisher publisher = queryQueueManager.createPublisher(queryId); for (int resultId = 0; resultId < numEvents; resultId++) { DefaultEvent event = new DefaultEvent(); long currentTime = System.currentTimeMillis(); @@ -154,7 +159,7 @@ protected void publishEventsToQueue(String queryId, int numEvents, MultiValueMap } } event.setFields(fields); - queryQueueManager.createPublisher(queryId).publish(new Result(Integer.toString(resultId), event)); + publisher.publish(new Result(Integer.toString(resultId), event)); } } @@ -504,6 +509,13 @@ public void handleError(ClientHttpResponse response) throws IOException { @Profile("QueryServiceTest") @ComponentScan(basePackages = "datawave.microservice") public static class QueryServiceTestConfiguration { + @Bean + public HazelcastInstance hazelcastInstance() { + Config config = new Config(); + config.setClusterName(UUID.randomUUID().toString()); + return Hazelcast.newHazelcastInstance(config); + } + @Bean public LinkedList queryRequestEvents() { return new LinkedList<>(); diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java index 1efc7b9b..755d40aa 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java @@ -73,9 +73,6 @@ public void testCancelSuccess() throws Exception { queryStatus); // @formatter:on - // verify that the result queue is gone - Assert.assertFalse(queryQueueManager.queueExists(queryId)); - // verify that the query tasks are still present assertTasksCreated(queryId); @@ -131,13 +128,13 @@ public void testCancelSuccess_activeNextCall() throws Exception { Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); - // verify that query status was created correctly - QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - // wait for the next call to drop out before checking status // since we canceled, this should quit immediately, but just in case, we add a timeout nextFuture.get(10, TimeUnit.SECONDS); + // verify that query status was created correctly + QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); + // @formatter:off assertQueryStatus( QueryStatus.QUERY_STATE.CANCELED, @@ -149,9 +146,6 @@ public void testCancelSuccess_activeNextCall() throws Exception { queryStatus); // @formatter:on - // verify that the result queue is gone - Assert.assertFalse(queryQueueManager.queueExists(queryId)); - // wait for the next call to return nextFuture.get(); @@ -326,9 +320,6 @@ public void testAdminCancelSuccess() throws Exception { queryStatus); // @formatter:on - // verify that the result queue is gone - Assert.assertFalse(queryQueueManager.queueExists(queryId)); - // verify that the query tasks are still present assertTasksCreated(queryId); @@ -429,9 +420,6 @@ public void testAdminCancelAllSuccess() throws Exception { String queryId = queryStatus.getQueryKey().getQueryId(); - // verify that the result queue is gone - Assert.assertFalse(queryQueueManager.queueExists(queryStatus.getQueryKey().getQueryId())); - // verify that the query tasks are still present assertTasksCreated(queryStatus.getQueryKey().getQueryId()); diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java index b02fec65..2de43f3e 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java @@ -75,9 +75,6 @@ public void testCloseSuccess() throws Exception { queryStatus); // @formatter:on - // verify that the result queue is gone - Assert.assertFalse(queryQueueManager.queueExists(queryId)); - // verify that the query tasks are still present assertTasksCreated(queryId); @@ -162,9 +159,6 @@ public void testCloseSuccess_activeNextCall() throws Exception { // wait for the next call to return nextFuture.get(); - // verify that the result queue is now gone - Assert.assertFalse(queryQueueManager.queueExists(queryId)); - // verify that the query tasks are still present assertTasksCreated(queryId); @@ -326,9 +320,6 @@ public void testAdminCloseSuccess() throws Exception { queryStatus); // @formatter:on - // verify that the result queue is gone - Assert.assertFalse(queryQueueManager.queueExists(queryId)); - // verify that the query tasks are still present assertTasksCreated(queryId); @@ -422,9 +413,6 @@ public void testAdminCloseAllSuccess() throws Exception { String queryId = queryStatus.getQueryKey().getQueryId(); - // verify that the result queue is gone - Assert.assertFalse(queryQueueManager.queueExists(queryStatus.getQueryKey().getQueryId())); - // verify that the query tasks are still present assertTasksCreated(queryStatus.getQueryKey().getQueryId()); diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java index e0ac00a6..9ced2d4f 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java @@ -102,9 +102,6 @@ public void testDefineSuccess() throws ParseException, IOException { // verify that no audit message was sent assertAuditNotSent(); - // verify that the results queue wasn't created - Assert.assertFalse(queryQueueManager.queueExists(queryId)); - // verify that query tasks weren't created assertTasksNotCreated(queryId); } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java index 71214b6f..afe9844c 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java @@ -16,6 +16,7 @@ import org.junit.runner.RunWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.ResponseEntity; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.util.LinkedMultiValueMap; @@ -589,10 +590,14 @@ public void testNextFailure_ownershipFailure() throws Exception { // @formatter:on } + @DirtiesContext @Test public void testNextFailure_timeout() throws Exception { ProxiedUserDetails authUser = createUserDetails(); + // override the call timeout for this test + queryProperties.getExpiration().setCallTimeout(0); + // create a valid query String queryId = createQuery(authUser, createParams()); diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java index 3aa72408..59c4b910 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java @@ -6,6 +6,7 @@ import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.query.AbstractQueryServiceTest; import datawave.microservice.query.DefaultQueryParameters; +import datawave.microservice.query.messaging.QueryResultsPublisher; import datawave.microservice.query.messaging.Result; import datawave.microservice.query.remote.QueryRequest; import datawave.microservice.query.storage.QueryStatus; @@ -35,6 +36,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -45,6 +47,7 @@ import java.util.stream.Collectors; import static datawave.microservice.query.QueryParameters.QUERY_MAX_CONCURRENT_TASKS; +import static datawave.microservice.query.QueryParameters.QUERY_MAX_RESULTS_OVERRIDE; import static datawave.microservice.query.lookup.LookupService.LOOKUP_UUID_PAIRS; @RunWith(SpringRunner.class) @@ -294,6 +297,7 @@ public void testLookupContentUUIDSuccess() throws Exception { contentQueryIds = queryStatuses.stream().map(QueryStatus::getQueryKey).map(QueryKey::getQueryId) .filter(contentQueryId -> !contentQueryId.equals(eventQueryId)).collect(Collectors.toSet()); } + Thread.sleep(500); } Assert.assertNotNull(contentQueryIds); @@ -394,6 +398,7 @@ public void testBatchLookupContentUUIDSuccess() throws Exception { MultiValueMap uuidParams = createUUIDParams(); uuidParams.add(LOOKUP_UUID_PAIRS, "PAGE_TITLE:anarchy"); uuidParams.add(LOOKUP_UUID_PAIRS, "PAGE_TITLE:accessiblecomputing"); + uuidParams.add(QUERY_MAX_RESULTS_OVERRIDE, "10"); Future> future = batchLookupContentUUID(authUser, uuidParams); @@ -434,6 +439,7 @@ public void testBatchLookupContentUUIDSuccess() throws Exception { contentQueryIds = queryStatuses.stream().map(QueryStatus::getQueryKey).map(QueryKey::getQueryId) .filter(contentQueryId -> !contentQueryId.equals(eventQueryId)).collect(Collectors.toSet()); } + Thread.sleep(500); } Assert.assertNotNull(contentQueryIds); @@ -450,6 +456,16 @@ public void testBatchLookupContentUUIDSuccess() throws Exception { // @formatter:on } + // wait until each query has read its results, and then close it + for (String contentQueryId : contentQueryIds) { + QueryStatus status = queryStorageCache.getQueryStatus(contentQueryId); + startTime = System.currentTimeMillis(); + while ((System.currentTimeMillis() - startTime) < 5000 && status.getNumResultsConsumed() < pageSize) { + Thread.sleep(500); + status = queryStorageCache.getQueryStatus(contentQueryId); + } + } + ResponseEntity response = future.get(); Assert.assertEquals(200, response.getStatusCodeValue()); @@ -771,13 +787,14 @@ protected Future> lookupContentUUID(ProxiedUse } protected void publishEventsToQueue(String queryId, int numEvents, MultiValueMap fieldValues, String visibility) throws Exception { + QueryResultsPublisher publisher = queryQueueManager.createPublisher(queryId); for (int resultId = 0; resultId < numEvents; resultId++) { DefaultEvent event = new DefaultEvent(); long currentTime = System.currentTimeMillis(); List fields = new ArrayList<>(); for (Map.Entry> entry : fieldValues.entrySet()) { for (String value : entry.getValue()) { - fields.add(new DefaultField(entry.getKey(), visibility, currentTime, value)); + fields.add(new DefaultField(entry.getKey(), visibility, new HashMap<>(), currentTime, value)); } } event.setFields(fields); @@ -788,7 +805,7 @@ protected void publishEventsToQueue(String queryId, int numEvents, MultiValueMap metadata.setDataType("prince"); metadata.setInternalId(UUID.randomUUID().toString()); event.setMetadata(metadata); - queryQueueManager.createPublisher(queryId).publish(new Result(Integer.toString(resultId), event)); + publisher.publish(new Result(Integer.toString(resultId), event)); } } diff --git a/query-microservices/query-service/service/src/test/resources/config/application-QueryStarterOverrides.yml b/query-microservices/query-service/service/src/test/resources/config/application-QueryStarterOverrides.yml index c4d7e5e9..4cfa1f62 100644 --- a/query-microservices/query-service/service/src/test/resources/config/application-QueryStarterOverrides.yml +++ b/query-microservices/query-service/service/src/test/resources/config/application-QueryStarterOverrides.yml @@ -7,9 +7,6 @@ datawave: nextCall: resultPollInterval: 500 statusUpdateInterval: 500 - expiration: - callTimeout: 5 - callTimeUnit: SECONDS logic: factory: # Uncomment the following line to override the query logic beans to load @@ -18,6 +15,9 @@ datawave: BaseEventQuery: maxResults: 369 auditType: "ACTIVE" + ContentQuery: + maxResults: 10 + lookup: types: 'EVENT_ID': diff --git a/query-microservices/query-service/service/src/test/resources/config/application.yml b/query-microservices/query-service/service/src/test/resources/config/application.yml index b2dd5d8c..cdcf6b68 100644 --- a/query-microservices/query-service/service/src/test/resources/config/application.yml +++ b/query-microservices/query-service/service/src/test/resources/config/application.yml @@ -41,6 +41,10 @@ management: web: base-path: "/mgmt" +query: + messaging: + backend: hazelcast + logging: level: root: FATAL From dbf4c683e93538834a6f154549f3c56237b6c270 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 2 Feb 2022 10:31:07 -0500 Subject: [PATCH 128/218] Allow query service tests to run in parallel. --- query-microservices/query-service/service/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/query-microservices/query-service/service/pom.xml b/query-microservices/query-service/service/pom.xml index 8edbd518..a6099a52 100644 --- a/query-microservices/query-service/service/pom.xml +++ b/query-microservices/query-service/service/pom.xml @@ -12,7 +12,6 @@ DATAWAVE Query Microservice datawave.microservice.query.QueryService - 1 1.0-SNAPSHOT 1.3-SNAPSHOT 1.0-SNAPSHOT From b7330e629f2dc56e01f6ef3c5cb7dbfc81d88cd0 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 4 Feb 2022 16:11:42 -0500 Subject: [PATCH 129/218] Added remaining query logics to QueryLogicFactory.xml - many of which still need to be tweaked/tested. --- .../datawave/microservice/query/QueryServiceListTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java index f517d063..104484bc 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java @@ -419,7 +419,9 @@ public void testListQueryLogicSuccess() throws Exception { QueryLogicResponse qlResponse = response.getBody(); - String[] expectedQueryLogics = new String[] {"AltEventQuery", "EventQuery", "LuceneUUIDEventQuery", "DiscoveryQuery", "ContentQuery", "EdgeQuery"}; + String[] expectedQueryLogics = new String[] {"AltEventQuery", "ContentQuery", "CountQuery", "DiscoveryQuery", "EdgeEventQuery", "EdgeQuery", + "ErrorCountQuery", "ErrorDiscoveryQuery", "ErrorEventQuery", "ErrorFieldIndexCountQuery", "EventQuery", "FacetedQuery", "FieldIndexCountQuery", + "HitHighlights", "IndexStatsQuery", "LuceneUUIDEventQuery", "QueryMetricsQuery", "TermFrequencyQuery"}; Assert.assertEquals(expectedQueryLogics.length, qlResponse.getQueryLogicList().size()); From aa1a88c257fdef54fb7c38102cc7818173ca551c Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Wed, 9 Feb 2022 21:19:34 +0000 Subject: [PATCH 130/218] Added a RUNNING query status and changed checks appropriately --- .../query/QueryManagementService.java | 35 ++++++++++--------- .../query/QueryServiceCancelTest.java | 2 +- .../query/QueryServiceCloseTest.java | 2 +- .../query/QueryServiceCreateTest.java | 2 +- .../query/QueryServiceDuplicateTest.java | 12 +++---- .../query/QueryServiceResetTest.java | 8 ++--- .../query/lookup/LookupServiceTest.java | 2 +- 7 files changed, 32 insertions(+), 31 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 4c1634b9..a9567068 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -78,11 +78,12 @@ import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CANCELED; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CLOSED; -import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CREATED; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CREATE; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.RUNNING; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.DEFINED; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.FAILED; -import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.PLANNED; -import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.PREDICTED; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.PLAN; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.PREDICT; import static datawave.webservice.query.QueryImpl.DN_LIST; import static datawave.webservice.query.QueryImpl.QUERY_ID; import static datawave.webservice.query.QueryImpl.USER_DN; @@ -328,7 +329,7 @@ public GenericResponse create(String queryLogicName, MultiValueMap response = new GenericResponse<>(); response.setResult(taskKey.getQueryId()); response.setHasResults(true); @@ -384,7 +385,7 @@ public GenericResponse plan(String queryLogicName, MultiValueMap response = new GenericResponse<>(); @@ -443,7 +444,7 @@ public GenericResponse predict(String queryLogicName, MultiValueMap response = new GenericResponse<>(); @@ -521,7 +522,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p } // if this is a create request, or a plan request where we are expanding values, send an audit record to the auditor - if (queryType == CREATED || (queryType == PLANNED && queryParameters.isExpandValues())) { + if (queryType == CREATE || (queryType == PLAN && queryParameters.isExpandValues())) { audit(query, queryLogic, parameters, currentUser); } @@ -536,7 +537,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p downgradedAuthorizations, getMaxConcurrentTasks(queryLogic)); // @formatter:on - } else if (queryType == CREATED) { + } else if (queryType == CREATE) { // @formatter:off taskKey = queryStorageCache.createQuery( getPoolName(), @@ -546,7 +547,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p // @formatter:on sendRequestAwaitResponse(QueryRequest.create(taskKey.getQueryId()), queryProperties.isAwaitExecutorCreateResponse()); - } else if (queryType == PLANNED) { + } else if (queryType == PLAN) { // @formatter:off taskKey = queryStorageCache.planQuery( getPoolName(), @@ -555,7 +556,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p // @formatter:on sendRequestAwaitResponse(QueryRequest.plan(taskKey.getQueryId()), true); - } else if (queryType == PREDICTED) { + } else if (queryType == PREDICT) { // @formatter:off taskKey = queryStorageCache.predictQuery( getPoolName(), @@ -573,7 +574,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p // update the query metric BaseQueryMetric baseQueryMetric = getBaseQueryMetric(); - if (queryType == DEFINED || queryType == CREATED) { + if (queryType == DEFINED || queryType == CREATE) { baseQueryMetric.setQueryId(taskKey.getQueryId()); baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.DEFINED); baseQueryMetric.populate(query); @@ -744,8 +745,8 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th // make sure the query is valid, and the user can act on it QueryStatus queryStatus = validateRequest(queryId, currentUser); - // make sure the state is created - if (queryStatus.getQueryState() == CREATED) { + // make sure the state is running or in the process of being created + if (queryStatus.getQueryState() == RUNNING || queryStatus.getQueryState() == CREATE) { return next(queryId, currentUser.getPrimaryUser().getRoles()); } else { throw new BadRequestQueryException("Cannot call next on a query that is not running", HttpStatus.SC_BAD_REQUEST + "-1"); @@ -971,7 +972,7 @@ public VoidResponse adminCancelAll(ProxiedUserDetails currentUser) throws QueryE try { List queryStatuses = queryStorageCache.getQueryStatus(); - queryStatuses.removeIf(s -> s.getQueryState() != CREATED); + queryStatuses.removeIf(s -> (s.getQueryState() != CREATE && s.getQueryState() != RUNNING)); VoidResponse response = new VoidResponse(); for (QueryStatus queryStatus : queryStatuses) { @@ -1188,7 +1189,7 @@ public VoidResponse adminCloseAll(ProxiedUserDetails currentUser) throws QueryEx try { List queryStatuses = queryStorageCache.getQueryStatus(); - queryStatuses.removeIf(s -> s.getQueryState() != CREATED); + queryStatuses.removeIf(s -> (s.getQueryState() != CREATE && s.getQueryState() != RUNNING)); VoidResponse response = new VoidResponse(); for (QueryStatus queryStatus : queryStatuses) { @@ -1237,7 +1238,7 @@ private VoidResponse close(String queryId, ProxiedUserDetails currentUser, boole // make sure the query is valid, and the user can act on it QueryStatus queryStatus = validateRequest(queryId, currentUser, adminOverride); - if (queryStatus.getQueryState() == CREATED) { + if (queryStatus.getQueryState() == CREATE || queryStatus.getQueryState() == RUNNING) { close(queryId); } else { throw new BadRequestQueryException("Cannot call close on a query that is not running", HttpStatus.SC_BAD_REQUEST + "-1"); @@ -1744,7 +1745,7 @@ private TaskKey duplicate(QueryStatus queryStatus, MultiValueMap updateParameters(parameters, currentParams); // define a duplicate query - return storeQuery(currentParams.getFirst(QUERY_LOGIC_NAME), currentParams, currentUser, CREATED); + return storeQuery(currentParams.getFirst(QUERY_LOGIC_NAME), currentParams, currentUser, CREATE); } private boolean updateParameters(MultiValueMap newParameters, MultiValueMap currentParams) throws BadRequestQueryException { diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java index 755d40aa..c9d20d69 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java @@ -482,7 +482,7 @@ public void testAdminCancelAllFailure_notAdminUser() throws Exception { for (QueryStatus queryStatus : queryStatusList) { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java index 2de43f3e..6bcf5634 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java @@ -470,7 +470,7 @@ public void testAdminCloseAllFailure_notAdminUser() throws Exception { for (QueryStatus queryStatus : queryStatusList) { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java index cce2ef89..0a477172 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java @@ -88,7 +88,7 @@ public void testCreateSuccess() throws ParseException, IOException { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java index 0c55606b..c2f9aa95 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java @@ -88,7 +88,7 @@ public void testDuplicateSuccess_duplicateOnDefined() throws Exception { QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, @@ -147,7 +147,7 @@ public void testDuplicateSuccess_duplicateOnCreated() throws Exception { QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, @@ -159,7 +159,7 @@ public void testDuplicateSuccess_duplicateOnCreated() throws Exception { QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, @@ -243,7 +243,7 @@ public void testDuplicateSuccess_duplicateOnCanceled() throws Exception { QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, @@ -337,7 +337,7 @@ public void testDuplicateSuccess_duplicateOnClosed() throws Exception { QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, @@ -429,7 +429,7 @@ public void testDuplicateSuccess_update() throws Exception { QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java index 8387feea..58d67481 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java @@ -93,7 +93,7 @@ public void testResetSuccess_resetOnDefined() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, @@ -170,7 +170,7 @@ public void testResetSuccess_resetOnCreated() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, @@ -270,7 +270,7 @@ public void testResetSuccess_resetOnClosed() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, @@ -365,7 +365,7 @@ public void testResetSuccess_resetOnCanceled() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java index 59c4b910..ead16c47 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java @@ -82,7 +82,7 @@ public void testLookupUUIDSuccess() throws Exception { // get the lookup query id long startTime = System.currentTimeMillis(); - while ((System.currentTimeMillis() - startTime) < 5000 && queryId == null) { + while ((System.currentTimeMillis() - startTime) < Long.MAX_VALUE && queryId == null) { List queryStatuses = queryStorageCache.getQueryStatus(); if (queryStatuses.size() > 0) { queryId = queryStatuses.get(0).getQueryKey().getQueryId(); From 6494b7161b835fc7eaaa863374ea8610a19a4932 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Wed, 9 Feb 2022 22:26:29 +0000 Subject: [PATCH 131/218] Revert "Added a RUNNING query status and changed checks appropriately" This reverts commit c74256943c3d8fc2883a2e8f68b77fbfd04caac7. --- .../query/QueryManagementService.java | 35 +++++++++---------- .../query/QueryServiceCancelTest.java | 2 +- .../query/QueryServiceCloseTest.java | 2 +- .../query/QueryServiceCreateTest.java | 2 +- .../query/QueryServiceDuplicateTest.java | 12 +++---- .../query/QueryServiceResetTest.java | 8 ++--- .../query/lookup/LookupServiceTest.java | 2 +- 7 files changed, 31 insertions(+), 32 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index a9567068..4c1634b9 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -78,12 +78,11 @@ import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CANCELED; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CLOSED; -import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CREATE; -import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.RUNNING; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CREATED; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.DEFINED; import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.FAILED; -import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.PLAN; -import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.PREDICT; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.PLANNED; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.PREDICTED; import static datawave.webservice.query.QueryImpl.DN_LIST; import static datawave.webservice.query.QueryImpl.QUERY_ID; import static datawave.webservice.query.QueryImpl.USER_DN; @@ -329,7 +328,7 @@ public GenericResponse create(String queryLogicName, MultiValueMap response = new GenericResponse<>(); response.setResult(taskKey.getQueryId()); response.setHasResults(true); @@ -385,7 +384,7 @@ public GenericResponse plan(String queryLogicName, MultiValueMap response = new GenericResponse<>(); @@ -444,7 +443,7 @@ public GenericResponse predict(String queryLogicName, MultiValueMap response = new GenericResponse<>(); @@ -522,7 +521,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p } // if this is a create request, or a plan request where we are expanding values, send an audit record to the auditor - if (queryType == CREATE || (queryType == PLAN && queryParameters.isExpandValues())) { + if (queryType == CREATED || (queryType == PLANNED && queryParameters.isExpandValues())) { audit(query, queryLogic, parameters, currentUser); } @@ -537,7 +536,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p downgradedAuthorizations, getMaxConcurrentTasks(queryLogic)); // @formatter:on - } else if (queryType == CREATE) { + } else if (queryType == CREATED) { // @formatter:off taskKey = queryStorageCache.createQuery( getPoolName(), @@ -547,7 +546,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p // @formatter:on sendRequestAwaitResponse(QueryRequest.create(taskKey.getQueryId()), queryProperties.isAwaitExecutorCreateResponse()); - } else if (queryType == PLAN) { + } else if (queryType == PLANNED) { // @formatter:off taskKey = queryStorageCache.planQuery( getPoolName(), @@ -556,7 +555,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p // @formatter:on sendRequestAwaitResponse(QueryRequest.plan(taskKey.getQueryId()), true); - } else if (queryType == PREDICT) { + } else if (queryType == PREDICTED) { // @formatter:off taskKey = queryStorageCache.predictQuery( getPoolName(), @@ -574,7 +573,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p // update the query metric BaseQueryMetric baseQueryMetric = getBaseQueryMetric(); - if (queryType == DEFINED || queryType == CREATE) { + if (queryType == DEFINED || queryType == CREATED) { baseQueryMetric.setQueryId(taskKey.getQueryId()); baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.DEFINED); baseQueryMetric.populate(query); @@ -745,8 +744,8 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th // make sure the query is valid, and the user can act on it QueryStatus queryStatus = validateRequest(queryId, currentUser); - // make sure the state is running or in the process of being created - if (queryStatus.getQueryState() == RUNNING || queryStatus.getQueryState() == CREATE) { + // make sure the state is created + if (queryStatus.getQueryState() == CREATED) { return next(queryId, currentUser.getPrimaryUser().getRoles()); } else { throw new BadRequestQueryException("Cannot call next on a query that is not running", HttpStatus.SC_BAD_REQUEST + "-1"); @@ -972,7 +971,7 @@ public VoidResponse adminCancelAll(ProxiedUserDetails currentUser) throws QueryE try { List queryStatuses = queryStorageCache.getQueryStatus(); - queryStatuses.removeIf(s -> (s.getQueryState() != CREATE && s.getQueryState() != RUNNING)); + queryStatuses.removeIf(s -> s.getQueryState() != CREATED); VoidResponse response = new VoidResponse(); for (QueryStatus queryStatus : queryStatuses) { @@ -1189,7 +1188,7 @@ public VoidResponse adminCloseAll(ProxiedUserDetails currentUser) throws QueryEx try { List queryStatuses = queryStorageCache.getQueryStatus(); - queryStatuses.removeIf(s -> (s.getQueryState() != CREATE && s.getQueryState() != RUNNING)); + queryStatuses.removeIf(s -> s.getQueryState() != CREATED); VoidResponse response = new VoidResponse(); for (QueryStatus queryStatus : queryStatuses) { @@ -1238,7 +1237,7 @@ private VoidResponse close(String queryId, ProxiedUserDetails currentUser, boole // make sure the query is valid, and the user can act on it QueryStatus queryStatus = validateRequest(queryId, currentUser, adminOverride); - if (queryStatus.getQueryState() == CREATE || queryStatus.getQueryState() == RUNNING) { + if (queryStatus.getQueryState() == CREATED) { close(queryId); } else { throw new BadRequestQueryException("Cannot call close on a query that is not running", HttpStatus.SC_BAD_REQUEST + "-1"); @@ -1745,7 +1744,7 @@ private TaskKey duplicate(QueryStatus queryStatus, MultiValueMap updateParameters(parameters, currentParams); // define a duplicate query - return storeQuery(currentParams.getFirst(QUERY_LOGIC_NAME), currentParams, currentUser, CREATE); + return storeQuery(currentParams.getFirst(QUERY_LOGIC_NAME), currentParams, currentUser, CREATED); } private boolean updateParameters(MultiValueMap newParameters, MultiValueMap currentParams) throws BadRequestQueryException { diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java index c9d20d69..755d40aa 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java @@ -482,7 +482,7 @@ public void testAdminCancelAllFailure_notAdminUser() throws Exception { for (QueryStatus queryStatus : queryStatusList) { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATE, + QueryStatus.QUERY_STATE.CREATED, 0, 0, 0, diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java index 6bcf5634..2de43f3e 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java @@ -470,7 +470,7 @@ public void testAdminCloseAllFailure_notAdminUser() throws Exception { for (QueryStatus queryStatus : queryStatusList) { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATE, + QueryStatus.QUERY_STATE.CREATED, 0, 0, 0, diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java index 0a477172..cce2ef89 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java @@ -88,7 +88,7 @@ public void testCreateSuccess() throws ParseException, IOException { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATE, + QueryStatus.QUERY_STATE.CREATED, 0, 0, 0, diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java index c2f9aa95..0c55606b 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java @@ -88,7 +88,7 @@ public void testDuplicateSuccess_duplicateOnDefined() throws Exception { QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATE, + QueryStatus.QUERY_STATE.CREATED, 0, 0, 0, @@ -147,7 +147,7 @@ public void testDuplicateSuccess_duplicateOnCreated() throws Exception { QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATE, + QueryStatus.QUERY_STATE.CREATED, 0, 0, 0, @@ -159,7 +159,7 @@ public void testDuplicateSuccess_duplicateOnCreated() throws Exception { QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATE, + QueryStatus.QUERY_STATE.CREATED, 0, 0, 0, @@ -243,7 +243,7 @@ public void testDuplicateSuccess_duplicateOnCanceled() throws Exception { QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATE, + QueryStatus.QUERY_STATE.CREATED, 0, 0, 0, @@ -337,7 +337,7 @@ public void testDuplicateSuccess_duplicateOnClosed() throws Exception { QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATE, + QueryStatus.QUERY_STATE.CREATED, 0, 0, 0, @@ -429,7 +429,7 @@ public void testDuplicateSuccess_update() throws Exception { QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATE, + QueryStatus.QUERY_STATE.CREATED, 0, 0, 0, diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java index 58d67481..8387feea 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java @@ -93,7 +93,7 @@ public void testResetSuccess_resetOnDefined() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATE, + QueryStatus.QUERY_STATE.CREATED, 0, 0, 0, @@ -170,7 +170,7 @@ public void testResetSuccess_resetOnCreated() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATE, + QueryStatus.QUERY_STATE.CREATED, 0, 0, 0, @@ -270,7 +270,7 @@ public void testResetSuccess_resetOnClosed() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATE, + QueryStatus.QUERY_STATE.CREATED, 0, 0, 0, @@ -365,7 +365,7 @@ public void testResetSuccess_resetOnCanceled() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATE, + QueryStatus.QUERY_STATE.CREATED, 0, 0, 0, diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java index ead16c47..59c4b910 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java @@ -82,7 +82,7 @@ public void testLookupUUIDSuccess() throws Exception { // get the lookup query id long startTime = System.currentTimeMillis(); - while ((System.currentTimeMillis() - startTime) < Long.MAX_VALUE && queryId == null) { + while ((System.currentTimeMillis() - startTime) < 5000 && queryId == null) { List queryStatuses = queryStorageCache.getQueryStatus(); if (queryStatuses.size() > 0) { queryId = queryStatuses.get(0).getQueryKey().getQueryId(); From 72f1ca5b1bef6b6d3db981eff8caa13a9bbcadbd Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Thu, 10 Feb 2022 19:35:54 +0000 Subject: [PATCH 132/218] Changed query status constants --- .../query/QueryManagementService.java | 58 +++++++++---------- .../microservice/query/runner/NextCall.java | 4 +- .../query/QueryServiceCancelTest.java | 10 ++-- .../query/QueryServiceCloseTest.java | 10 ++-- .../query/QueryServiceCreateTest.java | 2 +- .../query/QueryServiceDefineTest.java | 2 +- .../query/QueryServiceDuplicateTest.java | 20 +++---- .../query/QueryServiceResetTest.java | 16 ++--- 8 files changed, 61 insertions(+), 61 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 4c1634b9..95d16fb4 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -76,13 +76,13 @@ import java.util.stream.Collectors; import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; -import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CANCELED; -import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CLOSED; -import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CREATED; -import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.DEFINED; -import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.FAILED; -import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.PLANNED; -import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.PREDICTED; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CANCEL; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CLOSE; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.CREATE; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.DEFINE; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.FAIL; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.PLAN; +import static datawave.microservice.query.storage.QueryStatus.QUERY_STATE.PREDICT; import static datawave.webservice.query.QueryImpl.DN_LIST; import static datawave.webservice.query.QueryImpl.QUERY_ID; import static datawave.webservice.query.QueryImpl.USER_DN; @@ -272,7 +272,7 @@ public GenericResponse define(String queryLogicName, MultiValueMap response = new GenericResponse<>(); response.setResult(taskKey.getQueryId()); return response; @@ -328,7 +328,7 @@ public GenericResponse create(String queryLogicName, MultiValueMap response = new GenericResponse<>(); response.setResult(taskKey.getQueryId()); response.setHasResults(true); @@ -384,7 +384,7 @@ public GenericResponse plan(String queryLogicName, MultiValueMap response = new GenericResponse<>(); @@ -443,7 +443,7 @@ public GenericResponse predict(String queryLogicName, MultiValueMap response = new GenericResponse<>(); @@ -521,14 +521,14 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p } // if this is a create request, or a plan request where we are expanding values, send an audit record to the auditor - if (queryType == CREATED || (queryType == PLANNED && queryParameters.isExpandValues())) { + if (queryType == CREATE || (queryType == PLAN && queryParameters.isExpandValues())) { audit(query, queryLogic, parameters, currentUser); } try { // persist the query w/ query id in the query storage cache TaskKey taskKey = null; - if (queryType == DEFINED) { + if (queryType == DEFINE) { // @formatter:off taskKey = queryStorageCache.defineQuery( getPoolName(), @@ -536,7 +536,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p downgradedAuthorizations, getMaxConcurrentTasks(queryLogic)); // @formatter:on - } else if (queryType == CREATED) { + } else if (queryType == CREATE) { // @formatter:off taskKey = queryStorageCache.createQuery( getPoolName(), @@ -546,7 +546,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p // @formatter:on sendRequestAwaitResponse(QueryRequest.create(taskKey.getQueryId()), queryProperties.isAwaitExecutorCreateResponse()); - } else if (queryType == PLANNED) { + } else if (queryType == PLAN) { // @formatter:off taskKey = queryStorageCache.planQuery( getPoolName(), @@ -555,7 +555,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p // @formatter:on sendRequestAwaitResponse(QueryRequest.plan(taskKey.getQueryId()), true); - } else if (queryType == PREDICTED) { + } else if (queryType == PREDICT) { // @formatter:off taskKey = queryStorageCache.predictQuery( getPoolName(), @@ -573,7 +573,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p // update the query metric BaseQueryMetric baseQueryMetric = getBaseQueryMetric(); - if (queryType == DEFINED || queryType == CREATED) { + if (queryType == DEFINE || queryType == CREATE) { baseQueryMetric.setQueryId(taskKey.getQueryId()); baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.DEFINED); baseQueryMetric.populate(query); @@ -615,7 +615,7 @@ private void sendRequestAwaitResponse(QueryRequest request, boolean isAwaitRespo // did the request fail? QueryStatus queryStatus = queryStorageCache.getQueryStatus(request.getQueryId()); - if (queryStorageCache.getQueryStatus(request.getQueryId()).getQueryState() == FAILED) { + if (queryStorageCache.getQueryStatus(request.getQueryId()).getQueryState() == FAIL) { log.error("Query {} failed for queryId {}: {}", request.getMethod().name(), request.getQueryId(), queryStatus.getFailureMessage()); throw new QueryException(queryStatus.getErrorCode(), "Query " + request.getMethod().name() + " failed for queryId " + request.getQueryId() + ": " + queryStatus.getFailureMessage()); @@ -745,7 +745,7 @@ public BaseQueryResponse next(String queryId, ProxiedUserDetails currentUser) th QueryStatus queryStatus = validateRequest(queryId, currentUser); // make sure the state is created - if (queryStatus.getQueryState() == CREATED) { + if (queryStatus.getQueryState() == CREATE) { return next(queryId, currentUser.getPrimaryUser().getRoles()); } else { throw new BadRequestQueryException("Cannot call next on a query that is not running", HttpStatus.SC_BAD_REQUEST + "-1"); @@ -971,7 +971,7 @@ public VoidResponse adminCancelAll(ProxiedUserDetails currentUser) throws QueryE try { List queryStatuses = queryStorageCache.getQueryStatus(); - queryStatuses.removeIf(s -> s.getQueryState() != CREATED); + queryStatuses.removeIf(s -> s.getQueryState() != CREATE); VoidResponse response = new VoidResponse(); for (QueryStatus queryStatus : queryStatuses) { @@ -1045,7 +1045,7 @@ private VoidResponse cancel(String queryId, ProxiedUserDetails currentUser, bool *

* Cancels any locally-running next calls.
* Called with {@code publishEvent} set to true when the user calls cancel.
- * When {@code publishEvent} is true, changes the query state to {@link QueryStatus.QUERY_STATE#CANCELED}.
+ * When {@code publishEvent} is true, changes the query state to {@link QueryStatus.QUERY_STATE#CANCEL}.
* Called with {@code publishEvent} set to false when handling a remote cancel event. * * @param queryId @@ -1070,7 +1070,7 @@ public void cancel(String queryId, boolean publishEvent) throws InterruptedExcep // only the initial event publisher should update the status QueryStatus queryStatus = queryStatusUpdateUtil.lockedUpdate(queryId, status -> { // update query state to CANCELED - status.setQueryState(CANCELED); + status.setQueryState(CANCEL); }); // delete the results queue @@ -1188,7 +1188,7 @@ public VoidResponse adminCloseAll(ProxiedUserDetails currentUser) throws QueryEx try { List queryStatuses = queryStorageCache.getQueryStatus(); - queryStatuses.removeIf(s -> s.getQueryState() != CREATED); + queryStatuses.removeIf(s -> s.getQueryState() != CREATE); VoidResponse response = new VoidResponse(); for (QueryStatus queryStatus : queryStatuses) { @@ -1237,7 +1237,7 @@ private VoidResponse close(String queryId, ProxiedUserDetails currentUser, boole // make sure the query is valid, and the user can act on it QueryStatus queryStatus = validateRequest(queryId, currentUser, adminOverride); - if (queryStatus.getQueryState() == CREATED) { + if (queryStatus.getQueryState() == CREATE) { close(queryId); } else { throw new BadRequestQueryException("Cannot call close on a query that is not running", HttpStatus.SC_BAD_REQUEST + "-1"); @@ -1258,7 +1258,7 @@ private VoidResponse close(String queryId, ProxiedUserDetails currentUser, boole /** * Closes the specified query, and publishes a close event to the executor services. *

- * Changes the query state to {@link QueryStatus.QUERY_STATE#CLOSED}. + * Changes the query state to {@link QueryStatus.QUERY_STATE#CLOSE}. * * @param queryId * the query id, not null @@ -1272,7 +1272,7 @@ private VoidResponse close(String queryId, ProxiedUserDetails currentUser, boole public void close(String queryId) throws InterruptedException, QueryException { QueryStatus queryStatus = queryStatusUpdateUtil.lockedUpdate(queryId, status -> { // update query state to CLOSED - status.setQueryState(CLOSED); + status.setQueryState(CLOSE); }); // if the query has no active next calls, delete the results queue @@ -1572,13 +1572,13 @@ public GenericResponse update(String queryId, MultiValueMap updateParameters(parameters, currentParams); // define a duplicate query - return storeQuery(currentParams.getFirst(QUERY_LOGIC_NAME), currentParams, currentUser, CREATED); + return storeQuery(currentParams.getFirst(QUERY_LOGIC_NAME), currentParams, currentUser, CREATE); } private boolean updateParameters(MultiValueMap newParameters, MultiValueMap currentParams) throws BadRequestQueryException { diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 20eca070..346b545c 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -161,7 +161,7 @@ private boolean isFinished(String queryId) throws QueryException { QueryStatus queryStatus = getQueryStatus(); // if the query state is FAILED, throw an exception up to the query management service with the failure message - if (queryStatus.getQueryState() == QueryStatus.QUERY_STATE.FAILED) { + if (queryStatus.getQueryState() == QueryStatus.QUERY_STATE.FAIL) { log.error("Query [{}]: query failed, aborting next call. Cause: {}", queryId, queryStatus.getFailureMessage()); throw new QueryException(queryStatus.getErrorCode(), queryStatus.getFailureMessage()); @@ -182,7 +182,7 @@ private boolean isFinished(String queryId) throws QueryException { } // 3) was this query canceled? - if (!finished && (canceled || queryStatus.getQueryState() == QueryStatus.QUERY_STATE.CANCELED)) { + if (!finished && (canceled || queryStatus.getQueryState() == QueryStatus.QUERY_STATE.CANCEL)) { log.info("Query [{}]: query cancelled, aborting next call", queryId); // no query metric lifecycle update - assumption is that the cancel call handled that diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java index 755d40aa..4c3bcec9 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java @@ -64,7 +64,7 @@ public void testCancelSuccess() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CANCELED, + QueryStatus.QUERY_STATE.CANCEL, 0, 0, 0, @@ -137,7 +137,7 @@ public void testCancelSuccess_activeNextCall() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CANCELED, + QueryStatus.QUERY_STATE.CANCEL, 0, 0, 0, @@ -311,7 +311,7 @@ public void testAdminCancelSuccess() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CANCELED, + QueryStatus.QUERY_STATE.CANCEL, 0, 0, 0, @@ -409,7 +409,7 @@ public void testAdminCancelAllSuccess() throws Exception { for (QueryStatus queryStatus : queryStatusList) { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CANCELED, + QueryStatus.QUERY_STATE.CANCEL, 0, 0, 0, @@ -482,7 +482,7 @@ public void testAdminCancelAllFailure_notAdminUser() throws Exception { for (QueryStatus queryStatus : queryStatusList) { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java index 2de43f3e..2af2c944 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java @@ -66,7 +66,7 @@ public void testCloseSuccess() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CLOSED, + QueryStatus.QUERY_STATE.CLOSE, 0, 0, 0, @@ -130,7 +130,7 @@ public void testCloseSuccess_activeNextCall() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CLOSED, + QueryStatus.QUERY_STATE.CLOSE, 0, 0, 1, @@ -311,7 +311,7 @@ public void testAdminCloseSuccess() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CLOSED, + QueryStatus.QUERY_STATE.CLOSE, 0, 0, 0, @@ -402,7 +402,7 @@ public void testAdminCloseAllSuccess() throws Exception { for (QueryStatus queryStatus : queryStatusList) { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CLOSED, + QueryStatus.QUERY_STATE.CLOSE, 0, 0, 0, @@ -470,7 +470,7 @@ public void testAdminCloseAllFailure_notAdminUser() throws Exception { for (QueryStatus queryStatus : queryStatusList) { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java index cce2ef89..0a477172 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java @@ -88,7 +88,7 @@ public void testCreateSuccess() throws ParseException, IOException { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java index 9ced2d4f..800df689 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java @@ -77,7 +77,7 @@ public void testDefineSuccess() throws ParseException, IOException { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.DEFINED, + QueryStatus.QUERY_STATE.DEFINE, 0, 0, 0, diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java index 0c55606b..4b79d472 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java @@ -76,7 +76,7 @@ public void testDuplicateSuccess_duplicateOnDefined() throws Exception { QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.DEFINED, + QueryStatus.QUERY_STATE.DEFINE, 0, 0, 0, @@ -88,7 +88,7 @@ public void testDuplicateSuccess_duplicateOnDefined() throws Exception { QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, @@ -147,7 +147,7 @@ public void testDuplicateSuccess_duplicateOnCreated() throws Exception { QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, @@ -159,7 +159,7 @@ public void testDuplicateSuccess_duplicateOnCreated() throws Exception { QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, @@ -231,7 +231,7 @@ public void testDuplicateSuccess_duplicateOnCanceled() throws Exception { QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CANCELED, + QueryStatus.QUERY_STATE.CANCEL, 0, 0, 0, @@ -243,7 +243,7 @@ public void testDuplicateSuccess_duplicateOnCanceled() throws Exception { QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, @@ -325,7 +325,7 @@ public void testDuplicateSuccess_duplicateOnClosed() throws Exception { QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CLOSED, + QueryStatus.QUERY_STATE.CLOSE, 0, 0, 0, @@ -337,7 +337,7 @@ public void testDuplicateSuccess_duplicateOnClosed() throws Exception { QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, @@ -417,7 +417,7 @@ public void testDuplicateSuccess_update() throws Exception { QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.DEFINED, + QueryStatus.QUERY_STATE.DEFINE, 0, 0, 0, @@ -429,7 +429,7 @@ public void testDuplicateSuccess_update() throws Exception { QueryStatus dupeQueryStatus = queryStorageCache.getQueryStatus(dupeQueryId); // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java index 8387feea..44034255 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java @@ -79,7 +79,7 @@ public void testResetSuccess_resetOnDefined() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.DEFINED, + QueryStatus.QUERY_STATE.DEFINE, 0, 0, 0, @@ -93,7 +93,7 @@ public void testResetSuccess_resetOnDefined() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, @@ -156,7 +156,7 @@ public void testResetSuccess_resetOnCreated() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CANCELED, + QueryStatus.QUERY_STATE.CANCEL, 0, 0, 0, @@ -170,7 +170,7 @@ public void testResetSuccess_resetOnCreated() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, @@ -256,7 +256,7 @@ public void testResetSuccess_resetOnClosed() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CLOSED, + QueryStatus.QUERY_STATE.CLOSE, 0, 0, 0, @@ -270,7 +270,7 @@ public void testResetSuccess_resetOnClosed() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, @@ -351,7 +351,7 @@ public void testResetSuccess_resetOnCanceled() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CANCELED, + QueryStatus.QUERY_STATE.CANCEL, 0, 0, 0, @@ -365,7 +365,7 @@ public void testResetSuccess_resetOnCanceled() throws Exception { // @formatter:off assertQueryStatus( - QueryStatus.QUERY_STATE.CREATED, + QueryStatus.QUERY_STATE.CREATE, 0, 0, 0, From 5df80318bb2d9b04ac4151a0fc9b52589ffdae81 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Fri, 11 Feb 2022 22:00:47 +0000 Subject: [PATCH 133/218] Created separate stages for the create task Updated the executor task to periodically update the task lastUpdatedMillis for ALL tasks (not just the NextTask) Updated the orphaned task updater to fail orphaned create tasks and tasks then the query has been closed or cancelled. --- .../java/datawave/microservice/query/runner/NextCall.java | 2 +- .../datawave/microservice/query/QueryServiceNextTest.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 346b545c..337d7539 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -202,7 +202,7 @@ private boolean isFinished(String queryId) throws QueryException { } // 5) have we retrieved all of the results? - if (!finished && !getTaskStates().isCreatingTasks() && !getTaskStates().hasUnfinishedTasks() + if (!finished && (queryStatus.getCreateStage() == QueryStatus.CREATE_STAGE.RESULTS) && !getTaskStates().hasUnfinishedTasks() && queryResultsManager.getNumResultsRemaining(queryId) == 0) { // update the number of results consumed queryStatus = updateNumResultsConsumed(); diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java index afe9844c..c9906995 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java @@ -1,9 +1,11 @@ package datawave.microservice.query; import com.google.common.collect.Iterables; +import com.google.common.collect.Iterators; import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.query.remote.QueryRequest; +import datawave.microservice.query.storage.QueryStatus; import datawave.microservice.query.storage.TaskStates; import datawave.webservice.query.result.event.DefaultEvent; import datawave.webservice.result.BaseResponse; @@ -444,8 +446,8 @@ public void testNextSuccess_noResults() throws Exception { for (int i = 0; i < taskStates.getNextTaskId(); i++) { taskStates.setState(i, TaskStates.TASK_STATE.COMPLETED); } - taskStates.setCreatingTasks(false); queryStorageCache.updateTaskStates(taskStates); + queryStorageCache.updateCreateStage(queryId, QueryStatus.CREATE_STAGE.RESULTS); // make the next call asynchronously Future> future = nextQuery(authUser, queryId); From 5150527c0846eafc254083ef0240092a68661779 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Fri, 18 Feb 2022 21:25:13 +0000 Subject: [PATCH 134/218] Working on stress testing the system Plugged a hole discovered during some stress testing where results may be dropped while closing a listener Ensure we update the last updated tasks immediately when setting them to a running state --- .../java/datawave/microservice/query/runner/NextCall.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 337d7539..9643e35a 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -202,8 +202,8 @@ private boolean isFinished(String queryId) throws QueryException { } // 5) have we retrieved all of the results? - if (!finished && (queryStatus.getCreateStage() == QueryStatus.CREATE_STAGE.RESULTS) && !getTaskStates().hasUnfinishedTasks() - && queryResultsManager.getNumResultsRemaining(queryId) == 0) { + if (!finished && (queryStatus.getCreateStage() == QueryStatus.CREATE_STAGE.RESULTS) && !getTaskStates().hasUnfinishedTasks()) { + // update the number of results consumed queryStatus = updateNumResultsConsumed(); @@ -221,6 +221,9 @@ private boolean isFinished(String queryId) throws QueryException { // how many results does the broker think are left long brokerResultsRemaining = queryResultsManager.getNumResultsRemaining(queryId); + log.info("All tasks appear to be completed for " + queryId + " with " + queryResultsRemaining + " yet to be retrieved and " + brokerResultsRemaining + + " left in broker"); + // if the broker thinks there are not results left, we may be done if (brokerResultsRemaining == 0) { From ff8468aed2a9c4408c0101c969dc182b66d5c405 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Fri, 25 Feb 2022 19:01:19 +0000 Subject: [PATCH 135/218] tweaks to the configurations and scripts for stress testing purposes. --- .../microservice/query/QueryManagementService.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 95d16fb4..15cc8924 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -444,7 +444,14 @@ public GenericResponse predict(String queryLogicName, MultiValueMap predictions = status.getPredictions(); + if (predictions != null && !predictions.isEmpty()) { + queryPrediction = predictions.toString(); + } + } queryStorageCache.deleteQuery(taskKey.getQueryId()); GenericResponse response = new GenericResponse<>(); response.setResult(queryPrediction); From e13704dbcd2c0358fa3e10643f0d8c35d4beb734 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 9 Mar 2022 18:05:41 -0500 Subject: [PATCH 136/218] Tested edge event, hit hightlights, and term frequency query logics from the query logic factory. --- .../datawave/microservice/query/QueryManagementService.java | 4 ++++ .../datawave/microservice/query/QueryServiceNextTest.java | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 15cc8924..529bd980 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -540,6 +540,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p taskKey = queryStorageCache.defineQuery( getPoolName(), query, + currentUser, downgradedAuthorizations, getMaxConcurrentTasks(queryLogic)); // @formatter:on @@ -548,6 +549,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p taskKey = queryStorageCache.createQuery( getPoolName(), query, + currentUser, downgradedAuthorizations, getMaxConcurrentTasks(queryLogic)); // @formatter:on @@ -558,6 +560,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p taskKey = queryStorageCache.planQuery( getPoolName(), query, + currentUser, downgradedAuthorizations); // @formatter:on @@ -567,6 +570,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p taskKey = queryStorageCache.predictQuery( getPoolName(), query, + currentUser, downgradedAuthorizations); // @formatter:on diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java index c9906995..dfbb790d 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java @@ -1,7 +1,6 @@ package datawave.microservice.query; import com.google.common.collect.Iterables; -import com.google.common.collect.Iterators; import datawave.microservice.authorization.service.RemoteAuthorizationServiceUserDetailsService; import datawave.microservice.authorization.user.ProxiedUserDetails; import datawave.microservice.query.remote.QueryRequest; From 011415382c607f2c0bca713742adfda71044c3b4 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 24 Mar 2022 16:34:37 -0400 Subject: [PATCH 137/218] Updated query metric service to lookup query metrics using the query microservices. --- .../query/QueryManagementService.java | 70 +++++++++++-------- .../query/QueryServiceListTest.java | 2 +- 2 files changed, 43 insertions(+), 29 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 529bd980..c6817c85 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -1095,21 +1095,28 @@ public void cancel(String queryId, boolean publishEvent) throws InterruptedExcep // publish a cancel event to the executor pool publishExecutorEvent(cancelRequest, queryStatus.getQueryKey().getQueryPool()); - // update query metrics - BaseQueryMetric baseQueryMetric = getBaseQueryMetric(); - baseQueryMetric.setQueryId(queryId); - baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.CANCELLED); - baseQueryMetric.setLastUpdated(new Date()); try { - // @formatter:off - queryMetricClient.submit( - new QueryMetricClient.Request.Builder() - .withMetric(baseQueryMetric.duplicate()) - .withMetricType(QueryMetricType.DISTRIBUTED) - .build()); - // @formatter:on - } catch (Exception e) { - log.error("Error updating query metric", e); + QueryLogic logic = queryLogicFactory.getQueryLogic(queryStatus.getQuery().getQueryLogicName()); + if (logic.getCollectQueryMetrics()) { + // update query metrics + BaseQueryMetric baseQueryMetric = getBaseQueryMetric(); + baseQueryMetric.setQueryId(queryId); + baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.CANCELLED); + baseQueryMetric.setLastUpdated(new Date()); + try { + // @formatter:off + queryMetricClient.submit( + new QueryMetricClient.Request.Builder() + .withMetric(baseQueryMetric.duplicate()) + .withMetricType(QueryMetricType.DISTRIBUTED) + .build()); + // @formatter:on + } catch (Exception e) { + log.error("Error updating query metric", e); + } + } + } catch (CloneNotSupportedException e) { + log.warn("Could not determine whether the query logic supports metrics"); } } } @@ -1294,21 +1301,28 @@ public void close(String queryId) throws InterruptedException, QueryException { // publish a close event to the executor pool publishExecutorEvent(QueryRequest.close(queryId), queryStatus.getQueryKey().getQueryPool()); - // update query metrics - BaseQueryMetric baseQueryMetric = getBaseQueryMetric(); - baseQueryMetric.setQueryId(queryId); - baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.CLOSED); - baseQueryMetric.setLastUpdated(new Date()); try { - // @formatter:off - queryMetricClient.submit( - new QueryMetricClient.Request.Builder() - .withMetric(baseQueryMetric.duplicate()) - .withMetricType(QueryMetricType.DISTRIBUTED) - .build()); - // @formatter:on - } catch (Exception e) { - log.error("Error updating query metric", e); + QueryLogic logic = queryLogicFactory.getQueryLogic(queryStatus.getQuery().getQueryLogicName()); + if (logic.getCollectQueryMetrics()) { + // update query metrics + BaseQueryMetric baseQueryMetric = getBaseQueryMetric(); + baseQueryMetric.setQueryId(queryId); + baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.CLOSED); + baseQueryMetric.setLastUpdated(new Date()); + try { + // @formatter:off + queryMetricClient.submit( + new QueryMetricClient.Request.Builder() + .withMetric(baseQueryMetric.duplicate()) + .withMetricType(QueryMetricType.DISTRIBUTED) + .build()); + // @formatter:on + } catch (Exception e) { + log.error("Error updating query metric", e); + } + } + } catch (CloneNotSupportedException e) { + log.warn("Could not determine whether the query logic supports metrics"); } } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java index 104484bc..af5bd486 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java @@ -421,7 +421,7 @@ public void testListQueryLogicSuccess() throws Exception { String[] expectedQueryLogics = new String[] {"AltEventQuery", "ContentQuery", "CountQuery", "DiscoveryQuery", "EdgeEventQuery", "EdgeQuery", "ErrorCountQuery", "ErrorDiscoveryQuery", "ErrorEventQuery", "ErrorFieldIndexCountQuery", "EventQuery", "FacetedQuery", "FieldIndexCountQuery", - "HitHighlights", "IndexStatsQuery", "LuceneUUIDEventQuery", "QueryMetricsQuery", "TermFrequencyQuery"}; + "HitHighlights", "IndexStatsQuery", "LuceneUUIDEventQuery", "QueryMetricsQuery", "InternalQueryMetricsQuery", "TermFrequencyQuery"}; Assert.assertEquals(expectedQueryLogics.length, qlResponse.getQueryLogicList().size()); From 13f2f670a816e824e05042ea3e53fdc2dbdd8222 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 1 Apr 2022 15:04:54 -0400 Subject: [PATCH 138/218] Added translate id functionality and rest endpoints. --- .../microservice/query/QueryController.java | 21 +- .../query/lookup/LookupService.java | 45 ++-- .../query/translateid/TranslateIdService.java | 212 ++++++++++++++++++ 3 files changed, 256 insertions(+), 22 deletions(-) create mode 100644 query-microservices/query-service/service/src/main/java/datawave/microservice/query/translateid/TranslateIdService.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 766bf3eb..7a4844aa 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -6,6 +6,7 @@ import datawave.microservice.query.stream.StreamingProperties; import datawave.microservice.query.stream.StreamingService; import datawave.microservice.query.stream.listener.CountingResponseBodyEmitterListener; +import datawave.microservice.query.translateid.TranslateIdService; import datawave.microservice.query.web.annotation.EnrichQueryMetrics; import datawave.microservice.query.web.filter.BaseMethodStatsFilter; import datawave.microservice.query.web.filter.CountingResponseBodyEmitter; @@ -43,6 +44,7 @@ public class QueryController { private final QueryManagementService queryManagementService; private final LookupService lookupService; private final StreamingService streamingService; + private final TranslateIdService translateIdService; private final StreamingProperties streamingProperties; @@ -52,11 +54,13 @@ public class QueryController { private final QueryMetricsEnrichmentFilterAdvice.QueryMetricsEnrichmentContext queryMetricsEnrichmentContext; public QueryController(QueryManagementService queryManagementService, LookupService lookupService, StreamingService streamingService, - StreamingProperties streamingProperties, BaseMethodStatsFilter.BaseMethodStatsContext baseMethodStatsContext, + TranslateIdService translateIdService, StreamingProperties streamingProperties, + BaseMethodStatsFilter.BaseMethodStatsContext baseMethodStatsContext, QueryMetricsEnrichmentFilterAdvice.QueryMetricsEnrichmentContext queryMetricsEnrichmentContext) { this.queryManagementService = queryManagementService; this.lookupService = lookupService; this.streamingService = streamingService; + this.translateIdService = translateIdService; this.streamingProperties = streamingProperties; this.baseMethodStatsContext = baseMethodStatsContext; this.queryMetricsEnrichmentContext = queryMetricsEnrichmentContext; @@ -169,6 +173,21 @@ public Object lookupContentUUIDBatch(@RequestParam MultiValueMap } } + @RequestMapping(path = "translateId/{id}", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public BaseQueryResponse translateId(@PathVariable String id, @RequestParam MultiValueMap parameters, + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + return translateIdService.translateId(id, parameters, currentUser); + } + + // TODO: Shouldn't the case for this path be the same as the singular call? + @RequestMapping(path = "translateIDs", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", + "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) + public BaseQueryResponse translateIDs(@RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) + throws QueryException { + return translateIdService.translateIds(parameters, currentUser); + } + @Timed(name = "dw.query.createAndNext", absolute = true) @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE_AND_NEXT) @RequestMapping(path = "{queryLogic}/createAndNext", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/lookup/LookupService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/lookup/LookupService.java index 56af7cd1..188da450 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/lookup/LookupService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/lookup/LookupService.java @@ -39,7 +39,11 @@ import java.util.Set; import java.util.UUID; +import static datawave.microservice.query.QueryParameters.QUERY_AUTHORIZATIONS; +import static datawave.microservice.query.QueryParameters.QUERY_BEGIN; +import static datawave.microservice.query.QueryParameters.QUERY_END; import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; +import static datawave.microservice.query.QueryParameters.QUERY_NAME; import static datawave.microservice.query.QueryParameters.QUERY_STRING; import static datawave.query.QueryParameters.QUERY_SYNTAX; import static datawave.services.query.logic.lookup.LookupQueryLogic.LOOKUP_KEY_VALUE_DELIMITER; @@ -315,7 +319,7 @@ private T lookupEvents(MultiValueMap lookupTermMap, LookupQue parameters.put(QUERY_STRING, Collections.singletonList(lookupQueryLogic.createQueryFromLookupTerms(lookupTermMap))); // update the parameters for query - setEventQueryParameters(parameters, currentUser); + setupEventQueryParameters(parameters, currentUser); // create the query queryId = queryManagementService.create(parameters.getFirst(QUERY_LOGIC_NAME), parameters, currentUser).getResult(); @@ -417,33 +421,32 @@ else if (!queryLogicName.equals(uuidType.getQueryLogic())) { } @SuppressWarnings("ConstantConditions") - protected void setEventQueryParameters(MultiValueMap parameters, ProxiedUserDetails currentUser) { + protected void setupEventQueryParameters(MultiValueMap parameters, ProxiedUserDetails currentUser) { String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); - - setOptionalQueryParameters(parameters); - - parameters.set(QUERY_SYNTAX, LUCENE_UUID_SYNTAX); + final String queryName = user + "-" + UUID.randomUUID().toString(); // Override the extraneous query details String userAuths; - if (parameters.containsKey(QueryParameters.QUERY_AUTHORIZATIONS)) { - userAuths = AuthorizationsUtil.downgradeUserAuths(currentUser, parameters.getFirst(QueryParameters.QUERY_AUTHORIZATIONS)); + if (parameters.containsKey(QUERY_AUTHORIZATIONS)) { + userAuths = AuthorizationsUtil.downgradeUserAuths(currentUser, parameters.getFirst(QUERY_AUTHORIZATIONS)); } else { userAuths = AuthorizationsUtil.buildUserAuthorizationString(currentUser); } - parameters.set(QueryParameters.QUERY_AUTHORIZATIONS, userAuths); - - final String queryName = user + "-" + UUID.randomUUID(); - parameters.set(QueryParameters.QUERY_NAME, queryName); - parameters.set(QueryParameters.QUERY_BEGIN, lookupProperties.getBeginDate()); - - final Date endDate = DateUtils.addDays(new Date(), 2); + final String endDate; try { - parameters.set(QueryParameters.QUERY_END, DefaultQueryParameters.formatDate(endDate)); + endDate = DefaultQueryParameters.formatDate(DateUtils.addDays(new Date(), 2)); } catch (ParseException e) { - throw new RuntimeException("Unable to format new query end date: " + endDate); + throw new RuntimeException("Unable to format new query end date"); } + + setOptionalQueryParameters(parameters); + + parameters.set(QUERY_SYNTAX, LUCENE_UUID_SYNTAX); + parameters.set(QUERY_NAME, queryName); + parameters.set(QUERY_AUTHORIZATIONS, userAuths); + parameters.set(QUERY_BEGIN, lookupProperties.getBeginDate()); + parameters.set(QUERY_END, endDate); } protected void setOptionalQueryParameters(MultiValueMap parameters) { @@ -553,19 +556,19 @@ protected void setContentQueryParameters(MultiValueMap parameters // all content queries use the same query logic parameters.put(QUERY_LOGIC_NAME, Collections.singletonList(lookupProperties.getContentQueryLogicName())); - parameters.set(QueryParameters.QUERY_NAME, user + '-' + UUID.randomUUID()); + parameters.set(QUERY_NAME, user + '-' + UUID.randomUUID()); - parameters.set(QueryParameters.QUERY_BEGIN, lookupProperties.getBeginDate()); + parameters.set(QUERY_BEGIN, lookupProperties.getBeginDate()); final Date endDate = new Date(); try { - parameters.set(QueryParameters.QUERY_END, DefaultQueryParameters.formatDate(endDate)); + parameters.set(QUERY_END, DefaultQueryParameters.formatDate(endDate)); } catch (ParseException e1) { throw new RuntimeException("Error formatting end date: " + endDate); } final String userAuths = AuthorizationsUtil.buildUserAuthorizationString(currentUser); - parameters.set(QueryParameters.QUERY_AUTHORIZATIONS, userAuths); + parameters.set(QUERY_AUTHORIZATIONS, userAuths); } protected EventQueryResponseBase runContentQuery(MultiValueMap parameters, ProxiedUserDetails currentUser) { diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/translateid/TranslateIdService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/translateid/TranslateIdService.java new file mode 100644 index 00000000..83a00cc5 --- /dev/null +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/translateid/TranslateIdService.java @@ -0,0 +1,212 @@ +package datawave.microservice.query.translateid; + +import datawave.microservice.authorization.user.ProxiedUserDetails; +import datawave.microservice.authorization.util.AuthorizationsUtil; +import datawave.microservice.query.DefaultQueryParameters; +import datawave.microservice.query.QueryManagementService; +import datawave.microservice.query.QueryParameters; +import datawave.security.util.ProxiedEntityUtils; +import datawave.webservice.query.exception.BadRequestQueryException; +import datawave.webservice.query.exception.DatawaveErrorCode; +import datawave.webservice.query.exception.NoResultsQueryException; +import datawave.webservice.query.exception.QueryException; +import datawave.webservice.query.exception.TimeoutQueryException; +import datawave.webservice.query.exception.UnauthorizedQueryException; +import datawave.webservice.result.BaseQueryResponse; +import org.apache.commons.lang.time.DateUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; + +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +import static datawave.microservice.query.QueryParameters.QUERY_AUTHORIZATIONS; +import static datawave.microservice.query.QueryParameters.QUERY_BEGIN; +import static datawave.microservice.query.QueryParameters.QUERY_END; +import static datawave.microservice.query.QueryParameters.QUERY_LOGIC_NAME; +import static datawave.microservice.query.QueryParameters.QUERY_NAME; +import static datawave.microservice.query.QueryParameters.QUERY_STRING; +import static datawave.query.QueryParameters.QUERY_SYNTAX; +import static datawave.webservice.query.exception.DatawaveErrorCode.MISSING_REQUIRED_PARAMETER; + +@Service +public class TranslateIdService { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + + public static final String TRANSLATE_ID = "id"; + public static final String TRANSLATE_TLD_ONLY = "TLDonly"; + public static final String LUCENE_SYNTAX = "LUCENE"; + + private final TranslateIdProperties translateIdProperties; + + private final QueryManagementService queryManagementService; + + public TranslateIdService(TranslateIdProperties translateIdProperties, QueryManagementService queryManagementService) { + this.translateIdProperties = translateIdProperties; + this.queryManagementService = queryManagementService; + } + + /** + * Get one or more ID(s), if any, that correspond to the given ID. This method only returns the first page, so set pagesize appropriately. Since the + * underlying query is automatically closed, callers are NOT expected to request additional pages or close the query. + * + * @param id + * the id to translate + * @param parameters + * the query parameters, not null + * @param currentUser + * the user who called this method, not null + * @return a base query response containing the first page of results + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws TimeoutQueryException + * if the next call times out + * @throws NoResultsQueryException + * if no query results are found + * @throws QueryException + * if this next task is rejected by the executor + * @throws QueryException + * if there is an unknown error + */ + public BaseQueryResponse translateId(String id, MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + String queryId = null; + try { + parameters.set(TRANSLATE_ID, id); + + BaseQueryResponse response = translateIds(parameters, currentUser); + queryId = response.getQueryId(); + return response; + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error with translateId", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error with translateId."); + } finally { + // close the query if applicable + if (queryId != null) { + queryManagementService.close(queryId, currentUser); + } + } + } + + /** + * Get the ID(s), if any, associated with the specified IDs. Because the query created by this call may return multiple pages, callers are expected to + * request additional pages and eventually close the query. + * + * @param parameters + * the query parameters, not null + * @param currentUser + * the user who called this method, not null + * @return a base query response containing the first page of results + * @throws BadRequestQueryException + * if parameter validation fails + * @throws BadRequestQueryException + * if query logic parameter validation fails + * @throws UnauthorizedQueryException + * if the user doesn't have access to the requested query logic + * @throws BadRequestQueryException + * if security marking validation fails + * @throws BadRequestQueryException + * if auditing fails + * @throws QueryException + * if query storage fails + * @throws TimeoutQueryException + * if the next call times out + * @throws NoResultsQueryException + * if no query results are found + * @throws QueryException + * if this next task is rejected by the executor + * @throws QueryException + * if there is an unknown error + */ + public BaseQueryResponse translateIds(MultiValueMap parameters, ProxiedUserDetails currentUser) throws QueryException { + if (!parameters.containsKey(TRANSLATE_ID)) { + throw new BadRequestQueryException(MISSING_REQUIRED_PARAMETER, "Missing required parameter: " + TRANSLATE_ID); + } + + try { + MultiValueMap queryParams = setupQueryParameters(parameters, currentUser); + return queryManagementService.createAndNext(parameters.getFirst(QUERY_LOGIC_NAME), queryParams, currentUser); + } catch (QueryException e) { + throw e; + } catch (Exception e) { + log.error("Unknown error with translateIds", e); + throw new QueryException(DatawaveErrorCode.QUERY_SETUP_ERROR, e, "Unknown error with translateIds."); + } + } + + protected MultiValueMap setupQueryParameters(MultiValueMap parameters, ProxiedUserDetails currentUser) { + MultiValueMap queryParams = new LinkedMultiValueMap<>(); + + // copy over any query parameters which are explicitly allowed to be set, ignoring ones that aren't + for (String queryParam : translateIdProperties.getAllowedQueryParameters()) { + if (parameters.containsKey(queryParam)) { + queryParams.put(queryParam, parameters.get(queryParam)); + } + } + + String user = ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()); + String queryName = user + "-" + UUID.randomUUID().toString(); + + String queryLogic; + if (Boolean.parseBoolean(parameters.getFirst(TRANSLATE_TLD_ONLY))) { + queryLogic = translateIdProperties.getTldQueryLogicName(); + } else { + queryLogic = translateIdProperties.getQueryLogicName(); + } + + String endDate; + try { + endDate = DefaultQueryParameters.formatDate(DateUtils.addDays(new Date(), 2)); + } catch (ParseException e) { + throw new RuntimeException("Unable to format new query end date"); + } + + setOptionalQueryParameters(queryParams); + + queryParams.set(QUERY_SYNTAX, LUCENE_SYNTAX); + queryParams.add(QUERY_NAME, queryName); + queryParams.add(QUERY_LOGIC_NAME, queryLogic); + queryParams.add(QUERY_STRING, buildQuery(parameters.get(TRANSLATE_ID))); + queryParams.set(QUERY_AUTHORIZATIONS, AuthorizationsUtil.buildUserAuthorizationString(currentUser)); + queryParams.add(QUERY_BEGIN, translateIdProperties.getBeginDate()); + queryParams.set(QUERY_END, endDate); + + return queryParams; + } + + protected void setOptionalQueryParameters(MultiValueMap parameters) { + if (translateIdProperties.getColumnVisibility() != null) { + parameters.set(QueryParameters.QUERY_VISIBILITY, translateIdProperties.getColumnVisibility()); + } + } + + private String buildQuery(List ids) { + List uuidTypes = new ArrayList<>(); + translateIdProperties.getTypes().keySet().forEach(uuidType -> uuidTypes.add(uuidType.toUpperCase())); + + // @formatter:off + return ids.stream() + .map(id -> "\"" + id + "\"") + .flatMap(id -> uuidTypes.stream().map(uuidType -> uuidType + ":" + id)) + .collect(Collectors.joining(" OR ")); + // @formatter:on + } +} From 23e51f8af9d44ca9608033cd0a70a3ef610a730e Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Wed, 20 Apr 2022 19:28:29 +0000 Subject: [PATCH 139/218] Merge branch 'integration' into feature/queryMicroservices --- .../microservice/query/QueryManagementService.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java index c6817c85..618ac195 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -515,7 +515,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p String userDn = currentUser.getPrimaryUser().getDn().subjectDN(); setInternalAuditParameters(queryLogicName, userDn, parameters); - Query query = createQuery(queryLogicName, parameters, userDn, currentUser.getDNs(), queryId); + Query query = createQuery(queryLogicName, parameters, userDn, getDNs(currentUser), queryId); // TODO: Downgrade the auths before or after auditing??? // downgrade the auths @@ -588,7 +588,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p baseQueryMetric.setQueryId(taskKey.getQueryId()); baseQueryMetric.setLifecycle(BaseQueryMetric.Lifecycle.DEFINED); baseQueryMetric.populate(query); - baseQueryMetric.setProxyServers(currentUser.getProxyServers()); + baseQueryMetric.setProxyServers(getDNs(currentUser)); } return taskKey; @@ -1623,7 +1623,7 @@ public GenericResponse update(String queryId, MultiValueMap status.setQuery(query)); @@ -2425,7 +2425,7 @@ protected void validateQueryLogic(QueryLogic queryLogic, MultiValueMap dnList = currentUser.getDNs(); + List dnList = getDNs(currentUser); if (!queryLogic.containsDNWithAccess(dnList)) { throw new UnauthorizedQueryException("None of the DNs used have access to this query logic: " + dnList, 401); } @@ -2514,4 +2514,8 @@ public ThreadLocal getSecurityMarkingOverride() { public ThreadLocal getBaseQueryMetricOverride() { return baseQueryMetricOverride; } + + public List getDNs(ProxiedUserDetails user) { + return user.getProxiedUsers().stream().map(u -> u.getDn().subjectDN()).collect(Collectors.toList()); + } } From fb74779d2570a3a6b8a5a2a4560b2e3d2bbbecec Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Mon, 25 Apr 2022 17:03:41 +0000 Subject: [PATCH 140/218] Merge remote-tracking branch 'origin/integration' into feature/queryMicroservices --- query-microservices/query-service/api/pom.xml | 10 ---------- .../microservice/query/QueryController.java | 16 ++++++++-------- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/query-microservices/query-service/api/pom.xml b/query-microservices/query-service/api/pom.xml index 3a27995a..7f785dc1 100644 --- a/query-microservices/query-service/api/pom.xml +++ b/query-microservices/query-service/api/pom.xml @@ -36,21 +36,11 @@ jakarta.validation jakarta.validation-api - - junit - junit - test - org.junit.jupiter junit-jupiter-engine test - - org.junit.vintage - junit-vintage-engine - test - diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java index 7a4844aa..8780d9cc 100644 --- a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java @@ -21,6 +21,7 @@ import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; +import org.springframework.security.access.annotation.Secured; import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.util.MultiValueMap; import org.springframework.web.bind.annotation.PathVariable; @@ -31,7 +32,6 @@ import org.springframework.web.bind.annotation.RestController; import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyEmitter; -import javax.annotation.security.RolesAllowed; import java.util.List; import static datawave.microservice.query.lookup.LookupService.LOOKUP_STREAMING; @@ -213,7 +213,7 @@ public VoidResponse cancel(@PathVariable String queryId, @AuthenticationPrincipa } @Timed(name = "dw.query.adminCancel", absolute = true) - @RolesAllowed({"Administrator", "JBossAdministrator"}) + @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "{queryId}/adminCancel", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public VoidResponse adminCancel(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { @@ -228,7 +228,7 @@ public VoidResponse close(@PathVariable String queryId, @AuthenticationPrincipal } @Timed(name = "dw.query.adminClose", absolute = true) - @RolesAllowed({"Administrator", "JBossAdministrator"}) + @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "{queryId}/adminClose", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public VoidResponse adminClose(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { @@ -250,7 +250,7 @@ public VoidResponse remove(@PathVariable String queryId, @AuthenticationPrincipa } @Timed(name = "dw.query.adminRemove", absolute = true) - @RolesAllowed({"Administrator", "JBossAdministrator"}) + @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "{queryId}/adminRemove", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public VoidResponse adminRemove(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { @@ -282,7 +282,7 @@ public QueryImplListResponse list(@RequestParam(required = false) String queryId } @Timed(name = "dw.query.adminList", absolute = true) - @RolesAllowed({"Administrator", "JBossAdministrator"}) + @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "adminList", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public QueryImplListResponse adminList(@RequestParam(required = false) String queryId, @RequestParam(required = false) String user, @@ -312,7 +312,7 @@ public GenericResponse predictions(@PathVariable String queryId, @Authen } @Timed(name = "dw.query.adminCancelAll", absolute = true) - @RolesAllowed({"Administrator", "JBossAdministrator"}) + @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "adminCancelAll", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public VoidResponse adminCancelAll(@AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { @@ -320,7 +320,7 @@ public VoidResponse adminCancelAll(@AuthenticationPrincipal ProxiedUserDetails c } @Timed(name = "dw.query.adminCloseAll", absolute = true) - @RolesAllowed({"Administrator", "JBossAdministrator"}) + @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "adminCloseAll", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public VoidResponse adminCloseAll(@AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { @@ -328,7 +328,7 @@ public VoidResponse adminCloseAll(@AuthenticationPrincipal ProxiedUserDetails cu } @Timed(name = "dw.query.adminRemoveAll", absolute = true) - @RolesAllowed({"Administrator", "JBossAdministrator"}) + @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "adminRemoveAll", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public VoidResponse adminRemoveAll(@AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { From d4f03e4e4d931e7fb90177774b85a1dbc95fb340 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Mon, 25 Apr 2022 20:12:05 +0000 Subject: [PATCH 141/218] xml.bind from javax to jakarta Changed required for integration branch merge --- .../microservice/query/stream/StreamingServiceTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java index 71aa0799..4929d87f 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java @@ -29,9 +29,9 @@ import org.springframework.util.MultiValueMap; import org.springframework.web.util.UriComponents; -import javax.xml.bind.JAXBContext; -import javax.xml.bind.JAXBException; -import javax.xml.bind.Unmarshaller; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.JAXBException; +import jakarta.xml.bind.Unmarshaller; import java.io.StringReader; import java.util.ArrayList; import java.util.Arrays; From 8a5ae94c5f4f0911a7b6578018736921944d0147 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Tue, 26 Apr 2022 18:59:13 +0000 Subject: [PATCH 142/218] Revert "xml.bind from javax to jakarta" This reverts commit e384e9ad8038e21286f18a43b9dbf8af79c9548a. --- .../microservice/query/stream/StreamingServiceTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java index 4929d87f..71aa0799 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java @@ -29,9 +29,9 @@ import org.springframework.util.MultiValueMap; import org.springframework.web.util.UriComponents; -import jakarta.xml.bind.JAXBContext; -import jakarta.xml.bind.JAXBException; -import jakarta.xml.bind.Unmarshaller; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Unmarshaller; import java.io.StringReader; import java.util.ArrayList; import java.util.Arrays; From 27361717df7c314ba78453ce5a5c7b44e6a4d57b Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 29 Apr 2022 12:20:11 -0400 Subject: [PATCH 143/218] Removed Junit 4 and vintage junit for the query microservices. --- .../query-service/service/pom.xml | 10 -- .../query/AbstractQueryServiceTest.java | 90 ++++++------- .../query/QueryServiceCancelTest.java | 54 ++++---- .../query/QueryServiceCloseTest.java | 54 ++++---- .../query/QueryServiceCreateTest.java | 70 +++++----- .../query/QueryServiceDefineTest.java | 68 +++++----- .../query/QueryServiceDuplicateTest.java | 124 +++++++++--------- .../query/QueryServiceListTest.java | 94 ++++++------- .../query/QueryServiceNextTest.java | 113 ++++++++-------- .../query/QueryServicePlanTest.java | 28 ++-- .../query/QueryServiceRemoveTest.java | 90 ++++++------- .../query/QueryServiceResetTest.java | 62 ++++----- .../query/QueryServiceUpdateTest.java | 80 +++++------ .../query/lookup/LookupServiceTest.java | 92 ++++++------- .../query/stream/StreamingServiceTest.java | 30 ++--- 15 files changed, 525 insertions(+), 534 deletions(-) diff --git a/query-microservices/query-service/service/pom.xml b/query-microservices/query-service/service/pom.xml index a6099a52..bb8a9866 100644 --- a/query-microservices/query-service/service/pom.xml +++ b/query-microservices/query-service/service/pom.xml @@ -72,21 +72,11 @@ test-jar test - - junit - junit - test - org.junit.jupiter junit-jupiter-engine test - - org.junit.vintage - junit-vintage-engine - test - org.springframework.cloud spring-cloud-stream-test-support diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java index 8aa28ec8..140ce3b4 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java @@ -30,7 +30,7 @@ import datawave.webservice.result.VoidResponse; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URLEncodedUtils; -import org.junit.Assert; +import org.junit.jupiter.api.Assertions; import org.springframework.beans.DirectFieldAccessor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.web.client.RestTemplateBuilder; @@ -379,68 +379,68 @@ protected MultiValueMap createParams() { } protected void assertDefaultEvent(List fields, List values, DefaultEvent event) { - Assert.assertEquals(fields, event.getFields().stream().map(DefaultField::getName).collect(Collectors.toList())); - Assert.assertEquals(values, event.getFields().stream().map(DefaultField::getValueString).collect(Collectors.toList())); + Assertions.assertEquals(fields, event.getFields().stream().map(DefaultField::getName).collect(Collectors.toList())); + Assertions.assertEquals(values, event.getFields().stream().map(DefaultField::getValueString).collect(Collectors.toList())); } protected void assertQueryResponse(String queryId, String logicName, long pageNumber, boolean partialResults, long operationTimeInMS, int numFields, List fieldNames, int numEvents, DefaultEventQueryResponse queryResponse) { - Assert.assertEquals(queryId, queryResponse.getQueryId()); - Assert.assertEquals(logicName, queryResponse.getLogicName()); - Assert.assertEquals(pageNumber, queryResponse.getPageNumber()); - Assert.assertEquals(partialResults, queryResponse.isPartialResults()); - Assert.assertEquals(operationTimeInMS, queryResponse.getOperationTimeMS()); - Assert.assertEquals(numFields, queryResponse.getFields().size()); - Assert.assertEquals(fieldNames, queryResponse.getFields()); - Assert.assertEquals(numEvents, queryResponse.getEvents().size()); + Assertions.assertEquals(queryId, queryResponse.getQueryId()); + Assertions.assertEquals(logicName, queryResponse.getLogicName()); + Assertions.assertEquals(pageNumber, queryResponse.getPageNumber()); + Assertions.assertEquals(partialResults, queryResponse.isPartialResults()); + Assertions.assertEquals(operationTimeInMS, queryResponse.getOperationTimeMS()); + Assertions.assertEquals(numFields, queryResponse.getFields().size()); + Assertions.assertEquals(fieldNames, queryResponse.getFields()); + Assertions.assertEquals(numEvents, queryResponse.getEvents().size()); } protected void assertQueryRequestEvent(String destination, QueryRequest.Method method, String queryId, RemoteQueryRequestEvent queryRequestEvent) { - Assert.assertEquals(destination, queryRequestEvent.getDestinationService()); - Assert.assertEquals(queryId, queryRequestEvent.getRequest().getQueryId()); - Assert.assertEquals(method, queryRequestEvent.getRequest().getMethod()); + Assertions.assertEquals(destination, queryRequestEvent.getDestinationService()); + Assertions.assertEquals(queryId, queryRequestEvent.getRequest().getQueryId()); + Assertions.assertEquals(method, queryRequestEvent.getRequest().getMethod()); } protected void assertQueryStatus(QueryStatus.QUERY_STATE queryState, long numResultsReturned, long numResultsGenerated, long activeNextCalls, long lastPageNumber, long lastCallTimeMillis, QueryStatus queryStatus) { - Assert.assertEquals(queryState, queryStatus.getQueryState()); - Assert.assertEquals(numResultsReturned, queryStatus.getNumResultsReturned()); - Assert.assertEquals(numResultsGenerated, queryStatus.getNumResultsGenerated()); - Assert.assertEquals(activeNextCalls, queryStatus.getActiveNextCalls()); - Assert.assertEquals(lastPageNumber, queryStatus.getLastPageNumber()); - Assert.assertTrue(queryStatus.getLastUsedMillis() > lastCallTimeMillis); - Assert.assertTrue(queryStatus.getLastUpdatedMillis() > lastCallTimeMillis); + Assertions.assertEquals(queryState, queryStatus.getQueryState()); + Assertions.assertEquals(numResultsReturned, queryStatus.getNumResultsReturned()); + Assertions.assertEquals(numResultsGenerated, queryStatus.getNumResultsGenerated()); + Assertions.assertEquals(activeNextCalls, queryStatus.getActiveNextCalls()); + Assertions.assertEquals(lastPageNumber, queryStatus.getLastPageNumber()); + Assertions.assertTrue(queryStatus.getLastUsedMillis() > lastCallTimeMillis); + Assertions.assertTrue(queryStatus.getLastUpdatedMillis() > lastCallTimeMillis); } protected void assertQuery(String queryString, String queryName, String authorizations, String begin, String end, String visibility, Query query) throws ParseException { SimpleDateFormat sdf = new SimpleDateFormat(DefaultQueryParameters.formatPattern); - Assert.assertEquals(queryString, query.getQuery()); - Assert.assertEquals(queryName, query.getQueryName()); - Assert.assertEquals(authorizations, query.getQueryAuthorizations()); - Assert.assertEquals(sdf.parse(begin), query.getBeginDate()); - Assert.assertEquals(sdf.parse(end), query.getEndDate()); - Assert.assertEquals(visibility, query.getColumnVisibility()); + Assertions.assertEquals(queryString, query.getQuery()); + Assertions.assertEquals(queryName, query.getQueryName()); + Assertions.assertEquals(authorizations, query.getQueryAuthorizations()); + Assertions.assertEquals(sdf.parse(begin), query.getBeginDate()); + Assertions.assertEquals(sdf.parse(end), query.getEndDate()); + Assertions.assertEquals(visibility, query.getColumnVisibility()); } protected void assertTasksCreated(String queryId) throws IOException { // verify that the query task states were created TaskStates taskStates = queryStorageCache.getTaskStates(queryId); - Assert.assertNotNull(taskStates); + Assertions.assertNotNull(taskStates); // verify that a query task was created List taskKeys = queryStorageCache.getTasks(queryId); - Assert.assertFalse(taskKeys.isEmpty()); + Assertions.assertFalse(taskKeys.isEmpty()); } protected void assertTasksNotCreated(String queryId) throws IOException { // verify that the query task states were not created TaskStates taskStates = queryStorageCache.getTaskStates(queryId); - Assert.assertNull(taskStates); + Assertions.assertNull(taskStates); // verify that a query task was not created List taskKeys = queryStorageCache.getTasks(queryId); - Assert.assertTrue(taskKeys.isEmpty()); + Assertions.assertTrue(taskKeys.isEmpty()); } public RequestMatcher auditIdGrabber() { @@ -464,37 +464,37 @@ protected void auditNotSentSetup() { protected void assertAuditSent(String queryId) { mockServer.verify(); - Assert.assertEquals(1, auditIds.size()); - Assert.assertEquals(queryId, auditIds.get(0)); + Assertions.assertEquals(1, auditIds.size()); + Assertions.assertEquals(queryId, auditIds.get(0)); } protected void assertAuditNotSent() { mockServer.verify(); - Assert.assertEquals(0, auditIds.size()); + Assertions.assertEquals(0, auditIds.size()); } protected void assertQueryException(String message, String cause, String code, QueryExceptionType queryException) { - Assert.assertEquals(message, queryException.getMessage()); - Assert.assertEquals(cause, queryException.getCause()); - Assert.assertEquals(code, queryException.getCode()); + Assertions.assertEquals(message, queryException.getMessage()); + Assertions.assertEquals(cause, queryException.getCause()); + Assertions.assertEquals(code, queryException.getCode()); } protected BaseResponse assertBaseResponse(boolean hasResults, HttpStatus.Series series, ResponseEntity response) { - Assert.assertEquals(series, response.getStatusCode().series()); - Assert.assertNotNull(response); + Assertions.assertEquals(series, response.getStatusCode().series()); + Assertions.assertNotNull(response); BaseResponse baseResponse = response.getBody(); - Assert.assertNotNull(baseResponse); - Assert.assertEquals(hasResults, baseResponse.getHasResults()); + Assertions.assertNotNull(baseResponse); + Assertions.assertEquals(hasResults, baseResponse.getHasResults()); return baseResponse; } @SuppressWarnings("unchecked") protected GenericResponse assertGenericResponse(boolean hasResults, HttpStatus.Series series, ResponseEntity response) { - Assert.assertEquals(series, response.getStatusCode().series()); - Assert.assertNotNull(response); + Assertions.assertEquals(series, response.getStatusCode().series()); + Assertions.assertNotNull(response); GenericResponse genericResponse = (GenericResponse) response.getBody(); - Assert.assertNotNull(genericResponse); - Assert.assertEquals(hasResults, genericResponse.getHasResults()); + Assertions.assertNotNull(genericResponse); + Assertions.assertEquals(hasResults, genericResponse.getHasResults()); return genericResponse; } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java index 4c3bcec9..05e71ea8 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java @@ -7,17 +7,17 @@ import datawave.microservice.query.storage.QueryStatus; import datawave.webservice.result.BaseResponse; import datawave.webservice.result.VoidResponse; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.HttpMethod; import org.springframework.http.RequestEntity; import org.springframework.http.ResponseEntity; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.web.util.UriComponents; import java.util.ArrayList; @@ -29,16 +29,16 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceCancelTest extends AbstractQueryServiceTest { - @Before + @BeforeEach public void setup() { super.setup(); } - @After + @AfterEach public void teardown() throws Exception { super.teardown(); } @@ -57,7 +57,7 @@ public void testCancelSuccess() throws Exception { // the response should come back right away ResponseEntity cancelResponse = cancelFuture.get(); - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); // verify that query status was created correctly QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); @@ -77,7 +77,7 @@ public void testCancelSuccess() throws Exception { assertTasksCreated(queryId); // verify that the cancel event was published - Assert.assertEquals(3, queryRequestEvents.size()); + Assertions.assertEquals(3, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -126,7 +126,7 @@ public void testCancelSuccess_activeNextCall() throws Exception { // the response should come back right away ResponseEntity cancelResponse = cancelFuture.get(); - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); // wait for the next call to drop out before checking status // since we canceled, this should quit immediately, but just in case, we add a timeout @@ -153,7 +153,7 @@ public void testCancelSuccess_activeNextCall() throws Exception { assertTasksCreated(queryId); // verify that the close event was published - Assert.assertEquals(4, queryRequestEvents.size()); + Assertions.assertEquals(4, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -190,7 +190,7 @@ public void testCancelFailure_queryNotFound() throws Exception { // the response should come back right away ResponseEntity cancelResponse = cancelFuture.get(); - Assert.assertEquals(404, cancelResponse.getStatusCodeValue()); + Assertions.assertEquals(404, cancelResponse.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -216,7 +216,7 @@ public void testCancelFailure_ownershipFailure() throws Exception { // the response should come back right away ResponseEntity response = future.get(); - Assert.assertEquals(401, response.getStatusCodeValue()); + Assertions.assertEquals(401, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -227,7 +227,7 @@ public void testCancelFailure_ownershipFailure() throws Exception { // @formatter:on // verify that the next events were published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -250,7 +250,7 @@ public void testCancelFailure_queryNotRunning() throws Exception { // the response should come back right away ResponseEntity cancelResponse = cancelFuture.get(); - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); // try to cancel the query again cancelFuture = cancelQuery(authUser, queryId); @@ -258,7 +258,7 @@ public void testCancelFailure_queryNotRunning() throws Exception { // the response should come back right away cancelResponse = cancelFuture.get(); - Assert.assertEquals(400, cancelResponse.getStatusCodeValue()); + Assertions.assertEquals(400, cancelResponse.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -269,7 +269,7 @@ public void testCancelFailure_queryNotRunning() throws Exception { // @formatter:on // verify that the next events were published - Assert.assertEquals(3, queryRequestEvents.size()); + Assertions.assertEquals(3, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -304,7 +304,7 @@ public void testAdminCancelSuccess() throws Exception { // the response should come back right away ResponseEntity cancelResponse = cancelFuture.get(); - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); // verify that query status was created correctly QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); @@ -324,7 +324,7 @@ public void testAdminCancelSuccess() throws Exception { assertTasksCreated(queryId); // verify that the cancel event was published - Assert.assertEquals(3, queryRequestEvents.size()); + Assertions.assertEquals(3, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -360,10 +360,10 @@ public void testAdminCancelFailure_notAdminUser() throws Exception { // the response should come back right away ResponseEntity closeResponse = closeFuture.get(); - Assert.assertEquals(403, closeResponse.getStatusCodeValue()); + Assertions.assertEquals(403, closeResponse.getStatusCodeValue()); // verify that the create event was published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -401,7 +401,7 @@ public void testAdminCancelAllSuccess() throws Exception { // the response should come back right away ResponseEntity cancelResponse = cancelFuture.get(); - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); // verify that query status was created correctly List queryStatusList = queryStorageCache.getQueryStatus(); @@ -438,7 +438,7 @@ public void testAdminCancelAllSuccess() throws Exception { } // verify that there are no more events - Assert.assertEquals(0, queryRequestEvents.size()); + Assertions.assertEquals(0, queryRequestEvents.size()); } @Test @@ -473,7 +473,7 @@ public void testAdminCancelAllFailure_notAdminUser() throws Exception { // the response should come back right away ResponseEntity cancelResponse = cancelFuture.get(); - Assert.assertEquals(403, cancelResponse.getStatusCodeValue()); + Assertions.assertEquals(403, cancelResponse.getStatusCodeValue()); // verify that query status was created correctly List queryStatusList = queryStorageCache.getQueryStatus(); @@ -496,6 +496,6 @@ public void testAdminCancelAllFailure_notAdminUser() throws Exception { } // verify that there are no more events - Assert.assertEquals(0, queryRequestEvents.size()); + Assertions.assertEquals(0, queryRequestEvents.size()); } } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java index 2af2c944..70f5da52 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java @@ -7,17 +7,17 @@ import datawave.microservice.query.storage.QueryStatus; import datawave.webservice.result.BaseResponse; import datawave.webservice.result.VoidResponse; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.HttpMethod; import org.springframework.http.RequestEntity; import org.springframework.http.ResponseEntity; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.util.UriComponents; @@ -31,16 +31,16 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceCloseTest extends AbstractQueryServiceTest { - @Before + @BeforeEach public void setup() { super.setup(); } - @After + @AfterEach public void teardown() throws Exception { super.teardown(); } @@ -59,7 +59,7 @@ public void testCloseSuccess() throws Exception { // the response should come back right away ResponseEntity closeResponse = closeFuture.get(); - Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + Assertions.assertEquals(200, closeResponse.getStatusCodeValue()); // verify that query status was created correctly QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); @@ -79,7 +79,7 @@ public void testCloseSuccess() throws Exception { assertTasksCreated(queryId); // verify that the close event was published - Assert.assertEquals(2, queryRequestEvents.size()); + Assertions.assertEquals(2, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -123,7 +123,7 @@ public void testCloseSuccess_activeNextCall() throws Exception { // the response should come back right away ResponseEntity closeResponse = closeFuture.get(); - Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + Assertions.assertEquals(200, closeResponse.getStatusCodeValue()); // verify that query status was created correctly QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); @@ -163,7 +163,7 @@ public void testCloseSuccess_activeNextCall() throws Exception { assertTasksCreated(queryId); // verify that the close event was published - Assert.assertEquals(3, queryRequestEvents.size()); + Assertions.assertEquals(3, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -195,7 +195,7 @@ public void testCloseFailure_queryNotFound() throws Exception { // the response should come back right away ResponseEntity closeResponse = closeFuture.get(); - Assert.assertEquals(404, closeResponse.getStatusCodeValue()); + Assertions.assertEquals(404, closeResponse.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -221,7 +221,7 @@ public void testCloseFailure_ownershipFailure() throws Exception { // the response should come back right away ResponseEntity response = future.get(); - Assert.assertEquals(401, response.getStatusCodeValue()); + Assertions.assertEquals(401, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -232,7 +232,7 @@ public void testCloseFailure_ownershipFailure() throws Exception { // @formatter:on // verify that the next events were published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -255,7 +255,7 @@ public void testCloseFailure_queryNotRunning() throws Exception { // the response should come back right away ResponseEntity closeResponse = closeFuture.get(); - Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + Assertions.assertEquals(200, closeResponse.getStatusCodeValue()); // try to close the query again closeFuture = closeQuery(authUser, queryId); @@ -263,7 +263,7 @@ public void testCloseFailure_queryNotRunning() throws Exception { // the response should come back right away closeResponse = closeFuture.get(); - Assert.assertEquals(400, closeResponse.getStatusCodeValue()); + Assertions.assertEquals(400, closeResponse.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -274,7 +274,7 @@ public void testCloseFailure_queryNotRunning() throws Exception { // @formatter:on // verify that the next events were published - Assert.assertEquals(2, queryRequestEvents.size()); + Assertions.assertEquals(2, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -304,7 +304,7 @@ public void testAdminCloseSuccess() throws Exception { // the response should come back right away ResponseEntity closeResponse = closeFuture.get(); - Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + Assertions.assertEquals(200, closeResponse.getStatusCodeValue()); // verify that query status was created correctly QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); @@ -324,7 +324,7 @@ public void testAdminCloseSuccess() throws Exception { assertTasksCreated(queryId); // verify that the close event was published - Assert.assertEquals(2, queryRequestEvents.size()); + Assertions.assertEquals(2, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -356,10 +356,10 @@ public void testAdminCloseFailure_notAdminUser() throws Exception { // the response should come back right away ResponseEntity closeResponse = closeFuture.get(); - Assert.assertEquals(403, closeResponse.getStatusCodeValue()); + Assertions.assertEquals(403, closeResponse.getStatusCodeValue()); // verify that the create event was published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -394,7 +394,7 @@ public void testAdminCloseAllSuccess() throws Exception { // the response should come back right away ResponseEntity closeResponse = closeFuture.get(); - Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + Assertions.assertEquals(200, closeResponse.getStatusCodeValue()); // verify that query status was created correctly List queryStatusList = queryStorageCache.getQueryStatus(); @@ -426,7 +426,7 @@ public void testAdminCloseAllSuccess() throws Exception { } // verify that there are no more events - Assert.assertEquals(0, queryRequestEvents.size()); + Assertions.assertEquals(0, queryRequestEvents.size()); } @Test @@ -461,7 +461,7 @@ public void testAdminCloseAllFailure_notAdminUser() throws Exception { // the response should come back right away ResponseEntity closeResponse = closeFuture.get(); - Assert.assertEquals(403, closeResponse.getStatusCodeValue()); + Assertions.assertEquals(403, closeResponse.getStatusCodeValue()); // verify that query status was created correctly List queryStatusList = queryStorageCache.getQueryStatus(); @@ -484,6 +484,6 @@ public void testAdminCloseAllFailure_notAdminUser() throws Exception { } // verify that there are no more events - Assert.assertEquals(0, queryRequestEvents.size()); + Assertions.assertEquals(0, queryRequestEvents.size()); } } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java index 0a477172..0178cea7 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java @@ -9,18 +9,18 @@ import datawave.webservice.query.exception.QueryExceptionType; import datawave.webservice.result.BaseResponse; import datawave.webservice.result.GenericResponse; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.http.RequestEntity; import org.springframework.http.ResponseEntity; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.util.MultiValueMap; import org.springframework.web.util.UriComponents; @@ -35,16 +35,16 @@ import static datawave.webservice.common.audit.AuditParameters.QUERY_STRING; import static datawave.webservice.query.QueryImpl.BEGIN_DATE; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceCreateTest extends AbstractQueryServiceTest { - @Before + @BeforeEach public void setup() { super.setup(); } - @After + @AfterEach public void teardown() throws Exception { super.teardown(); } @@ -71,10 +71,10 @@ public void testCreateSuccess() throws ParseException, IOException { // verify that a query id was returned String queryId = genericResponse.getResult(); - Assert.assertNotNull(queryId); + Assertions.assertNotNull(queryId); // verify that the create event was published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -142,10 +142,10 @@ public void testCreateFailure_paramValidation() { // @formatter:on // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); + Assertions.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); + Assertions.assertEquals(1, baseResponse.getExceptions().size()); QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off @@ -157,7 +157,7 @@ public void testCreateFailure_paramValidation() { // @formatter:on // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); // verify that no audit message was sent assertAuditNotSent(); @@ -184,10 +184,10 @@ public void testCreateFailure_authValidation() { // @formatter:on // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); + Assertions.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); + Assertions.assertEquals(1, baseResponse.getExceptions().size()); QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off @@ -199,7 +199,7 @@ public void testCreateFailure_authValidation() { // @formatter:on // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); // verify that no audit message was sent assertAuditNotSent(); @@ -229,10 +229,10 @@ public void testCreateFailure_queryLogicValidation() { // @formatter:on // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); + Assertions.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); + Assertions.assertEquals(1, baseResponse.getExceptions().size()); QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off @@ -244,7 +244,7 @@ public void testCreateFailure_queryLogicValidation() { // @formatter:on // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); // verify that no audit message was sent assertAuditNotSent(); @@ -274,10 +274,10 @@ public void testCreateFailure_maxPageSize() { // @formatter:on // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); + Assertions.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); + Assertions.assertEquals(1, baseResponse.getExceptions().size()); QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off @@ -289,7 +289,7 @@ public void testCreateFailure_maxPageSize() { // @formatter:on // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); // verify that no audit message was sent assertAuditNotSent(); @@ -319,10 +319,10 @@ public void testCreateFailure_maxResultsOverride() { // @formatter:on // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); + Assertions.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); + Assertions.assertEquals(1, baseResponse.getExceptions().size()); QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off @@ -334,7 +334,7 @@ public void testCreateFailure_maxResultsOverride() { // @formatter:on // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); // verify that no audit message was sent assertAuditNotSent(); @@ -364,10 +364,10 @@ public void testCreateFailure_maxConcurrentTasksOverride() { // @formatter:on // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); + Assertions.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); + Assertions.assertEquals(1, baseResponse.getExceptions().size()); QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off @@ -379,7 +379,7 @@ public void testCreateFailure_maxConcurrentTasksOverride() { // @formatter:on // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); // verify that no audit message was sent assertAuditNotSent(); @@ -407,10 +407,10 @@ public void testCreateFailure_roleValidation() { // @formatter:on // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); + Assertions.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); + Assertions.assertEquals(1, baseResponse.getExceptions().size()); QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off @@ -422,7 +422,7 @@ public void testCreateFailure_roleValidation() { // @formatter:on // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); // verify that no audit message was sent assertAuditNotSent(); @@ -452,10 +452,10 @@ public void testCreateFailure_markingValidation() { // @formatter:on // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); + Assertions.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); + Assertions.assertEquals(1, baseResponse.getExceptions().size()); QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off @@ -467,7 +467,7 @@ public void testCreateFailure_markingValidation() { // @formatter:on // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); // verify that no audit message was sent assertAuditNotSent(); diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java index 800df689..6ad09cf1 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java @@ -8,18 +8,18 @@ import datawave.webservice.query.exception.QueryExceptionType; import datawave.webservice.result.BaseResponse; import datawave.webservice.result.GenericResponse; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.http.RequestEntity; import org.springframework.http.ResponseEntity; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.util.MultiValueMap; import org.springframework.web.util.UriComponents; @@ -34,16 +34,16 @@ import static datawave.webservice.common.audit.AuditParameters.QUERY_STRING; import static datawave.webservice.query.QueryImpl.BEGIN_DATE; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceDefineTest extends AbstractQueryServiceTest { - @Before + @BeforeEach public void setup() { super.setup(); } - @After + @AfterEach public void teardown() throws Exception { super.teardown(); } @@ -70,7 +70,7 @@ public void testDefineSuccess() throws ParseException, IOException { // verify that a query id was returned String queryId = genericResponse.getResult(); - Assert.assertNotNull(queryId); + Assertions.assertNotNull(queryId); // verify that query status was created correctly QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); @@ -127,10 +127,10 @@ public void testDefineFailure_paramValidation() { // @formatter:on // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); + Assertions.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); + Assertions.assertEquals(1, baseResponse.getExceptions().size()); QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off @@ -142,7 +142,7 @@ public void testDefineFailure_paramValidation() { // @formatter:on // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); } @Test @@ -163,10 +163,10 @@ public void testDefineFailure_authValidation() { // @formatter:on // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); + Assertions.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); + Assertions.assertEquals(1, baseResponse.getExceptions().size()); QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off @@ -178,7 +178,7 @@ public void testDefineFailure_authValidation() { // @formatter:on // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); } @Test @@ -202,10 +202,10 @@ public void testDefineFailure_queryLogicValidation() { // @formatter:on // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); + Assertions.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); + Assertions.assertEquals(1, baseResponse.getExceptions().size()); QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off @@ -217,7 +217,7 @@ public void testDefineFailure_queryLogicValidation() { // @formatter:on // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); } @Test @@ -241,10 +241,10 @@ public void testDefineFailure_maxPageSize() { // @formatter:on // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); + Assertions.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); + Assertions.assertEquals(1, baseResponse.getExceptions().size()); QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off @@ -256,7 +256,7 @@ public void testDefineFailure_maxPageSize() { // @formatter:on // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); } @Test @@ -280,10 +280,10 @@ public void testDefineFailure_maxResultsOverride() { // @formatter:on // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); + Assertions.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); + Assertions.assertEquals(1, baseResponse.getExceptions().size()); QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off @@ -295,7 +295,7 @@ public void testDefineFailure_maxResultsOverride() { // @formatter:on // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); } @Test @@ -319,10 +319,10 @@ public void testDefineFailure_maxConcurrentTasksOverride() { // @formatter:on // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); + Assertions.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); + Assertions.assertEquals(1, baseResponse.getExceptions().size()); QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off @@ -334,7 +334,7 @@ public void testDefineFailure_maxConcurrentTasksOverride() { // @formatter:on // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); } @Test @@ -356,10 +356,10 @@ public void testDefineFailure_roleValidation() { // @formatter:on // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); + Assertions.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); + Assertions.assertEquals(1, baseResponse.getExceptions().size()); QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off @@ -371,7 +371,7 @@ public void testDefineFailure_roleValidation() { // @formatter:on // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); } @Test @@ -395,10 +395,10 @@ public void testDefineFailure_markingValidation() { // @formatter:on // verify that there is no result - Assert.assertFalse(baseResponse.getHasResults()); + Assertions.assertFalse(baseResponse.getHasResults()); // verify that an exception was returned - Assert.assertEquals(1, baseResponse.getExceptions().size()); + Assertions.assertEquals(1, baseResponse.getExceptions().size()); QueryExceptionType queryException = baseResponse.getExceptions().get(0); // @formatter:off @@ -410,6 +410,6 @@ public void testDefineFailure_markingValidation() { // @formatter:on // verify that there are no query statuses - Assert.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); + Assertions.assertTrue(queryStorageCache.getQueryStatus().isEmpty()); } } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java index 4b79d472..8c299497 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java @@ -7,17 +7,17 @@ import datawave.microservice.query.storage.QueryStatus; import datawave.webservice.result.GenericResponse; import datawave.webservice.result.VoidResponse; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.HttpMethod; import org.springframework.http.RequestEntity; import org.springframework.http.ResponseEntity; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.util.UriComponents; @@ -33,16 +33,16 @@ import static datawave.webservice.query.QueryImpl.END_DATE; import static datawave.webservice.query.QueryImpl.QUERY; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceDuplicateTest extends AbstractQueryServiceTest { - @Before + @BeforeEach public void setup() { super.setup(); } - @After + @AfterEach public void teardown() throws Exception { super.teardown(); } @@ -66,7 +66,7 @@ public void testDuplicateSuccess_duplicateOnDefined() throws Exception { // the response should come back right away ResponseEntity response = duplicateFuture.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); String dupeQueryId = (String) response.getBody().getResult(); @@ -98,17 +98,17 @@ public void testDuplicateSuccess_duplicateOnDefined() throws Exception { // @formatter:on // make sure the queries are identical - Assert.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); - Assert.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); - Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), + Assertions.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); + Assertions.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); + Assertions.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); - Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), + Assertions.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); - Assert.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); - Assert.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); + Assertions.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); + Assertions.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); // verify that no events were published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -137,7 +137,7 @@ public void testDuplicateSuccess_duplicateOnCreated() throws Exception { // the response should come back right away ResponseEntity response = duplicateFuture.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); String dupeQueryId = (String) response.getBody().getResult(); @@ -169,17 +169,17 @@ public void testDuplicateSuccess_duplicateOnCreated() throws Exception { // @formatter:on // make sure the queries are identical - Assert.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); - Assert.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); - Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), + Assertions.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); + Assertions.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); + Assertions.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); - Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), + Assertions.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); - Assert.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); - Assert.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); + Assertions.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); + Assertions.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); // verify that no events were published - Assert.assertEquals(2, queryRequestEvents.size()); + Assertions.assertEquals(2, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -208,7 +208,7 @@ public void testDuplicateSuccess_duplicateOnCanceled() throws Exception { // this should return immediately ResponseEntity cancelResponse = cancelFuture.get(); - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); MultiValueMap updateParams = new LinkedMultiValueMap<>(); @@ -221,7 +221,7 @@ public void testDuplicateSuccess_duplicateOnCanceled() throws Exception { // the response should come back right away ResponseEntity response = duplicateFuture.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); String dupeQueryId = (String) response.getBody().getResult(); @@ -253,17 +253,17 @@ public void testDuplicateSuccess_duplicateOnCanceled() throws Exception { // @formatter:on // make sure the queries are identical - Assert.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); - Assert.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); - Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), + Assertions.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); + Assertions.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); + Assertions.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); - Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), + Assertions.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); - Assert.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); - Assert.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); + Assertions.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); + Assertions.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); // verify that no events were published - Assert.assertEquals(4, queryRequestEvents.size()); + Assertions.assertEquals(4, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -302,7 +302,7 @@ public void testDuplicateSuccess_duplicateOnClosed() throws Exception { // this should return immediately ResponseEntity closeResponse = closeFuture.get(); - Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + Assertions.assertEquals(200, closeResponse.getStatusCodeValue()); MultiValueMap updateParams = new LinkedMultiValueMap<>(); @@ -315,7 +315,7 @@ public void testDuplicateSuccess_duplicateOnClosed() throws Exception { // the response should come back right away ResponseEntity response = duplicateFuture.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); String dupeQueryId = (String) response.getBody().getResult(); @@ -347,17 +347,17 @@ public void testDuplicateSuccess_duplicateOnClosed() throws Exception { // @formatter:on // make sure the queries are identical - Assert.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); - Assert.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); - Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), + Assertions.assertEquals(queryStatus.getQuery().getQuery(), dupeQueryStatus.getQuery().getQuery()); + Assertions.assertEquals(queryStatus.getQuery().getQueryAuthorizations(), dupeQueryStatus.getQuery().getQueryAuthorizations()); + Assertions.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate()), DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); - Assert.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), + Assertions.assertEquals(DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate()), DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); - Assert.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); - Assert.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); + Assertions.assertEquals(queryStatus.getQuery().getQueryLogicName(), dupeQueryStatus.getQuery().getQueryLogicName()); + Assertions.assertEquals(queryStatus.getQuery().getPagesize(), dupeQueryStatus.getQuery().getPagesize()); // verify that no events were published - Assert.assertEquals(3, queryRequestEvents.size()); + Assertions.assertEquals(3, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -407,7 +407,7 @@ public void testDuplicateSuccess_update() throws Exception { // the response should come back right away ResponseEntity response = duplicateFuture.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); String dupeQueryId = (String) response.getBody().getResult(); @@ -439,21 +439,21 @@ public void testDuplicateSuccess_update() throws Exception { // @formatter:on // make sure the original query is unchanged - Assert.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); - Assert.assertEquals(TEST_QUERY_AUTHORIZATIONS, queryStatus.getQuery().getQueryAuthorizations()); - Assert.assertEquals(TEST_QUERY_BEGIN, DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate())); - Assert.assertEquals(TEST_QUERY_END, DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate())); - Assert.assertEquals("EventQuery", queryStatus.getQuery().getQueryLogicName()); + Assertions.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); + Assertions.assertEquals(TEST_QUERY_AUTHORIZATIONS, queryStatus.getQuery().getQueryAuthorizations()); + Assertions.assertEquals(TEST_QUERY_BEGIN, DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate())); + Assertions.assertEquals(TEST_QUERY_END, DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate())); + Assertions.assertEquals("EventQuery", queryStatus.getQuery().getQueryLogicName()); // make sure the duplicated query is updated - Assert.assertEquals(newQuery, dupeQueryStatus.getQuery().getQuery()); - Assert.assertEquals(newAuths, dupeQueryStatus.getQuery().getQueryAuthorizations()); - Assert.assertEquals(newBegin, DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); - Assert.assertEquals(newEnd, DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); - Assert.assertEquals(newLogic, dupeQueryStatus.getQuery().getQueryLogicName()); + Assertions.assertEquals(newQuery, dupeQueryStatus.getQuery().getQuery()); + Assertions.assertEquals(newAuths, dupeQueryStatus.getQuery().getQueryAuthorizations()); + Assertions.assertEquals(newBegin, DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getBeginDate())); + Assertions.assertEquals(newEnd, DefaultQueryParameters.formatDate(dupeQueryStatus.getQuery().getEndDate())); + Assertions.assertEquals(newLogic, dupeQueryStatus.getQuery().getQueryLogicName()); // verify that no events were published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -490,13 +490,13 @@ public void testDuplicateFailure_invalidUpdate() throws Exception { // the response should come back right away ResponseEntity response = duplicateFuture.get(); - Assert.assertEquals(400, response.getStatusCodeValue()); + Assertions.assertEquals(400, response.getStatusCodeValue()); // make sure an audit message wasn't sent assertAuditNotSent(); // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); + Assertions.assertEquals(0, queryRequestEvents.size()); } @Test @@ -522,7 +522,7 @@ public void testDuplicateFailure_queryNotFound() throws Exception { // the response should come back right away ResponseEntity response = duplicateFuture.get(); - Assert.assertEquals(404, response.getStatusCodeValue()); + Assertions.assertEquals(404, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -536,7 +536,7 @@ public void testDuplicateFailure_queryNotFound() throws Exception { assertAuditNotSent(); // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); + Assertions.assertEquals(0, queryRequestEvents.size()); } @Test @@ -564,7 +564,7 @@ public void testDuplicateFailure_ownershipFailure() throws Exception { // the response should come back right away ResponseEntity response = duplicateFuture.get(); - Assert.assertEquals(401, response.getStatusCodeValue()); + Assertions.assertEquals(401, response.getStatusCodeValue()); // make sure an audit message wasn't sent assertAuditNotSent(); @@ -580,9 +580,9 @@ public void testDuplicateFailure_ownershipFailure() throws Exception { QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); // make sure the query was not updated - Assert.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); + Assertions.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); + Assertions.assertEquals(0, queryRequestEvents.size()); } } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java index af5bd486..b69abf6f 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java @@ -7,17 +7,17 @@ import datawave.webservice.query.result.logic.QueryLogicDescription; import datawave.webservice.result.QueryImplListResponse; import datawave.webservice.result.QueryLogicResponse; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.HttpMethod; import org.springframework.http.RequestEntity; import org.springframework.http.ResponseEntity; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.util.MultiValueMap; import org.springframework.web.util.UriComponents; import org.springframework.web.util.UriComponentsBuilder; @@ -33,16 +33,16 @@ import static datawave.microservice.query.QueryParameters.QUERY_NAME; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceListTest extends AbstractQueryServiceTest { - @Before + @BeforeEach public void setup() { super.setup(); } - @After + @AfterEach public void teardown() throws Exception { super.teardown(); } @@ -76,18 +76,18 @@ public void testListSuccess() throws Exception { // this should return immediately ResponseEntity listResponse = listFuture.get(); - Assert.assertEquals(200, listResponse.getStatusCodeValue()); + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); QueryImplListResponse result = listResponse.getBody(); - Assert.assertEquals(5, result.getNumResults()); + Assertions.assertEquals(5, result.getNumResults()); List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); Collections.sort(queryIds); Collections.sort(actualQueryIds); - Assert.assertEquals(queryIds, actualQueryIds); + Assertions.assertEquals(queryIds, actualQueryIds); } @Test @@ -109,15 +109,15 @@ public void testListSuccess_filterOnQueryId() throws Exception { // this should return immediately ResponseEntity listResponse = listFuture.get(); - Assert.assertEquals(200, listResponse.getStatusCodeValue()); + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); QueryImplListResponse result = listResponse.getBody(); - Assert.assertEquals(1, result.getNumResults()); + Assertions.assertEquals(1, result.getNumResults()); List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); - Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + Assertions.assertEquals(queryIds.get(0), actualQueryIds.get(0)); } @Test @@ -146,15 +146,15 @@ public void testListSuccess_filterOnQueryName() throws Exception { // this should return immediately ResponseEntity listResponse = listFuture.get(); - Assert.assertEquals(200, listResponse.getStatusCodeValue()); + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); QueryImplListResponse result = listResponse.getBody(); - Assert.assertEquals(1, result.getNumResults()); + Assertions.assertEquals(1, result.getNumResults()); List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); - Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + Assertions.assertEquals(queryIds.get(0), actualQueryIds.get(0)); } @Test @@ -183,11 +183,11 @@ public void testListSuccess_filterOnMultiple() throws Exception { // this should return immediately ResponseEntity listResponse = listFuture.get(); - Assert.assertEquals(200, listResponse.getStatusCodeValue()); + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); QueryImplListResponse result = listResponse.getBody(); - Assert.assertEquals(0, result.getNumResults()); + Assertions.assertEquals(0, result.getNumResults()); // list queries with just the query name and a bogus ID listFuture = listQueries(authUser, UUID.randomUUID().toString(), uniqueQueryName); @@ -195,11 +195,11 @@ public void testListSuccess_filterOnMultiple() throws Exception { // this should return immediately listResponse = listFuture.get(); - Assert.assertEquals(200, listResponse.getStatusCodeValue()); + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); result = listResponse.getBody(); - Assert.assertEquals(0, result.getNumResults()); + Assertions.assertEquals(0, result.getNumResults()); // list queries with just the query name and a bogus ID listFuture = listQueries(authUser, queryIds.get(0), uniqueQueryName); @@ -207,15 +207,15 @@ public void testListSuccess_filterOnMultiple() throws Exception { // this should return immediately listResponse = listFuture.get(); - Assert.assertEquals(200, listResponse.getStatusCodeValue()); + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); result = listResponse.getBody(); - Assert.assertEquals(1, result.getNumResults()); + Assertions.assertEquals(1, result.getNumResults()); List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); - Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + Assertions.assertEquals(queryIds.get(0), actualQueryIds.get(0)); } @Test @@ -245,11 +245,11 @@ public void testListFailure_ownershipFailure() throws Exception { // this should return immediately ResponseEntity listResponse = listFuture.get(); - Assert.assertEquals(200, listResponse.getStatusCodeValue()); + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); QueryImplListResponse result = listResponse.getBody(); - Assert.assertEquals(0, result.getNumResults()); + Assertions.assertEquals(0, result.getNumResults()); // list queries with just the query name and a bogus ID listFuture = listQueries(altAuthUser, UUID.randomUUID().toString(), uniqueQueryName); @@ -257,11 +257,11 @@ public void testListFailure_ownershipFailure() throws Exception { // this should return immediately listResponse = listFuture.get(); - Assert.assertEquals(200, listResponse.getStatusCodeValue()); + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); result = listResponse.getBody(); - Assert.assertEquals(0, result.getNumResults()); + Assertions.assertEquals(0, result.getNumResults()); // list queries with the query name and query ID listFuture = listQueries(altAuthUser, queryIds.get(0), uniqueQueryName); @@ -269,11 +269,11 @@ public void testListFailure_ownershipFailure() throws Exception { // this should return immediately listResponse = listFuture.get(); - Assert.assertEquals(200, listResponse.getStatusCodeValue()); + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); result = listResponse.getBody(); - Assert.assertEquals(0, result.getNumResults()); + Assertions.assertEquals(0, result.getNumResults()); } @Test @@ -305,15 +305,15 @@ public void testAdminListSuccess() throws Exception { // this should return immediately ResponseEntity listResponse = listFuture.get(); - Assert.assertEquals(200, listResponse.getStatusCodeValue()); + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); QueryImplListResponse result = listResponse.getBody(); - Assert.assertEquals(1, result.getNumResults()); + Assertions.assertEquals(1, result.getNumResults()); List actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); - Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + Assertions.assertEquals(queryIds.get(0), actualQueryIds.get(0)); // list queries with just the query name listFuture = adminListQueries(adminUser, null, user, uniqueQueryName); @@ -321,15 +321,15 @@ public void testAdminListSuccess() throws Exception { // this should return immediately listResponse = listFuture.get(); - Assert.assertEquals(200, listResponse.getStatusCodeValue()); + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); result = listResponse.getBody(); - Assert.assertEquals(1, result.getNumResults()); + Assertions.assertEquals(1, result.getNumResults()); actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); - Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + Assertions.assertEquals(queryIds.get(0), actualQueryIds.get(0)); // list queries with the query name and query ID listFuture = adminListQueries(adminUser, queryIds.get(0), user, uniqueQueryName); @@ -337,15 +337,15 @@ public void testAdminListSuccess() throws Exception { // this should return immediately listResponse = listFuture.get(); - Assert.assertEquals(200, listResponse.getStatusCodeValue()); + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); result = listResponse.getBody(); - Assert.assertEquals(1, result.getNumResults()); + Assertions.assertEquals(1, result.getNumResults()); actualQueryIds = result.getQuery().stream().map(Query::getId).map(UUID::toString).collect(Collectors.toList()); - Assert.assertEquals(queryIds.get(0), actualQueryIds.get(0)); + Assertions.assertEquals(queryIds.get(0), actualQueryIds.get(0)); } @Test @@ -381,7 +381,7 @@ public void testAdminListFailure_notAdminUser() throws Exception { ResponseEntity listResponse = listFuture.get(); - Assert.assertEquals(403, listResponse.getStatusCodeValue()); + Assertions.assertEquals(403, listResponse.getStatusCodeValue()); } @Test @@ -398,13 +398,13 @@ public void testGetQuerySuccess() throws Exception { // this should return immediately ResponseEntity listResponse = listFuture.get(); - Assert.assertEquals(200, listResponse.getStatusCodeValue()); + Assertions.assertEquals(200, listResponse.getStatusCodeValue()); QueryImplListResponse result = listResponse.getBody(); - Assert.assertEquals(1, result.getNumResults()); + Assertions.assertEquals(1, result.getNumResults()); - Assert.assertEquals(queryId, result.getQuery().get(0).getId().toString()); + Assertions.assertEquals(queryId, result.getQuery().get(0).getId().toString()); } @Test @@ -415,7 +415,7 @@ public void testListQueryLogicSuccess() throws Exception { ResponseEntity response = future.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); QueryLogicResponse qlResponse = response.getBody(); @@ -423,12 +423,12 @@ public void testListQueryLogicSuccess() throws Exception { "ErrorCountQuery", "ErrorDiscoveryQuery", "ErrorEventQuery", "ErrorFieldIndexCountQuery", "EventQuery", "FacetedQuery", "FieldIndexCountQuery", "HitHighlights", "IndexStatsQuery", "LuceneUUIDEventQuery", "QueryMetricsQuery", "InternalQueryMetricsQuery", "TermFrequencyQuery"}; - Assert.assertEquals(expectedQueryLogics.length, qlResponse.getQueryLogicList().size()); + Assertions.assertEquals(expectedQueryLogics.length, qlResponse.getQueryLogicList().size()); List qlNames = qlResponse.getQueryLogicList().stream().map(QueryLogicDescription::getName).sorted().collect(Collectors.toList()); qlNames.removeAll(Arrays.asList(expectedQueryLogics)); - Assert.assertTrue(qlNames.isEmpty()); + Assertions.assertTrue(qlNames.isEmpty()); } } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java index dfbb790d..5f727fe5 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java @@ -10,16 +10,16 @@ import datawave.webservice.result.BaseResponse; import datawave.webservice.result.DefaultEventQueryResponse; import datawave.webservice.result.VoidResponse; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.ResponseEntity; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; @@ -29,16 +29,16 @@ import java.util.UUID; import java.util.concurrent.Future; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceNextTest extends AbstractQueryServiceTest { - @Before + @BeforeEach public void setup() { super.setup(); } - @After + @AfterEach public void teardown() throws Exception { super.teardown(); } @@ -72,12 +72,12 @@ public void testNextSuccess() throws Exception { // the response should come back right away ResponseEntity response = future.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); // verify some headers - Assert.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + Assertions.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); @@ -105,7 +105,7 @@ public void testNextSuccess() throws Exception { // @formatter:on // verify that the next event was published - Assert.assertEquals(2, queryRequestEvents.size()); + Assertions.assertEquals(2, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -136,7 +136,7 @@ public void testNextSuccess_multiplePages() throws Exception { fieldValues.add("LOKI", "CLASSIC"); // verify that the create event was published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -161,12 +161,12 @@ public void testNextSuccess_multiplePages() throws Exception { // the response should come back right away ResponseEntity response = future.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); // verify some headers - Assert.assertEquals(Integer.toString(page), Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + Assertions.assertEquals(Integer.toString(page), Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); @@ -194,7 +194,7 @@ public void testNextSuccess_multiplePages() throws Exception { // @formatter:on // verify that the next event was published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -244,17 +244,17 @@ public void testNextSuccess_cancelPartialResults() throws Exception { // the response should come back right away ResponseEntity cancelResponse = cancelFuture.get(); - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); // the response should come back right away ResponseEntity nextResponse = nextFuture.get(); - Assert.assertEquals(200, nextResponse.getStatusCodeValue()); + Assertions.assertEquals(200, nextResponse.getStatusCodeValue()); // verify some headers - Assert.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-query-page-number")))); - Assert.assertEquals("true", Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-Partial-Results")))); - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-query-last-page")))); + Assertions.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-query-page-number")))); + Assertions.assertEquals("true", Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-Partial-Results")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(nextResponse.getHeaders().get("X-query-last-page")))); DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) nextResponse.getBody(); @@ -282,7 +282,7 @@ public void testNextSuccess_cancelPartialResults() throws Exception { // @formatter:on // verify that the next events were published - Assert.assertEquals(4, queryRequestEvents.size()); + Assertions.assertEquals(4, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -323,7 +323,7 @@ public void testNextSuccess_maxResults() throws Exception { fieldValues.add("LOKI", "CLASSIC"); // verify that the create event was published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -349,16 +349,17 @@ public void testNextSuccess_maxResults() throws Exception { ResponseEntity response = future.get(); if (page != 4) { - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); } else { - Assert.assertEquals(204, response.getStatusCodeValue()); + Assertions.assertEquals(204, response.getStatusCodeValue()); } if (page != 4) { // verify some headers - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); - Assert.assertEquals(Integer.toString(page), Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assertions.assertEquals(Integer.toString(page), + Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); @@ -386,7 +387,7 @@ public void testNextSuccess_maxResults() throws Exception { // @formatter:on // verify that the next event was published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -395,10 +396,10 @@ public void testNextSuccess_maxResults() throws Exception { queryRequestEvents.removeLast()); // @formatter:on } else { - Assert.assertNull(response.getBody()); + Assertions.assertNull(response.getBody()); // verify that the next and close events were published - Assert.assertEquals(2, queryRequestEvents.size()); + Assertions.assertEquals(2, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -422,7 +423,7 @@ public void testNextSuccess_maxResults() throws Exception { // the response should come back right away ResponseEntity response = future.get(); - Assert.assertEquals(400, response.getStatusCodeValue()); + Assertions.assertEquals(400, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -454,11 +455,11 @@ public void testNextSuccess_noResults() throws Exception { // the response should come back right away ResponseEntity response = future.get(); - Assert.assertEquals(204, response.getStatusCodeValue()); - Assert.assertNull(response.getBody()); + Assertions.assertEquals(204, response.getStatusCodeValue()); + Assertions.assertNull(response.getBody()); // verify that the next event was published - Assert.assertEquals(3, queryRequestEvents.size()); + Assertions.assertEquals(3, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -490,7 +491,7 @@ public void testNextFailure_queryNotFound() throws Exception { // the response should come back right away ResponseEntity response = future.get(); - Assert.assertEquals(404, response.getStatusCodeValue()); + Assertions.assertEquals(404, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -501,7 +502,7 @@ public void testNextFailure_queryNotFound() throws Exception { // @formatter:on // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); + Assertions.assertEquals(0, queryRequestEvents.size()); } @Test @@ -517,7 +518,7 @@ public void testNextFailure_queryNotRunning() throws Exception { // the response should come back right away ResponseEntity cancelResponse = cancelFuture.get(); - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); // make the next call asynchronously Future> future = nextQuery(authUser, queryId); @@ -525,7 +526,7 @@ public void testNextFailure_queryNotRunning() throws Exception { // the response should come back right away ResponseEntity response = future.get(); - Assert.assertEquals(400, response.getStatusCodeValue()); + Assertions.assertEquals(400, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -536,7 +537,7 @@ public void testNextFailure_queryNotRunning() throws Exception { // @formatter:on // verify that the next events were published - Assert.assertEquals(3, queryRequestEvents.size()); + Assertions.assertEquals(3, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -570,7 +571,7 @@ public void testNextFailure_ownershipFailure() throws Exception { // the response should come back right away ResponseEntity response = future.get(); - Assert.assertEquals(401, response.getStatusCodeValue()); + Assertions.assertEquals(401, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -581,7 +582,7 @@ public void testNextFailure_ownershipFailure() throws Exception { // @formatter:on // verify that the next events were published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -608,7 +609,7 @@ public void testNextFailure_timeout() throws Exception { // the response should come back after the configured timeout (5 seconds) ResponseEntity response = future.get(); - Assert.assertEquals(500, response.getStatusCodeValue()); + Assertions.assertEquals(500, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -619,7 +620,7 @@ public void testNextFailure_timeout() throws Exception { // @formatter:on // verify that the next events were published - Assert.assertEquals(2, queryRequestEvents.size()); + Assertions.assertEquals(2, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -647,7 +648,7 @@ public void testNextFailure_nextOnDefined() throws Exception { // the response should come back right away ResponseEntity response = future.get(); - Assert.assertEquals(400, response.getStatusCodeValue()); + Assertions.assertEquals(400, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -658,7 +659,7 @@ public void testNextFailure_nextOnDefined() throws Exception { // @formatter:on // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); + Assertions.assertEquals(0, queryRequestEvents.size()); } @Test @@ -674,7 +675,7 @@ public void testNextFailure_nextOnClosed() throws Exception { // the response should come back right away ResponseEntity closeResponse = closeFuture.get(); - Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + Assertions.assertEquals(200, closeResponse.getStatusCodeValue()); // make the next call asynchronously Future> future = nextQuery(authUser, queryId); @@ -682,7 +683,7 @@ public void testNextFailure_nextOnClosed() throws Exception { // the response should come back right away ResponseEntity response = future.get(); - Assert.assertEquals(400, response.getStatusCodeValue()); + Assertions.assertEquals(400, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -693,7 +694,7 @@ public void testNextFailure_nextOnClosed() throws Exception { // @formatter:on // verify that no events were published - Assert.assertEquals(2, queryRequestEvents.size()); + Assertions.assertEquals(2, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -721,7 +722,7 @@ public void testNextFailure_nextOnCanceled() throws Exception { // the response should come back right away ResponseEntity cancelResponse = cancelFuture.get(); - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); // make the next call asynchronously Future> future = nextQuery(authUser, queryId); @@ -729,7 +730,7 @@ public void testNextFailure_nextOnCanceled() throws Exception { // the response should come back right away ResponseEntity response = future.get(); - Assert.assertEquals(400, response.getStatusCodeValue()); + Assertions.assertEquals(400, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -740,7 +741,7 @@ public void testNextFailure_nextOnCanceled() throws Exception { // @formatter:on // verify that the cancel event was published - Assert.assertEquals(3, queryRequestEvents.size()); + Assertions.assertEquals(3, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java index 39dc9433..cefd6461 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java @@ -5,11 +5,11 @@ import datawave.microservice.query.remote.QueryRequest; import datawave.microservice.query.storage.QueryStatus; import datawave.webservice.result.GenericResponse; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.cloud.bus.event.RemoteQueryRequestEvent; @@ -19,7 +19,7 @@ import org.springframework.http.RequestEntity; import org.springframework.http.ResponseEntity; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.util.MultiValueMap; import org.springframework.web.util.UriComponents; @@ -30,7 +30,7 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServicePlanTest extends AbstractQueryServiceTest { @@ -38,12 +38,12 @@ public class QueryServicePlanTest extends AbstractQueryServiceTest { @Autowired public ApplicationEventPublisher eventPublisher; - @Before + @BeforeEach public void setup() { super.setup(); } - @After + @AfterEach public void teardown() throws Exception { super.teardown(); } @@ -69,11 +69,11 @@ public void testPlanSuccess() throws ParseException, IOException, ExecutionExcep } // verify that the plan event was published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); RemoteQueryRequestEvent requestEvent = queryRequestEvents.removeLast(); - Assert.assertEquals("executor-unassigned:**", requestEvent.getDestinationService()); - Assert.assertEquals(QueryRequest.Method.PLAN, requestEvent.getRequest().getMethod()); + Assertions.assertEquals("executor-unassigned:**", requestEvent.getDestinationService()); + Assertions.assertEquals(QueryRequest.Method.PLAN, requestEvent.getRequest().getMethod()); String queryId = requestEvent.getRequest().getQueryId(); String plan = "some plan"; @@ -96,7 +96,7 @@ public void testPlanSuccess() throws ParseException, IOException, ExecutionExcep // @formatter:on String receivedPlan = genericResponse.getResult(); - Assert.assertEquals(receivedPlan, plan); + Assertions.assertEquals(receivedPlan, plan); // @formatter:off assertQueryRequestEvent( @@ -108,6 +108,6 @@ public void testPlanSuccess() throws ParseException, IOException, ExecutionExcep // verify that the query status was deleted queryStatus = queryStorageCache.getQueryStatus(queryId); - Assert.assertNull(queryStatus); + Assertions.assertNull(queryStatus); } } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java index fbb1c396..efaa63c4 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java @@ -7,17 +7,17 @@ import datawave.microservice.query.storage.QueryStatus; import datawave.webservice.result.BaseResponse; import datawave.webservice.result.VoidResponse; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.HttpMethod; import org.springframework.http.RequestEntity; import org.springframework.http.ResponseEntity; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.web.util.UriComponents; import java.util.Arrays; @@ -26,16 +26,16 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceRemoveTest extends AbstractQueryServiceTest { - @Before + @BeforeEach public void setup() { super.setup(); } - @After + @AfterEach public void teardown() throws Exception { super.teardown(); } @@ -53,15 +53,15 @@ public void testRemoveSuccess_removeOnDefined() throws Exception { // the response should come back right away ResponseEntity response = removeFuture.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); // verify that original query was removed QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - Assert.assertNull(queryStatus); + Assertions.assertNull(queryStatus); // verify that events were published - Assert.assertEquals(0, queryRequestEvents.size()); + Assertions.assertEquals(0, queryRequestEvents.size()); } @Test @@ -77,7 +77,7 @@ public void testRemoveSuccess_removeOnClosed() throws Exception { // the response should come back right away ResponseEntity closeResponse = closeFuture.get(); - Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + Assertions.assertEquals(200, closeResponse.getStatusCodeValue()); // remove the query Future> removeFuture = removeQuery(authUser, queryId); @@ -85,15 +85,15 @@ public void testRemoveSuccess_removeOnClosed() throws Exception { // the response should come back right away ResponseEntity response = removeFuture.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); // verify that original query was removed QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - Assert.assertNull(queryStatus); + Assertions.assertNull(queryStatus); // verify that events were published - Assert.assertEquals(2, queryRequestEvents.size()); + Assertions.assertEquals(2, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -121,7 +121,7 @@ public void testRemoveSuccess_removeOnCanceled() throws Exception { // the response should come back right away ResponseEntity cancelResponse = cancelFuture.get(); - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); // remove the query Future> removeFuture = removeQuery(authUser, queryId); @@ -129,15 +129,15 @@ public void testRemoveSuccess_removeOnCanceled() throws Exception { // the response should come back right away ResponseEntity response = removeFuture.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); // verify that original query was removed QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - Assert.assertNull(queryStatus); + Assertions.assertNull(queryStatus); // verify that events were published - Assert.assertEquals(3, queryRequestEvents.size()); + Assertions.assertEquals(3, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -170,17 +170,17 @@ public void testRemoveFailure_removeOnCreated() throws Exception { // the response should come back right away ResponseEntity response = removeFuture.get(); - Assert.assertEquals(400, response.getStatusCodeValue()); + Assertions.assertEquals(400, response.getStatusCodeValue()); - Assert.assertEquals("Cannot remove a running query.", Iterables.getOnlyElement(response.getBody().getExceptions()).getMessage()); + Assertions.assertEquals("Cannot remove a running query.", Iterables.getOnlyElement(response.getBody().getExceptions()).getMessage()); // verify that original query was not removed QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - Assert.assertNotNull(queryStatus); + Assertions.assertNotNull(queryStatus); // verify that events were published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -206,7 +206,7 @@ public void testRemoveFailure_removeOnClosedActiveNext() throws Exception { // the response should come back right away ResponseEntity closeResponse = closeFuture.get(); - Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + Assertions.assertEquals(200, closeResponse.getStatusCodeValue()); // remove the query Future> removeFuture = removeQuery(authUser, queryId); @@ -214,17 +214,17 @@ public void testRemoveFailure_removeOnClosedActiveNext() throws Exception { // the response should come back right away ResponseEntity response = removeFuture.get(); - Assert.assertEquals(400, response.getStatusCodeValue()); + Assertions.assertEquals(400, response.getStatusCodeValue()); - Assert.assertEquals("Cannot remove a running query.", Iterables.getOnlyElement(response.getBody().getExceptions()).getMessage()); + Assertions.assertEquals("Cannot remove a running query.", Iterables.getOnlyElement(response.getBody().getExceptions()).getMessage()); // verify that original query was not removed QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - Assert.assertNotNull(queryStatus); + Assertions.assertNotNull(queryStatus); // verify that events were published - Assert.assertEquals(3, queryRequestEvents.size()); + Assertions.assertEquals(3, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -261,7 +261,7 @@ public void testRemoveFailure_queryNotFound() throws Exception { // the response should come back right away ResponseEntity response = resetFuture.get(); - Assert.assertEquals(404, response.getStatusCodeValue()); + Assertions.assertEquals(404, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -272,7 +272,7 @@ public void testRemoveFailure_queryNotFound() throws Exception { // @formatter:on // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); + Assertions.assertEquals(0, queryRequestEvents.size()); } @Test @@ -294,7 +294,7 @@ public void testRemoveFailure_ownershipFailure() throws Exception { // the response should come back right away ResponseEntity response = resetFuture.get(); - Assert.assertEquals(401, response.getStatusCodeValue()); + Assertions.assertEquals(401, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -305,7 +305,7 @@ public void testRemoveFailure_ownershipFailure() throws Exception { // @formatter:on // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); + Assertions.assertEquals(0, queryRequestEvents.size()); } @Test @@ -322,15 +322,15 @@ public void testAdminRemoveSuccess() throws Exception { // the response should come back right away ResponseEntity response = removeFuture.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); // verify that original query was removed QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - Assert.assertNull(queryStatus); + Assertions.assertNull(queryStatus); // verify that events were published - Assert.assertEquals(0, queryRequestEvents.size()); + Assertions.assertEquals(0, queryRequestEvents.size()); } @Test @@ -350,15 +350,15 @@ public void testAdminRemoveFailure_notAdminUser() throws Exception { // the response should come back right away ResponseEntity response = removeFuture.get(); - Assert.assertEquals(403, response.getStatusCodeValue()); + Assertions.assertEquals(403, response.getStatusCodeValue()); // verify that original query was not removed QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); - Assert.assertNotNull(queryStatus); + Assertions.assertNotNull(queryStatus); // verify that events were published - Assert.assertEquals(0, queryRequestEvents.size()); + Assertions.assertEquals(0, queryRequestEvents.size()); } @Test @@ -376,15 +376,15 @@ public void testAdminRemoveAllSuccess() throws Exception { // the response should come back right away ResponseEntity removeResponse = removeFuture.get(); - Assert.assertEquals(200, removeResponse.getStatusCodeValue()); + Assertions.assertEquals(200, removeResponse.getStatusCodeValue()); // verify that query status was created correctly List queryStatusList = queryStorageCache.getQueryStatus(); - Assert.assertEquals(0, queryStatusList.size()); + Assertions.assertEquals(0, queryStatusList.size()); // verify that there are no events - Assert.assertEquals(0, queryRequestEvents.size()); + Assertions.assertEquals(0, queryRequestEvents.size()); } @Test @@ -405,14 +405,14 @@ public void testAdminRemoveAllFailure_notAdminUser() throws Exception { // the response should come back right away ResponseEntity removeResponse = removeFuture.get(); - Assert.assertEquals(403, removeResponse.getStatusCodeValue()); + Assertions.assertEquals(403, removeResponse.getStatusCodeValue()); // verify that query status was created correctly List queryStatusList = queryStorageCache.getQueryStatus(); - Assert.assertEquals(10, queryStatusList.size()); + Assertions.assertEquals(10, queryStatusList.size()); // verify that there are no events - Assert.assertEquals(0, queryRequestEvents.size()); + Assertions.assertEquals(0, queryRequestEvents.size()); } } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java index 44034255..3b832a3e 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java @@ -8,34 +8,34 @@ import datawave.webservice.result.BaseResponse; import datawave.webservice.result.GenericResponse; import datawave.webservice.result.VoidResponse; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.http.RequestEntity; import org.springframework.http.ResponseEntity; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.web.util.UriComponents; import java.util.UUID; import java.util.concurrent.Executors; import java.util.concurrent.Future; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceResetTest extends AbstractQueryServiceTest { - @Before + @BeforeEach public void setup() { super.setup(); } - @After + @AfterEach public void teardown() throws Exception { super.teardown(); } @@ -57,7 +57,7 @@ public void testResetSuccess_resetOnDefined() throws Exception { // the response should come back right away ResponseEntity response = resetFuture.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); // @formatter:off assertGenericResponse( @@ -69,7 +69,7 @@ public void testResetSuccess_resetOnDefined() throws Exception { String resetQueryId = (String) response.getBody().getResult(); // verify that a new query id was created - Assert.assertNotEquals(queryId, resetQueryId); + Assertions.assertNotEquals(queryId, resetQueryId); // verify that an audit record was sent assertAuditSent(resetQueryId); @@ -104,10 +104,10 @@ public void testResetSuccess_resetOnDefined() throws Exception { // make sure the queries are equal (ignoring the query id) queryStatus.getQuery().setId(resetQueryStatus.getQuery().getId()); - Assert.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); + Assertions.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); // verify that events were published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -134,7 +134,7 @@ public void testResetSuccess_resetOnCreated() throws Exception { // the response should come back right away ResponseEntity response = resetFuture.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); // @formatter:off assertGenericResponse( @@ -146,7 +146,7 @@ public void testResetSuccess_resetOnCreated() throws Exception { String resetQueryId = (String) response.getBody().getResult(); // verify that a new query id was created - Assert.assertNotEquals(queryId, resetQueryId); + Assertions.assertNotEquals(queryId, resetQueryId); // verify that an audit record was sent assertAuditSent(resetQueryId); @@ -181,10 +181,10 @@ public void testResetSuccess_resetOnCreated() throws Exception { // make sure the queries are equal (ignoring the query id) queryStatus.getQuery().setId(resetQueryStatus.getQuery().getId()); - Assert.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); + Assertions.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); // verify that events were published - Assert.assertEquals(4, queryRequestEvents.size()); + Assertions.assertEquals(4, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -223,7 +223,7 @@ public void testResetSuccess_resetOnClosed() throws Exception { // the response should come back right away ResponseEntity closeResponse = closeFuture.get(); - Assert.assertEquals(200, closeResponse.getStatusCodeValue()); + Assertions.assertEquals(200, closeResponse.getStatusCodeValue()); mockServer.reset(); auditSentSetup(); @@ -234,7 +234,7 @@ public void testResetSuccess_resetOnClosed() throws Exception { // the response should come back right away ResponseEntity response = resetFuture.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); // @formatter:off assertGenericResponse( @@ -246,7 +246,7 @@ public void testResetSuccess_resetOnClosed() throws Exception { String resetQueryId = (String) response.getBody().getResult(); // verify that a new query id was created - Assert.assertNotEquals(queryId, resetQueryId); + Assertions.assertNotEquals(queryId, resetQueryId); // verify that an audit record was sent assertAuditSent(resetQueryId); @@ -281,10 +281,10 @@ public void testResetSuccess_resetOnClosed() throws Exception { // make sure the queries are equal (ignoring the query id) queryStatus.getQuery().setId(resetQueryStatus.getQuery().getId()); - Assert.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); + Assertions.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); // verify that events were published - Assert.assertEquals(3, queryRequestEvents.size()); + Assertions.assertEquals(3, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -318,7 +318,7 @@ public void testResetSuccess_resetOnCanceled() throws Exception { // the response should come back right away ResponseEntity cancelResponse = cancelFuture.get(); - Assert.assertEquals(200, cancelResponse.getStatusCodeValue()); + Assertions.assertEquals(200, cancelResponse.getStatusCodeValue()); mockServer.reset(); auditSentSetup(); @@ -329,7 +329,7 @@ public void testResetSuccess_resetOnCanceled() throws Exception { // the response should come back right away ResponseEntity response = resetFuture.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); // @formatter:off assertGenericResponse( @@ -341,7 +341,7 @@ public void testResetSuccess_resetOnCanceled() throws Exception { String resetQueryId = (String) response.getBody().getResult(); // verify that a new query id was created - Assert.assertNotEquals(queryId, resetQueryId); + Assertions.assertNotEquals(queryId, resetQueryId); // verify that an audit record was sent assertAuditSent(resetQueryId); @@ -376,10 +376,10 @@ public void testResetSuccess_resetOnCanceled() throws Exception { // make sure the queries are equal (ignoring the query id) queryStatus.getQuery().setId(resetQueryStatus.getQuery().getId()); - Assert.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); + Assertions.assertEquals(queryStatus.getQuery(), resetQueryStatus.getQuery()); // verify that events were published - Assert.assertEquals(4, queryRequestEvents.size()); + Assertions.assertEquals(4, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -423,7 +423,7 @@ public void testResetFailure_queryNotFound() throws Exception { // the response should come back right away ResponseEntity response = resetFuture.get(); - Assert.assertEquals(404, response.getStatusCodeValue()); + Assertions.assertEquals(404, response.getStatusCodeValue()); // make sure no audits were sent assertAuditNotSent(); @@ -437,7 +437,7 @@ public void testResetFailure_queryNotFound() throws Exception { // @formatter:on // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); + Assertions.assertEquals(0, queryRequestEvents.size()); } @Test @@ -462,7 +462,7 @@ public void testResetFailure_ownershipFailure() throws Exception { // the response should come back right away ResponseEntity response = resetFuture.get(); - Assert.assertEquals(401, response.getStatusCodeValue()); + Assertions.assertEquals(401, response.getStatusCodeValue()); // make sure no audits were sent assertAuditNotSent(); @@ -476,7 +476,7 @@ public void testResetFailure_ownershipFailure() throws Exception { // @formatter:on // verify that no events were published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java index 927daffa..39b55a6d 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java @@ -7,17 +7,17 @@ import datawave.microservice.query.storage.QueryStatus; import datawave.webservice.result.GenericResponse; import datawave.webservice.result.VoidResponse; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.HttpMethod; import org.springframework.http.RequestEntity; import org.springframework.http.ResponseEntity; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.util.UriComponents; @@ -35,16 +35,16 @@ import static datawave.webservice.query.QueryImpl.PAGESIZE; import static datawave.webservice.query.QueryImpl.QUERY; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class QueryServiceUpdateTest extends AbstractQueryServiceTest { - @Before + @BeforeEach public void setup() { super.setup(); } - @After + @AfterEach public void teardown() throws Exception { super.teardown(); } @@ -77,20 +77,20 @@ public void testUpdateSuccess_updateOnDefined() throws Exception { // the response should come back right away ResponseEntity response = updateFuture.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); // make sure the query was updated - Assert.assertEquals(newQuery, queryStatus.getQuery().getQuery()); - Assert.assertEquals(newAuths, queryStatus.getQuery().getQueryAuthorizations()); - Assert.assertEquals(newBegin, DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate())); - Assert.assertEquals(newEnd, DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate())); - Assert.assertEquals(newLogic, queryStatus.getQuery().getQueryLogicName()); - Assert.assertEquals(newPageSize, queryStatus.getQuery().getPagesize()); + Assertions.assertEquals(newQuery, queryStatus.getQuery().getQuery()); + Assertions.assertEquals(newAuths, queryStatus.getQuery().getQueryAuthorizations()); + Assertions.assertEquals(newBegin, DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate())); + Assertions.assertEquals(newEnd, DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate())); + Assertions.assertEquals(newLogic, queryStatus.getQuery().getQueryLogicName()); + Assertions.assertEquals(newPageSize, queryStatus.getQuery().getPagesize()); // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); + Assertions.assertEquals(0, queryRequestEvents.size()); } @Test @@ -111,15 +111,15 @@ public void testUpdateSuccess_updateOnCreated() throws Exception { // the response should come back right away ResponseEntity response = updateFuture.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); // make sure the query was updated - Assert.assertEquals(newPageSize, queryStatus.getQuery().getPagesize()); + Assertions.assertEquals(newPageSize, queryStatus.getQuery().getPagesize()); // verify that events were published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -153,12 +153,12 @@ public void testUpdateFailure_unsafeParamUpdateQuery() throws Exception { // the response should come back right away ResponseEntity response = updateFuture.get(); - Assert.assertEquals(400, response.getStatusCodeValue()); + Assertions.assertEquals(400, response.getStatusCodeValue()); QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); // make sure the query was not updated - Assert.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); + Assertions.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); // @formatter:off assertQueryException( @@ -169,7 +169,7 @@ public void testUpdateFailure_unsafeParamUpdateQuery() throws Exception { // @formatter:on // verify that events were published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -205,13 +205,13 @@ public void testUpdateFailure_unsafeParamUpdateDate() throws Exception { // the response should come back right away ResponseEntity response = updateFuture.get(); - Assert.assertEquals(400, response.getStatusCodeValue()); + Assertions.assertEquals(400, response.getStatusCodeValue()); QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); // make sure the query was not updated - Assert.assertEquals(TEST_QUERY_BEGIN, DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate())); - Assert.assertEquals(TEST_QUERY_END, DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate())); + Assertions.assertEquals(TEST_QUERY_BEGIN, DefaultQueryParameters.formatDate(queryStatus.getQuery().getBeginDate())); + Assertions.assertEquals(TEST_QUERY_END, DefaultQueryParameters.formatDate(queryStatus.getQuery().getEndDate())); // @formatter:off assertQueryException( @@ -222,7 +222,7 @@ public void testUpdateFailure_unsafeParamUpdateDate() throws Exception { // @formatter:on // verify that events were published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -256,12 +256,12 @@ public void testUpdateFailure_unsafeParamUpdateLogic() throws Exception { // the response should come back right away ResponseEntity response = updateFuture.get(); - Assert.assertEquals(400, response.getStatusCodeValue()); + Assertions.assertEquals(400, response.getStatusCodeValue()); QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); // make sure the query was not updated - Assert.assertEquals("EventQuery", queryStatus.getQuery().getQueryLogicName()); + Assertions.assertEquals("EventQuery", queryStatus.getQuery().getQueryLogicName()); // @formatter:off assertQueryException( @@ -272,7 +272,7 @@ public void testUpdateFailure_unsafeParamUpdateLogic() throws Exception { // @formatter:on // verify that events were published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -306,12 +306,12 @@ public void testUpdateFailure_unsafeParamUpdateAuths() throws Exception { // the response should come back right away ResponseEntity response = updateFuture.get(); - Assert.assertEquals(400, response.getStatusCodeValue()); + Assertions.assertEquals(400, response.getStatusCodeValue()); QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); // make sure the query was not updated - Assert.assertEquals(TEST_QUERY_AUTHORIZATIONS, queryStatus.getQuery().getQueryAuthorizations()); + Assertions.assertEquals(TEST_QUERY_AUTHORIZATIONS, queryStatus.getQuery().getQueryAuthorizations()); // @formatter:off assertQueryException( @@ -322,7 +322,7 @@ public void testUpdateFailure_unsafeParamUpdateAuths() throws Exception { // @formatter:on // verify that events were published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -353,7 +353,7 @@ public void testUpdateFailure_nullParams() throws Exception { // the response should come back right away ResponseEntity response = updateFuture.get(); - Assert.assertEquals(400, response.getStatusCodeValue()); + Assertions.assertEquals(400, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -364,7 +364,7 @@ public void testUpdateFailure_nullParams() throws Exception { // @formatter:on // verify that events were published - Assert.assertEquals(1, queryRequestEvents.size()); + Assertions.assertEquals(1, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -395,7 +395,7 @@ public void testUpdateFailure_queryNotFound() throws Exception { // the response should come back right away ResponseEntity response = updateFuture.get(); - Assert.assertEquals(404, response.getStatusCodeValue()); + Assertions.assertEquals(404, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -406,7 +406,7 @@ public void testUpdateFailure_queryNotFound() throws Exception { // @formatter:on // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); + Assertions.assertEquals(0, queryRequestEvents.size()); } @Test @@ -434,7 +434,7 @@ public void testUpdateFailure_ownershipFailure() throws Exception { // the response should come back right away ResponseEntity response = updateFuture.get(); - Assert.assertEquals(401, response.getStatusCodeValue()); + Assertions.assertEquals(401, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -447,9 +447,9 @@ public void testUpdateFailure_ownershipFailure() throws Exception { QueryStatus queryStatus = queryStorageCache.getQueryStatus(queryId); // make sure the query was not updated - Assert.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); + Assertions.assertEquals(TEST_QUERY_STRING, queryStatus.getQuery().getQuery()); // verify that no events were published - Assert.assertEquals(0, queryRequestEvents.size()); + Assertions.assertEquals(0, queryRequestEvents.size()); } } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java index 59c4b910..7c846952 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java @@ -17,18 +17,18 @@ import datawave.webservice.result.BaseQueryResponse; import datawave.webservice.result.DefaultEventQueryResponse; import datawave.webservice.result.VoidResponse; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.HttpMethod; import org.springframework.http.RequestEntity; import org.springframework.http.ResponseEntity; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.util.UriComponents; @@ -50,7 +50,7 @@ import static datawave.microservice.query.QueryParameters.QUERY_MAX_RESULTS_OVERRIDE; import static datawave.microservice.query.lookup.LookupService.LOOKUP_UUID_PAIRS; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class LookupServiceTest extends AbstractQueryServiceTest { @@ -58,12 +58,12 @@ public class LookupServiceTest extends AbstractQueryServiceTest { @Autowired public LookupProperties lookupProperties; - @Before + @BeforeEach public void setup() { super.setup(); } - @After + @AfterEach public void teardown() throws Exception { super.teardown(); } @@ -106,12 +106,12 @@ public void testLookupUUIDSuccess() throws Exception { ResponseEntity response = future.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); // verify some headers - Assert.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + Assertions.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); @@ -139,7 +139,7 @@ public void testLookupUUIDSuccess() throws Exception { // @formatter:on // verify that the correct events were published - Assert.assertEquals(3, queryRequestEvents.size()); + Assertions.assertEquals(3, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -198,12 +198,12 @@ public void testBatchLookupUUIDSuccess() throws Exception { ResponseEntity response = future.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); // verify some headers - Assert.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + Assertions.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); @@ -231,7 +231,7 @@ public void testBatchLookupUUIDSuccess() throws Exception { // @formatter:on // verify that the correct events were published - Assert.assertEquals(3, queryRequestEvents.size()); + Assertions.assertEquals(3, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -300,7 +300,7 @@ public void testLookupContentUUIDSuccess() throws Exception { Thread.sleep(500); } - Assert.assertNotNull(contentQueryIds); + Assertions.assertNotNull(contentQueryIds); for (String contentQueryId : contentQueryIds) { MultiValueMap contentFieldValues = new LinkedMultiValueMap<>(); contentFieldValues.add("CONTENT", "look I made you some content!"); @@ -316,18 +316,18 @@ public void testLookupContentUUIDSuccess() throws Exception { ResponseEntity response = future.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); // verify some headers - Assert.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + Assertions.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); String responseQueryId = queryResponse.getQueryId(); - Assert.assertTrue(contentQueryIds.contains(responseQueryId)); + Assertions.assertTrue(contentQueryIds.contains(responseQueryId)); // verify the query response // @formatter:off @@ -351,7 +351,7 @@ public void testLookupContentUUIDSuccess() throws Exception { // @formatter:on // verify that the correct events were published - Assert.assertEquals(7, queryRequestEvents.size()); + Assertions.assertEquals(7, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -442,7 +442,7 @@ public void testBatchLookupContentUUIDSuccess() throws Exception { Thread.sleep(500); } - Assert.assertNotNull(contentQueryIds); + Assertions.assertNotNull(contentQueryIds); for (String contentQueryId : contentQueryIds) { MultiValueMap contentFieldValues = new LinkedMultiValueMap<>(); contentFieldValues.add("CONTENT", "look I made you some content!"); @@ -468,18 +468,18 @@ public void testBatchLookupContentUUIDSuccess() throws Exception { ResponseEntity response = future.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); // verify some headers - Assert.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); - Assert.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); + Assertions.assertEquals("1", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-page-number")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-Partial-Results")))); + Assertions.assertEquals("false", Iterables.getOnlyElement(Objects.requireNonNull(response.getHeaders().get("X-query-last-page")))); DefaultEventQueryResponse queryResponse = (DefaultEventQueryResponse) response.getBody(); String responseQueryId = queryResponse.getQueryId(); - Assert.assertTrue(contentQueryIds.contains(responseQueryId)); + Assertions.assertTrue(contentQueryIds.contains(responseQueryId)); // verify the query response // @formatter:off @@ -503,7 +503,7 @@ public void testBatchLookupContentUUIDSuccess() throws Exception { // @formatter:on // verify that the correct events were published - Assert.assertEquals(7, queryRequestEvents.size()); + Assertions.assertEquals(7, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -558,7 +558,7 @@ public void testBatchLookupUUIDFailure_noLookupUUIDPairs() throws Exception { ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); - Assert.assertEquals(400, response.getStatusCodeValue()); + Assertions.assertEquals(400, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -586,7 +586,7 @@ public void testBatchLookupUUIDFailure_mixedQueryLogics() throws Exception { ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); - Assert.assertEquals(400, response.getStatusCodeValue()); + Assertions.assertEquals(400, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -613,7 +613,7 @@ public void testBatchLookupUUIDFailure_nullUUIDType() throws Exception { ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); - Assert.assertEquals(400, response.getStatusCodeValue()); + Assertions.assertEquals(400, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -640,7 +640,7 @@ public void testBatchLookupUUIDFailure_emptyUUIDFieldValue() throws Exception { ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); - Assert.assertEquals(400, response.getStatusCodeValue()); + Assertions.assertEquals(400, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -667,7 +667,7 @@ public void testBatchLookupUUIDFailure_invalidUUIDPair() throws Exception { ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); - Assert.assertEquals(400, response.getStatusCodeValue()); + Assertions.assertEquals(400, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -697,7 +697,7 @@ public void testBatchLookupUUIDFailure_tooManyTerms() throws Exception { ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); - Assert.assertEquals(400, response.getStatusCodeValue()); + Assertions.assertEquals(400, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -724,7 +724,7 @@ public void testBatchLookupUUIDFailure_nonLookupQueryLogic() throws Exception { ResponseEntity response = jwtRestTemplate.exchange(requestEntity, VoidResponse.class); - Assert.assertEquals(500, response.getStatusCodeValue()); + Assertions.assertEquals(500, response.getStatusCodeValue()); // @formatter:off assertQueryException( @@ -811,11 +811,11 @@ protected void publishEventsToQueue(String queryId, int numEvents, MultiValueMap protected void assertContentQueryResponse(String queryId, String logicName, long pageNumber, boolean partialResults, long operationTimeInMS, int numEvents, DefaultEventQueryResponse queryResponse) { - Assert.assertEquals(queryId, queryResponse.getQueryId()); - Assert.assertEquals(logicName, queryResponse.getLogicName()); - Assert.assertEquals(pageNumber, queryResponse.getPageNumber()); - Assert.assertEquals(partialResults, queryResponse.isPartialResults()); - Assert.assertEquals(operationTimeInMS, queryResponse.getOperationTimeMS()); - Assert.assertEquals(numEvents, queryResponse.getEvents().size()); + Assertions.assertEquals(queryId, queryResponse.getQueryId()); + Assertions.assertEquals(logicName, queryResponse.getLogicName()); + Assertions.assertEquals(pageNumber, queryResponse.getPageNumber()); + Assertions.assertEquals(partialResults, queryResponse.isPartialResults()); + Assertions.assertEquals(operationTimeInMS, queryResponse.getOperationTimeMS()); + Assertions.assertEquals(numEvents, queryResponse.getEvents().size()); } } diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java index 71aa0799..6fcee8d2 100644 --- a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java +++ b/query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java @@ -12,11 +12,11 @@ import datawave.microservice.query.storage.QueryStatus; import datawave.webservice.query.result.event.DefaultEvent; import datawave.webservice.result.DefaultEventQueryResponse; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; @@ -24,7 +24,7 @@ import org.springframework.http.RequestEntity; import org.springframework.http.ResponseEntity; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.util.UriComponents; @@ -42,17 +42,17 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -@RunWith(SpringRunner.class) +@ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) @ActiveProfiles({"QueryStarterDefaults", "QueryStarterOverrides", "QueryServiceTest", RemoteAuthorizationServiceUserDetailsService.ACTIVATION_PROFILE}) public class StreamingServiceTest extends AbstractQueryServiceTest { - @Before + @BeforeEach public void setup() { super.setup(); } - @After + @AfterEach public void teardown() throws Exception { super.teardown(); } @@ -87,10 +87,10 @@ public void testExecuteSuccess() throws Throwable { // the response should come back right away ResponseEntity response = future.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); // verify some headers - Assert.assertEquals(MediaType.APPLICATION_XML, response.getHeaders().getContentType()); + Assertions.assertEquals(MediaType.APPLICATION_XML, response.getHeaders().getContentType()); int pageNumber = 1; @@ -121,7 +121,7 @@ public void testExecuteSuccess() throws Throwable { } // verify that the next event was published - Assert.assertEquals(6, queryRequestEvents.size()); + Assertions.assertEquals(6, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", @@ -198,10 +198,10 @@ public void testCreateAndExecuteSuccess() throws Throwable { // the response should come back right away ResponseEntity response = future.get(); - Assert.assertEquals(200, response.getStatusCodeValue()); + Assertions.assertEquals(200, response.getStatusCodeValue()); // verify some headers - Assert.assertEquals(MediaType.APPLICATION_XML, response.getHeaders().getContentType()); + Assertions.assertEquals(MediaType.APPLICATION_XML, response.getHeaders().getContentType()); int pageNumber = 1; @@ -232,7 +232,7 @@ public void testCreateAndExecuteSuccess() throws Throwable { } // verify that the next event was published - Assert.assertEquals(6, queryRequestEvents.size()); + Assertions.assertEquals(6, queryRequestEvents.size()); // @formatter:off assertQueryRequestEvent( "executor-unassigned:**", From d02bb065dae43fb124b94cbae1d351555005ea15 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 4 May 2022 14:56:31 -0400 Subject: [PATCH 144/218] Updated to exclude an unused slf4j binder, and log4j 1.x --- query-microservices/query-service/api/pom.xml | 10 ++++++++++ query-microservices/query-service/service/pom.xml | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/query-microservices/query-service/api/pom.xml b/query-microservices/query-service/api/pom.xml index 7f785dc1..8188f979 100644 --- a/query-microservices/query-service/api/pom.xml +++ b/query-microservices/query-service/api/pom.xml @@ -19,6 +19,16 @@ gov.nsa.datawave.microservice base-rest-responses ${version.microservice.base-rest-responses} + + + log4j + log4j + + + slf4j-reload4j + org.slf4j + + jakarta.validation diff --git a/query-microservices/query-service/service/pom.xml b/query-microservices/query-service/service/pom.xml index bb8a9866..7d3a8bdc 100644 --- a/query-microservices/query-service/service/pom.xml +++ b/query-microservices/query-service/service/pom.xml @@ -65,6 +65,16 @@ gov.nsa.datawave.services datawave-services-query + + + log4j + log4j + + + slf4j-reload4j + org.slf4j + + gov.nsa.datawave.microservice From 50c188a84cd90305c73e6b348f7768f38552a96d Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Thu, 5 May 2022 20:41:22 +0000 Subject: [PATCH 145/218] Moved the table cache reload endpoints, event handler, and monitor into the metadata-starter. --- .../service/src/main/resources/config/bootstrap.yml | 7 ++++++- .../service/src/test/resources/config/bootstrap.yml | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/query-microservices/query-service/service/src/main/resources/config/bootstrap.yml b/query-microservices/query-service/service/src/main/resources/config/bootstrap.yml index 63351c57..b39a0ecf 100644 --- a/query-microservices/query-service/service/src/main/resources/config/bootstrap.yml +++ b/query-microservices/query-service/service/src/main/resources/config/bootstrap.yml @@ -55,4 +55,9 @@ spring: enabled: false rabbitmq: discovery: - enabled: false \ No newline at end of file + enabled: false + +datawave: + table: + cache: + enabled: false diff --git a/query-microservices/query-service/service/src/test/resources/config/bootstrap.yml b/query-microservices/query-service/service/src/test/resources/config/bootstrap.yml index c8d89162..26eb3767 100644 --- a/query-microservices/query-service/service/src/test/resources/config/bootstrap.yml +++ b/query-microservices/query-service/service/src/test/resources/config/bootstrap.yml @@ -13,3 +13,8 @@ spring: # Starting with spring-boot 2.6, circular references are disabled by default # This is still needed for the evaluation-only function allow-circular-references: true + +datawave: + table: + cache: + enabled: false From 808541a4a7d9196874df4e9894dc9f6130bf8822 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Thu, 5 May 2022 22:54:16 +0000 Subject: [PATCH 146/218] Completed testing of table cache --- .../service/src/main/resources/config/bootstrap.yml | 10 +++++----- .../service/src/test/resources/config/bootstrap.yml | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/query-microservices/query-service/service/src/main/resources/config/bootstrap.yml b/query-microservices/query-service/service/src/main/resources/config/bootstrap.yml index b39a0ecf..d7aae5c8 100644 --- a/query-microservices/query-service/service/src/main/resources/config/bootstrap.yml +++ b/query-microservices/query-service/service/src/main/resources/config/bootstrap.yml @@ -20,6 +20,11 @@ spring: main: allow-circular-references: true +datawave: + table: + cache: + enabled: false + --- # For the dev profile, check localhost for the config server by default @@ -56,8 +61,3 @@ spring: rabbitmq: discovery: enabled: false - -datawave: - table: - cache: - enabled: false diff --git a/query-microservices/query-service/service/src/test/resources/config/bootstrap.yml b/query-microservices/query-service/service/src/test/resources/config/bootstrap.yml index 26eb3767..9188151a 100644 --- a/query-microservices/query-service/service/src/test/resources/config/bootstrap.yml +++ b/query-microservices/query-service/service/src/test/resources/config/bootstrap.yml @@ -18,3 +18,4 @@ datawave: table: cache: enabled: false + From 3808921a5468cae6f82547bedba0bffb6b85509d Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 12 May 2022 13:30:08 -0400 Subject: [PATCH 147/218] Updated project layout to combine all services into a single repo, with a single build. --- .../services/query}/README.md | 0 .../services/query}/api/pom.xml | 2 +- .../query/DefaultQueryParameters.java | 0 .../microservice/query/QueryParameters.java | 0 .../microservice/query/QueryPersistence.java | 0 .../config/QueryExpirationProperties.java | 0 .../api/src/main/resources/META-INF/beans.xml | 0 .../main/resources/META-INF/jboss-ejb3.xml | 0 .../services/query}/pom.xml | 0 .../services/query}/service/pom.xml | 30 +++++++++---------- .../query}/service/src/main/docker/Dockerfile | 0 .../microservice/query/QueryController.java | 0 .../query/QueryManagementService.java | 0 .../microservice/query/QueryService.java | 0 .../query/config/QueryServiceConfig.java | 0 .../query/lookup/LookupService.java | 0 .../query/monitor/MonitorTask.java | 0 .../query/monitor/QueryMonitor.java | 0 .../query/monitor/cache/MonitorStatus.java | 0 .../monitor/cache/MonitorStatusCache.java | 0 .../query/monitor/config/MonitorConfig.java | 0 .../monitor/config/MonitorProperties.java | 0 .../microservice/query/runner/NextCall.java | 0 .../query/stream/StreamingService.java | 0 .../CountingResponseBodyEmitterListener.java | 0 .../listener/StreamingResponseListener.java | 0 .../query/stream/runner/StreamingCall.java | 0 .../query/translateid/TranslateIdService.java | 0 .../microservice/query/util/QueryUtil.java | 0 .../query/web/BaseQueryResponseAdvice.java | 0 .../query/web/QuerySessionIdAdvice.java | 0 .../web/annotation/EnrichQueryMetrics.java | 0 .../annotation/GenerateQuerySessionId.java | 0 .../web/filter/BaseMethodStatsFilter.java | 0 .../filter/CountingResponseBodyEmitter.java | 0 .../query/web/filter/LoggingStatsFilter.java | 0 .../QueryMetricsEnrichmentFilterAdvice.java | 0 .../src/main/resources/config/application.yml | 0 .../src/main/resources/config/bootstrap.yml | 0 .../service/src/main/resources/log4j2.yml | 0 .../query/AbstractQueryServiceTest.java | 0 .../query/QueryServiceCancelTest.java | 0 .../query/QueryServiceCloseTest.java | 0 .../query/QueryServiceCreateTest.java | 0 .../query/QueryServiceDefineTest.java | 0 .../query/QueryServiceDuplicateTest.java | 0 .../query/QueryServiceListTest.java | 0 .../query/QueryServiceNextTest.java | 0 .../query/QueryServicePlanTest.java | 0 .../query/QueryServiceRemoveTest.java | 0 .../query/QueryServiceResetTest.java | 0 .../query/QueryServiceUpdateTest.java | 0 .../query/lookup/LookupServiceTest.java | 0 .../query/stream/StreamingServiceTest.java | 0 .../test/resources/TestQueryLogicFactory.xml | 0 .../application-QueryStarterOverrides.yml | 0 .../src/test/resources/config/application.yml | 0 .../src/test/resources/config/bootstrap.yml | 0 .../src/test/resources/log4j2-test.xml | 0 59 files changed, 16 insertions(+), 16 deletions(-) rename {query-microservices/query-service => microservices/services/query}/README.md (100%) rename {query-microservices/query-service => microservices/services/query}/api/pom.xml (98%) rename {query-microservices/query-service => microservices/services/query}/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java (100%) rename {query-microservices/query-service => microservices/services/query}/api/src/main/java/datawave/microservice/query/QueryParameters.java (100%) rename {query-microservices/query-service => microservices/services/query}/api/src/main/java/datawave/microservice/query/QueryPersistence.java (100%) rename {query-microservices/query-service => microservices/services/query}/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java (100%) rename {query-microservices/query-service => microservices/services/query}/api/src/main/resources/META-INF/beans.xml (100%) rename {query-microservices/query-service => microservices/services/query}/api/src/main/resources/META-INF/jboss-ejb3.xml (100%) rename {query-microservices/query-service => microservices/services/query}/pom.xml (100%) rename {query-microservices/query-service => microservices/services/query}/service/pom.xml (97%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/docker/Dockerfile (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/QueryController.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/QueryManagementService.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/QueryService.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/lookup/LookupService.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/runner/NextCall.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/stream/StreamingService.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/translateid/TranslateIdService.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/util/QueryUtil.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/resources/config/application.yml (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/resources/config/bootstrap.yml (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/main/resources/log4j2.yml (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/test/resources/TestQueryLogicFactory.xml (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/test/resources/config/application-QueryStarterOverrides.yml (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/test/resources/config/application.yml (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/test/resources/config/bootstrap.yml (100%) rename {query-microservices/query-service => microservices/services/query}/service/src/test/resources/log4j2-test.xml (100%) diff --git a/query-microservices/query-service/README.md b/microservices/services/query/README.md similarity index 100% rename from query-microservices/query-service/README.md rename to microservices/services/query/README.md diff --git a/query-microservices/query-service/api/pom.xml b/microservices/services/query/api/pom.xml similarity index 98% rename from query-microservices/query-service/api/pom.xml rename to microservices/services/query/api/pom.xml index 8188f979..8626f6c7 100644 --- a/query-microservices/query-service/api/pom.xml +++ b/microservices/services/query/api/pom.xml @@ -5,7 +5,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent 1.9-SNAPSHOT - + ../../../microservice-parent/pom.xml query-api 1.0-SNAPSHOT diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java b/microservices/services/query/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java similarity index 100% rename from query-microservices/query-service/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java rename to microservices/services/query/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParameters.java b/microservices/services/query/api/src/main/java/datawave/microservice/query/QueryParameters.java similarity index 100% rename from query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryParameters.java rename to microservices/services/query/api/src/main/java/datawave/microservice/query/QueryParameters.java diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryPersistence.java b/microservices/services/query/api/src/main/java/datawave/microservice/query/QueryPersistence.java similarity index 100% rename from query-microservices/query-service/api/src/main/java/datawave/microservice/query/QueryPersistence.java rename to microservices/services/query/api/src/main/java/datawave/microservice/query/QueryPersistence.java diff --git a/query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java b/microservices/services/query/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java similarity index 100% rename from query-microservices/query-service/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java rename to microservices/services/query/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java diff --git a/query-microservices/query-service/api/src/main/resources/META-INF/beans.xml b/microservices/services/query/api/src/main/resources/META-INF/beans.xml similarity index 100% rename from query-microservices/query-service/api/src/main/resources/META-INF/beans.xml rename to microservices/services/query/api/src/main/resources/META-INF/beans.xml diff --git a/query-microservices/query-service/api/src/main/resources/META-INF/jboss-ejb3.xml b/microservices/services/query/api/src/main/resources/META-INF/jboss-ejb3.xml similarity index 100% rename from query-microservices/query-service/api/src/main/resources/META-INF/jboss-ejb3.xml rename to microservices/services/query/api/src/main/resources/META-INF/jboss-ejb3.xml diff --git a/query-microservices/query-service/pom.xml b/microservices/services/query/pom.xml similarity index 100% rename from query-microservices/query-service/pom.xml rename to microservices/services/query/pom.xml diff --git a/query-microservices/query-service/service/pom.xml b/microservices/services/query/service/pom.xml similarity index 97% rename from query-microservices/query-service/service/pom.xml rename to microservices/services/query/service/pom.xml index 7d3a8bdc..98739165 100644 --- a/query-microservices/query-service/service/pom.xml +++ b/microservices/services/query/service/pom.xml @@ -5,7 +5,7 @@ gov.nsa.datawave.microservice datawave-microservice-service-parent 2.1-SNAPSHOT - + ../../../microservice-service-parent/pom.xml query-service 1.0-SNAPSHOT @@ -51,20 +51,8 @@ - gov.nsa.datawave.microservice - query-api - - - gov.nsa.datawave.microservice - spring-boot-starter-datawave-audit - - - gov.nsa.datawave.microservice - spring-boot-starter-datawave-query - - - gov.nsa.datawave.services - datawave-services-query + gov.nsa.datawave.core + datawave-core-query log4j @@ -76,6 +64,18 @@ + + gov.nsa.datawave.microservice + query-api + + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-audit + + + gov.nsa.datawave.microservice + spring-boot-starter-datawave-query + gov.nsa.datawave.microservice spring-boot-starter-datawave-query diff --git a/query-microservices/query-service/service/src/main/docker/Dockerfile b/microservices/services/query/service/src/main/docker/Dockerfile similarity index 100% rename from query-microservices/query-service/service/src/main/docker/Dockerfile rename to microservices/services/query/service/src/main/docker/Dockerfile diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/QueryController.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryController.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/QueryController.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryManagementService.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryService.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/QueryService.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/QueryService.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/QueryService.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/lookup/LookupService.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/lookup/LookupService.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/lookup/LookupService.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/lookup/LookupService.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/runner/NextCall.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/runner/NextCall.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/runner/NextCall.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/StreamingService.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/stream/StreamingService.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/StreamingService.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/stream/StreamingService.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/translateid/TranslateIdService.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/translateid/TranslateIdService.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/translateid/TranslateIdService.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/translateid/TranslateIdService.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/util/QueryUtil.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/util/QueryUtil.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/util/QueryUtil.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/util/QueryUtil.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java diff --git a/query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java b/microservices/services/query/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java similarity index 100% rename from query-microservices/query-service/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java rename to microservices/services/query/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java diff --git a/query-microservices/query-service/service/src/main/resources/config/application.yml b/microservices/services/query/service/src/main/resources/config/application.yml similarity index 100% rename from query-microservices/query-service/service/src/main/resources/config/application.yml rename to microservices/services/query/service/src/main/resources/config/application.yml diff --git a/query-microservices/query-service/service/src/main/resources/config/bootstrap.yml b/microservices/services/query/service/src/main/resources/config/bootstrap.yml similarity index 100% rename from query-microservices/query-service/service/src/main/resources/config/bootstrap.yml rename to microservices/services/query/service/src/main/resources/config/bootstrap.yml diff --git a/query-microservices/query-service/service/src/main/resources/log4j2.yml b/microservices/services/query/service/src/main/resources/log4j2.yml similarity index 100% rename from query-microservices/query-service/service/src/main/resources/log4j2.yml rename to microservices/services/query/service/src/main/resources/log4j2.yml diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java similarity index 100% rename from query-microservices/query-service/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java rename to microservices/services/query/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java similarity index 100% rename from query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java rename to microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java similarity index 100% rename from query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java rename to microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java similarity index 100% rename from query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java rename to microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java similarity index 100% rename from query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java rename to microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java similarity index 100% rename from query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java rename to microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java similarity index 100% rename from query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java rename to microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java similarity index 100% rename from query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java rename to microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java similarity index 100% rename from query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java rename to microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java similarity index 100% rename from query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java rename to microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java similarity index 100% rename from query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java rename to microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java similarity index 100% rename from query-microservices/query-service/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java rename to microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java similarity index 100% rename from query-microservices/query-service/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java rename to microservices/services/query/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java diff --git a/query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java b/microservices/services/query/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java similarity index 100% rename from query-microservices/query-service/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java rename to microservices/services/query/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java diff --git a/query-microservices/query-service/service/src/test/resources/TestQueryLogicFactory.xml b/microservices/services/query/service/src/test/resources/TestQueryLogicFactory.xml similarity index 100% rename from query-microservices/query-service/service/src/test/resources/TestQueryLogicFactory.xml rename to microservices/services/query/service/src/test/resources/TestQueryLogicFactory.xml diff --git a/query-microservices/query-service/service/src/test/resources/config/application-QueryStarterOverrides.yml b/microservices/services/query/service/src/test/resources/config/application-QueryStarterOverrides.yml similarity index 100% rename from query-microservices/query-service/service/src/test/resources/config/application-QueryStarterOverrides.yml rename to microservices/services/query/service/src/test/resources/config/application-QueryStarterOverrides.yml diff --git a/query-microservices/query-service/service/src/test/resources/config/application.yml b/microservices/services/query/service/src/test/resources/config/application.yml similarity index 100% rename from query-microservices/query-service/service/src/test/resources/config/application.yml rename to microservices/services/query/service/src/test/resources/config/application.yml diff --git a/query-microservices/query-service/service/src/test/resources/config/bootstrap.yml b/microservices/services/query/service/src/test/resources/config/bootstrap.yml similarity index 100% rename from query-microservices/query-service/service/src/test/resources/config/bootstrap.yml rename to microservices/services/query/service/src/test/resources/config/bootstrap.yml diff --git a/query-microservices/query-service/service/src/test/resources/log4j2-test.xml b/microservices/services/query/service/src/test/resources/log4j2-test.xml similarity index 100% rename from query-microservices/query-service/service/src/test/resources/log4j2-test.xml rename to microservices/services/query/service/src/test/resources/log4j2-test.xml From a7654e2b188820ac067f0460b8c5742de45a6c62 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 20 May 2022 11:06:39 -0400 Subject: [PATCH 148/218] Added a module which builds the quickstart docker image. Moved the microservices docker folder to the top level. --- microservices/services/query/service/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/microservices/services/query/service/pom.xml b/microservices/services/query/service/pom.xml index 98739165..7c1bef63 100644 --- a/microservices/services/query/service/pom.xml +++ b/microservices/services/query/service/pom.xml @@ -165,6 +165,11 @@ docker + + + microservice-docker + + From 6c37a6230a0193b86eb3d27851fb358d579da559 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 25 May 2022 13:38:01 -0400 Subject: [PATCH 149/218] Moved project files to root directory. --- microservices/services/query/README.md => README.md | 0 {microservices/services/query/api => api}/pom.xml | 0 .../java/datawave/microservice/query/DefaultQueryParameters.java | 0 .../main/java/datawave/microservice/query/QueryParameters.java | 0 .../main/java/datawave/microservice/query/QueryPersistence.java | 0 .../microservice/query/config/QueryExpirationProperties.java | 0 .../query/api => api}/src/main/resources/META-INF/beans.xml | 0 .../query/api => api}/src/main/resources/META-INF/jboss-ejb3.xml | 0 microservices/services/query/pom.xml => pom.xml | 0 {microservices/services/query/service => service}/pom.xml | 0 .../services/query/service => service}/src/main/docker/Dockerfile | 0 .../main/java/datawave/microservice/query/QueryController.java | 0 .../java/datawave/microservice/query/QueryManagementService.java | 0 .../src/main/java/datawave/microservice/query/QueryService.java | 0 .../datawave/microservice/query/config/QueryServiceConfig.java | 0 .../java/datawave/microservice/query/lookup/LookupService.java | 0 .../java/datawave/microservice/query/monitor/MonitorTask.java | 0 .../java/datawave/microservice/query/monitor/QueryMonitor.java | 0 .../datawave/microservice/query/monitor/cache/MonitorStatus.java | 0 .../microservice/query/monitor/cache/MonitorStatusCache.java | 0 .../datawave/microservice/query/monitor/config/MonitorConfig.java | 0 .../microservice/query/monitor/config/MonitorProperties.java | 0 .../main/java/datawave/microservice/query/runner/NextCall.java | 0 .../java/datawave/microservice/query/stream/StreamingService.java | 0 .../stream/listener/CountingResponseBodyEmitterListener.java | 0 .../query/stream/listener/StreamingResponseListener.java | 0 .../datawave/microservice/query/stream/runner/StreamingCall.java | 0 .../microservice/query/translateid/TranslateIdService.java | 0 .../src/main/java/datawave/microservice/query/util/QueryUtil.java | 0 .../datawave/microservice/query/web/BaseQueryResponseAdvice.java | 0 .../datawave/microservice/query/web/QuerySessionIdAdvice.java | 0 .../microservice/query/web/annotation/EnrichQueryMetrics.java | 0 .../microservice/query/web/annotation/GenerateQuerySessionId.java | 0 .../microservice/query/web/filter/BaseMethodStatsFilter.java | 0 .../query/web/filter/CountingResponseBodyEmitter.java | 0 .../microservice/query/web/filter/LoggingStatsFilter.java | 0 .../query/web/filter/QueryMetricsEnrichmentFilterAdvice.java | 0 .../service => service}/src/main/resources/config/application.yml | 0 .../service => service}/src/main/resources/config/bootstrap.yml | 0 .../query/service => service}/src/main/resources/log4j2.yml | 0 .../datawave/microservice/query/AbstractQueryServiceTest.java | 0 .../java/datawave/microservice/query/QueryServiceCancelTest.java | 0 .../java/datawave/microservice/query/QueryServiceCloseTest.java | 0 .../java/datawave/microservice/query/QueryServiceCreateTest.java | 0 .../java/datawave/microservice/query/QueryServiceDefineTest.java | 0 .../datawave/microservice/query/QueryServiceDuplicateTest.java | 0 .../java/datawave/microservice/query/QueryServiceListTest.java | 0 .../java/datawave/microservice/query/QueryServiceNextTest.java | 0 .../java/datawave/microservice/query/QueryServicePlanTest.java | 0 .../java/datawave/microservice/query/QueryServiceRemoveTest.java | 0 .../java/datawave/microservice/query/QueryServiceResetTest.java | 0 .../java/datawave/microservice/query/QueryServiceUpdateTest.java | 0 .../datawave/microservice/query/lookup/LookupServiceTest.java | 0 .../datawave/microservice/query/stream/StreamingServiceTest.java | 0 .../src/test/resources/TestQueryLogicFactory.xml | 0 .../test/resources/config/application-QueryStarterOverrides.yml | 0 .../service => service}/src/test/resources/config/application.yml | 0 .../service => service}/src/test/resources/config/bootstrap.yml | 0 .../query/service => service}/src/test/resources/log4j2-test.xml | 0 59 files changed, 0 insertions(+), 0 deletions(-) rename microservices/services/query/README.md => README.md (100%) rename {microservices/services/query/api => api}/pom.xml (100%) rename {microservices/services/query/api => api}/src/main/java/datawave/microservice/query/DefaultQueryParameters.java (100%) rename {microservices/services/query/api => api}/src/main/java/datawave/microservice/query/QueryParameters.java (100%) rename {microservices/services/query/api => api}/src/main/java/datawave/microservice/query/QueryPersistence.java (100%) rename {microservices/services/query/api => api}/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java (100%) rename {microservices/services/query/api => api}/src/main/resources/META-INF/beans.xml (100%) rename {microservices/services/query/api => api}/src/main/resources/META-INF/jboss-ejb3.xml (100%) rename microservices/services/query/pom.xml => pom.xml (100%) rename {microservices/services/query/service => service}/pom.xml (100%) rename {microservices/services/query/service => service}/src/main/docker/Dockerfile (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/QueryController.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/QueryManagementService.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/QueryService.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/lookup/LookupService.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/monitor/MonitorTask.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/runner/NextCall.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/stream/StreamingService.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/translateid/TranslateIdService.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/util/QueryUtil.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java (100%) rename {microservices/services/query/service => service}/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java (100%) rename {microservices/services/query/service => service}/src/main/resources/config/application.yml (100%) rename {microservices/services/query/service => service}/src/main/resources/config/bootstrap.yml (100%) rename {microservices/services/query/service => service}/src/main/resources/log4j2.yml (100%) rename {microservices/services/query/service => service}/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java (100%) rename {microservices/services/query/service => service}/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java (100%) rename {microservices/services/query/service => service}/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java (100%) rename {microservices/services/query/service => service}/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java (100%) rename {microservices/services/query/service => service}/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java (100%) rename {microservices/services/query/service => service}/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java (100%) rename {microservices/services/query/service => service}/src/test/java/datawave/microservice/query/QueryServiceListTest.java (100%) rename {microservices/services/query/service => service}/src/test/java/datawave/microservice/query/QueryServiceNextTest.java (100%) rename {microservices/services/query/service => service}/src/test/java/datawave/microservice/query/QueryServicePlanTest.java (100%) rename {microservices/services/query/service => service}/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java (100%) rename {microservices/services/query/service => service}/src/test/java/datawave/microservice/query/QueryServiceResetTest.java (100%) rename {microservices/services/query/service => service}/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java (100%) rename {microservices/services/query/service => service}/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java (100%) rename {microservices/services/query/service => service}/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java (100%) rename {microservices/services/query/service => service}/src/test/resources/TestQueryLogicFactory.xml (100%) rename {microservices/services/query/service => service}/src/test/resources/config/application-QueryStarterOverrides.yml (100%) rename {microservices/services/query/service => service}/src/test/resources/config/application.yml (100%) rename {microservices/services/query/service => service}/src/test/resources/config/bootstrap.yml (100%) rename {microservices/services/query/service => service}/src/test/resources/log4j2-test.xml (100%) diff --git a/microservices/services/query/README.md b/README.md similarity index 100% rename from microservices/services/query/README.md rename to README.md diff --git a/microservices/services/query/api/pom.xml b/api/pom.xml similarity index 100% rename from microservices/services/query/api/pom.xml rename to api/pom.xml diff --git a/microservices/services/query/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java b/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java similarity index 100% rename from microservices/services/query/api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java rename to api/src/main/java/datawave/microservice/query/DefaultQueryParameters.java diff --git a/microservices/services/query/api/src/main/java/datawave/microservice/query/QueryParameters.java b/api/src/main/java/datawave/microservice/query/QueryParameters.java similarity index 100% rename from microservices/services/query/api/src/main/java/datawave/microservice/query/QueryParameters.java rename to api/src/main/java/datawave/microservice/query/QueryParameters.java diff --git a/microservices/services/query/api/src/main/java/datawave/microservice/query/QueryPersistence.java b/api/src/main/java/datawave/microservice/query/QueryPersistence.java similarity index 100% rename from microservices/services/query/api/src/main/java/datawave/microservice/query/QueryPersistence.java rename to api/src/main/java/datawave/microservice/query/QueryPersistence.java diff --git a/microservices/services/query/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java b/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java similarity index 100% rename from microservices/services/query/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java rename to api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java diff --git a/microservices/services/query/api/src/main/resources/META-INF/beans.xml b/api/src/main/resources/META-INF/beans.xml similarity index 100% rename from microservices/services/query/api/src/main/resources/META-INF/beans.xml rename to api/src/main/resources/META-INF/beans.xml diff --git a/microservices/services/query/api/src/main/resources/META-INF/jboss-ejb3.xml b/api/src/main/resources/META-INF/jboss-ejb3.xml similarity index 100% rename from microservices/services/query/api/src/main/resources/META-INF/jboss-ejb3.xml rename to api/src/main/resources/META-INF/jboss-ejb3.xml diff --git a/microservices/services/query/pom.xml b/pom.xml similarity index 100% rename from microservices/services/query/pom.xml rename to pom.xml diff --git a/microservices/services/query/service/pom.xml b/service/pom.xml similarity index 100% rename from microservices/services/query/service/pom.xml rename to service/pom.xml diff --git a/microservices/services/query/service/src/main/docker/Dockerfile b/service/src/main/docker/Dockerfile similarity index 100% rename from microservices/services/query/service/src/main/docker/Dockerfile rename to service/src/main/docker/Dockerfile diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/QueryController.java b/service/src/main/java/datawave/microservice/query/QueryController.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/QueryController.java rename to service/src/main/java/datawave/microservice/query/QueryController.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/service/src/main/java/datawave/microservice/query/QueryManagementService.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/QueryManagementService.java rename to service/src/main/java/datawave/microservice/query/QueryManagementService.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/QueryService.java b/service/src/main/java/datawave/microservice/query/QueryService.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/QueryService.java rename to service/src/main/java/datawave/microservice/query/QueryService.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java rename to service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/lookup/LookupService.java b/service/src/main/java/datawave/microservice/query/lookup/LookupService.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/lookup/LookupService.java rename to service/src/main/java/datawave/microservice/query/lookup/LookupService.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java b/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java rename to service/src/main/java/datawave/microservice/query/monitor/MonitorTask.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java b/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java rename to service/src/main/java/datawave/microservice/query/monitor/QueryMonitor.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java b/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java rename to service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatus.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java b/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java rename to service/src/main/java/datawave/microservice/query/monitor/cache/MonitorStatusCache.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java b/service/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java rename to service/src/main/java/datawave/microservice/query/monitor/config/MonitorConfig.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java b/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java rename to service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/service/src/main/java/datawave/microservice/query/runner/NextCall.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/runner/NextCall.java rename to service/src/main/java/datawave/microservice/query/runner/NextCall.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/stream/StreamingService.java b/service/src/main/java/datawave/microservice/query/stream/StreamingService.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/stream/StreamingService.java rename to service/src/main/java/datawave/microservice/query/stream/StreamingService.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java b/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java rename to service/src/main/java/datawave/microservice/query/stream/listener/CountingResponseBodyEmitterListener.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java b/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java rename to service/src/main/java/datawave/microservice/query/stream/listener/StreamingResponseListener.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java b/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java rename to service/src/main/java/datawave/microservice/query/stream/runner/StreamingCall.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/translateid/TranslateIdService.java b/service/src/main/java/datawave/microservice/query/translateid/TranslateIdService.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/translateid/TranslateIdService.java rename to service/src/main/java/datawave/microservice/query/translateid/TranslateIdService.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/util/QueryUtil.java b/service/src/main/java/datawave/microservice/query/util/QueryUtil.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/util/QueryUtil.java rename to service/src/main/java/datawave/microservice/query/util/QueryUtil.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java b/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java rename to service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java b/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java rename to service/src/main/java/datawave/microservice/query/web/QuerySessionIdAdvice.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java b/service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java rename to service/src/main/java/datawave/microservice/query/web/annotation/EnrichQueryMetrics.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java b/service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java rename to service/src/main/java/datawave/microservice/query/web/annotation/GenerateQuerySessionId.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java b/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java rename to service/src/main/java/datawave/microservice/query/web/filter/BaseMethodStatsFilter.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java b/service/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java rename to service/src/main/java/datawave/microservice/query/web/filter/CountingResponseBodyEmitter.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java b/service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java rename to service/src/main/java/datawave/microservice/query/web/filter/LoggingStatsFilter.java diff --git a/microservices/services/query/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java b/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java similarity index 100% rename from microservices/services/query/service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java rename to service/src/main/java/datawave/microservice/query/web/filter/QueryMetricsEnrichmentFilterAdvice.java diff --git a/microservices/services/query/service/src/main/resources/config/application.yml b/service/src/main/resources/config/application.yml similarity index 100% rename from microservices/services/query/service/src/main/resources/config/application.yml rename to service/src/main/resources/config/application.yml diff --git a/microservices/services/query/service/src/main/resources/config/bootstrap.yml b/service/src/main/resources/config/bootstrap.yml similarity index 100% rename from microservices/services/query/service/src/main/resources/config/bootstrap.yml rename to service/src/main/resources/config/bootstrap.yml diff --git a/microservices/services/query/service/src/main/resources/log4j2.yml b/service/src/main/resources/log4j2.yml similarity index 100% rename from microservices/services/query/service/src/main/resources/log4j2.yml rename to service/src/main/resources/log4j2.yml diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java b/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java similarity index 100% rename from microservices/services/query/service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java rename to service/src/test/java/datawave/microservice/query/AbstractQueryServiceTest.java diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java similarity index 100% rename from microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java rename to service/src/test/java/datawave/microservice/query/QueryServiceCancelTest.java diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java similarity index 100% rename from microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java rename to service/src/test/java/datawave/microservice/query/QueryServiceCloseTest.java diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java similarity index 100% rename from microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java rename to service/src/test/java/datawave/microservice/query/QueryServiceCreateTest.java diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java similarity index 100% rename from microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java rename to service/src/test/java/datawave/microservice/query/QueryServiceDefineTest.java diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java similarity index 100% rename from microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java rename to service/src/test/java/datawave/microservice/query/QueryServiceDuplicateTest.java diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java similarity index 100% rename from microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceListTest.java rename to service/src/test/java/datawave/microservice/query/QueryServiceListTest.java diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java similarity index 100% rename from microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java rename to service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java b/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java similarity index 100% rename from microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java rename to service/src/test/java/datawave/microservice/query/QueryServicePlanTest.java diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java similarity index 100% rename from microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java rename to service/src/test/java/datawave/microservice/query/QueryServiceRemoveTest.java diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java similarity index 100% rename from microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java rename to service/src/test/java/datawave/microservice/query/QueryServiceResetTest.java diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java similarity index 100% rename from microservices/services/query/service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java rename to service/src/test/java/datawave/microservice/query/QueryServiceUpdateTest.java diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java b/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java similarity index 100% rename from microservices/services/query/service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java rename to service/src/test/java/datawave/microservice/query/lookup/LookupServiceTest.java diff --git a/microservices/services/query/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java b/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java similarity index 100% rename from microservices/services/query/service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java rename to service/src/test/java/datawave/microservice/query/stream/StreamingServiceTest.java diff --git a/microservices/services/query/service/src/test/resources/TestQueryLogicFactory.xml b/service/src/test/resources/TestQueryLogicFactory.xml similarity index 100% rename from microservices/services/query/service/src/test/resources/TestQueryLogicFactory.xml rename to service/src/test/resources/TestQueryLogicFactory.xml diff --git a/microservices/services/query/service/src/test/resources/config/application-QueryStarterOverrides.yml b/service/src/test/resources/config/application-QueryStarterOverrides.yml similarity index 100% rename from microservices/services/query/service/src/test/resources/config/application-QueryStarterOverrides.yml rename to service/src/test/resources/config/application-QueryStarterOverrides.yml diff --git a/microservices/services/query/service/src/test/resources/config/application.yml b/service/src/test/resources/config/application.yml similarity index 100% rename from microservices/services/query/service/src/test/resources/config/application.yml rename to service/src/test/resources/config/application.yml diff --git a/microservices/services/query/service/src/test/resources/config/bootstrap.yml b/service/src/test/resources/config/bootstrap.yml similarity index 100% rename from microservices/services/query/service/src/test/resources/config/bootstrap.yml rename to service/src/test/resources/config/bootstrap.yml diff --git a/microservices/services/query/service/src/test/resources/log4j2-test.xml b/service/src/test/resources/log4j2-test.xml similarity index 100% rename from microservices/services/query/service/src/test/resources/log4j2-test.xml rename to service/src/test/resources/log4j2-test.xml From a8bcca5450a12a0bf397625dde08022c3e68ee05 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 25 May 2022 13:48:57 -0400 Subject: [PATCH 150/218] Updated the README --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5fd2ba9e..c6b87b28 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,7 @@ [![Apache License][li]][ll] ![Build Status](https://github.com/NationalSecurityAgency/datawave/workflows/Tests/badge.svg) -The query api service is a DATAWAVE microservice that provides -query capabilities. +The query service is a user-facing DATAWAVE microservice that serves as the main REST interface for DataWave query functionality. ### Query Context @@ -59,8 +58,9 @@ query capabilities. TBD -[getting-started]:https://github.com/NationalSecurityAgency/datawave-microservices-root/blob/master/README.md#getting-started -[pki-dir]:https://github.com/NationalSecurityAgency/datawave-spring-boot-starter/blob/master/src/main/resources/pki +For now, refer to the [Datawave Docker Compose Readme][getting-started] + +[getting-started]:https://github.com/NationalSecurityAgency/datawave/blob/feature/queryMicroservices/docker/README.md#datawave-docker-compose [li]: http://img.shields.io/badge/license-ASL-blue.svg -[ll]: https://www.apache.org/licenses/LICENSE-2.0 \ No newline at end of file +[ll]: https://www.apache.org/licenses/LICENSE-2.0 From ac79ddfca9479913b55b7d5bac03b8c16da5851f Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Fri, 10 Jun 2022 14:01:09 +0000 Subject: [PATCH 151/218] Add long running query page retry limit --- .../query/config/QueryExpirationProperties.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java b/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java index b8a4eb8d..855b7393 100644 --- a/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java +++ b/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java @@ -40,6 +40,8 @@ public class QueryExpirationProperties { private long shortCircuitTimeout = Math.round(0.97 * callTimeout); @NotNull private TimeUnit shortCircuitTimeUnit = TimeUnit.MINUTES; + @Positive + private int maxLongRunningTimeoutRetries = 3; public long getIdleTimeout() { return idleTimeout; @@ -200,4 +202,13 @@ public TimeUnit getShortCircuitTimeUnit() { public void setShortCircuitTimeUnit(TimeUnit shortCircuitTimeUnit) { this.shortCircuitTimeUnit = shortCircuitTimeUnit; } + + public int getMaxLongRunningTimeoutRetries() { + return maxLongRunningTimeoutRetries; + } + + public void setMaxLongRunningTimeoutRetries(int maxLongRunningTimeoutRetries) { + this.maxLongRunningTimeoutRetries = maxLongRunningTimeoutRetries; + } + } From b98b6490c04e365e81dc644491d01b24c4b4b02c Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 10 Jun 2022 15:23:37 -0400 Subject: [PATCH 152/218] Added a landing page which exposes swagger and javadoc endpoints --- pom.xml | 2 +- service/pom.xml | 19 ++ .../microservice/query/QueryController.java | 101 ++++++++ .../query/QueryManagementService.java | 4 +- .../microservice/query/WebController.java | 31 +++ .../query/config/QueryServiceConfig.java | 2 + .../src/main/resources/static/css/dwdocs.css | 17 ++ .../static/images/api_icon_finished.png | Bin 0 -> 97067 bytes .../resources/static/images/dwdocs_logo.png | Bin 0 -> 28442 bytes .../resources/static/images/dwquery_logo.png | Bin 0 -> 29731 bytes .../static/images/releaseNotes_icon.png | Bin 0 -> 46184 bytes .../static/images/userGuide_icon.png | Bin 0 -> 116652 bytes service/src/main/resources/static/js/index.js | 17 ++ .../src/main/resources/templates/index.html | 104 ++++++++ .../main/resources/templates/query_help.html | 241 ++++++++++++++++++ 15 files changed, 536 insertions(+), 2 deletions(-) create mode 100644 service/src/main/java/datawave/microservice/query/WebController.java create mode 100644 service/src/main/resources/static/css/dwdocs.css create mode 100644 service/src/main/resources/static/images/api_icon_finished.png create mode 100644 service/src/main/resources/static/images/dwdocs_logo.png create mode 100644 service/src/main/resources/static/images/dwquery_logo.png create mode 100644 service/src/main/resources/static/images/releaseNotes_icon.png create mode 100644 service/src/main/resources/static/images/userGuide_icon.png create mode 100644 service/src/main/resources/static/js/index.js create mode 100644 service/src/main/resources/templates/index.html create mode 100644 service/src/main/resources/templates/query_help.html diff --git a/pom.xml b/pom.xml index 088b40e1..86f56704 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ gov.nsa.datawave.microservice datawave-microservice-parent 1.9-SNAPSHOT - + ../../microservice-parent/pom.xml query-service-parent 1.0-SNAPSHOT diff --git a/service/pom.xml b/service/pom.xml index 7c1bef63..770ec34f 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -76,6 +76,25 @@ gov.nsa.datawave.microservice spring-boot-starter-datawave-query + + org.springframework.boot + spring-boot-starter-thymeleaf + + + org.webjars + foundation + 6.4.3-1 + + + org.webjars + jquery + 3.3.1-1 + + + org.webjars + webjars-locator-core + 0.30 + gov.nsa.datawave.microservice spring-boot-starter-datawave-query diff --git a/service/src/main/java/datawave/microservice/query/QueryController.java b/service/src/main/java/datawave/microservice/query/QueryController.java index 8780d9cc..25a65b4c 100644 --- a/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/service/src/main/java/datawave/microservice/query/QueryController.java @@ -6,6 +6,7 @@ import datawave.microservice.query.stream.StreamingProperties; import datawave.microservice.query.stream.StreamingService; import datawave.microservice.query.stream.listener.CountingResponseBodyEmitterListener; +import datawave.microservice.query.stream.listener.StreamingResponseListener; import datawave.microservice.query.translateid.TranslateIdService; import datawave.microservice.query.web.annotation.EnrichQueryMetrics; import datawave.microservice.query.web.filter.BaseMethodStatsFilter; @@ -66,6 +67,9 @@ public QueryController(QueryManagementService queryManagementService, LookupServ this.queryMetricsEnrichmentContext = queryMetricsEnrichmentContext; } + /** + * @see QueryManagementService#listQueryLogic(ProxiedUserDetails) + */ @Timed(name = "dw.query.listQueryLogic", absolute = true) @RequestMapping(path = "listQueryLogic", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "text/html"}) @@ -73,6 +77,9 @@ public QueryLogicResponse listQueryLogic(@AuthenticationPrincipal ProxiedUserDet return queryManagementService.listQueryLogic(currentUser); } + /** + * @see QueryManagementService#define(String, MultiValueMap, ProxiedUserDetails) + */ @Timed(name = "dw.query.defineQuery", absolute = true) @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE) @RequestMapping(path = "{queryLogic}/define", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", @@ -82,6 +89,9 @@ public GenericResponse define(@PathVariable String queryLogic, @RequestP return queryManagementService.define(queryLogic, parameters, currentUser); } + /** + * @see QueryManagementService#create(String, MultiValueMap, ProxiedUserDetails) + */ @Timed(name = "dw.query.createQuery", absolute = true) @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE) @RequestMapping(path = "{queryLogic}/create", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", @@ -91,6 +101,9 @@ public GenericResponse create(@PathVariable String queryLogic, @RequestP return queryManagementService.create(queryLogic, parameters, currentUser); } + /** + * @see QueryManagementService#plan(String, MultiValueMap, ProxiedUserDetails) + */ @Timed(name = "dw.query.planQuery", absolute = true) @RequestMapping(path = "{queryLogic}/plan", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) @@ -99,6 +112,9 @@ public GenericResponse plan(@PathVariable String queryLogic, @RequestPar return queryManagementService.plan(queryLogic, parameters, currentUser); } + /** + * @see QueryManagementService#predict(String, MultiValueMap, ProxiedUserDetails) + */ @Timed(name = "dw.query.predictQuery", absolute = true) @RequestMapping(path = "{queryLogic}/predict", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) @@ -107,6 +123,10 @@ public GenericResponse predict(@PathVariable String queryLogic, @Request return queryManagementService.predict(queryLogic, parameters, currentUser); } + /** + * @see LookupService#lookupUUID(MultiValueMap, ProxiedUserDetails) + * @see LookupService#lookupUUID(MultiValueMap, ProxiedUserDetails, StreamingResponseListener) + */ @Timed(name = "dw.query.lookupUUID", absolute = true) @RequestMapping(path = "lookupUUID/{uuidType}/{uuid}", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) @@ -125,6 +145,10 @@ public Object lookupUUID(@PathVariable(required = false) String uuidType, @PathV } } + /** + * @see LookupService#lookupUUID(MultiValueMap, ProxiedUserDetails) + * @see LookupService#lookupUUID(MultiValueMap, ProxiedUserDetails, StreamingResponseListener) + */ @Timed(name = "dw.query.lookupUUIDBatch", absolute = true) @RequestMapping(path = "lookupUUID", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) @@ -140,6 +164,10 @@ public Object lookupUUIDBatch(@RequestParam MultiValueMap paramet } } + /** + * @see LookupService#lookupContentUUID(MultiValueMap, ProxiedUserDetails) + * @see LookupService#lookupContentUUID(MultiValueMap, ProxiedUserDetails, StreamingResponseListener) + */ @Timed(name = "dw.query.lookupContentUUID", absolute = true) @RequestMapping(path = "lookupContentUUID/{uuidType}/{uuid}", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) @@ -158,6 +186,10 @@ public Object lookupContentUUID(@PathVariable(required = false) String uuidType, } } + /** + * @see LookupService#lookupContentUUID(MultiValueMap, ProxiedUserDetails) + * @see LookupService#lookupContentUUID(MultiValueMap, ProxiedUserDetails) + */ @Timed(name = "dw.query.lookupContentUUIDBatch", absolute = true) @RequestMapping(path = "lookupContentUUID", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) @@ -173,6 +205,9 @@ public Object lookupContentUUIDBatch(@RequestParam MultiValueMap } } + /** + * @see TranslateIdService#translateId(String, MultiValueMap, ProxiedUserDetails) + */ @RequestMapping(path = "translateId/{id}", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) public BaseQueryResponse translateId(@PathVariable String id, @RequestParam MultiValueMap parameters, @@ -180,6 +215,9 @@ public BaseQueryResponse translateId(@PathVariable String id, @RequestParam Mult return translateIdService.translateId(id, parameters, currentUser); } + /** + * @see TranslateIdService#translateIds(MultiValueMap, ProxiedUserDetails) + */ // TODO: Shouldn't the case for this path be the same as the singular call? @RequestMapping(path = "translateIDs", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) @@ -188,6 +226,9 @@ public BaseQueryResponse translateIDs(@RequestParam MultiValueMap return translateIdService.translateIds(parameters, currentUser); } + /** + * @see QueryManagementService#createAndNext(String, MultiValueMap, ProxiedUserDetails) + */ @Timed(name = "dw.query.createAndNext", absolute = true) @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE_AND_NEXT) @RequestMapping(path = "{queryLogic}/createAndNext", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", @@ -197,6 +238,9 @@ public BaseQueryResponse createAndNext(@PathVariable String queryLogic, @Request return queryManagementService.createAndNext(queryLogic, parameters, currentUser); } + /** + * @see QueryManagementService#next(String, ProxiedUserDetails) + */ @Timed(name = "dw.query.next", absolute = true) @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.NEXT) @RequestMapping(path = "{queryId}/next", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", @@ -205,6 +249,9 @@ public BaseQueryResponse next(@PathVariable String queryId, @AuthenticationPrinc return queryManagementService.next(queryId, currentUser); } + /** + * @see QueryManagementService#cancel(String, ProxiedUserDetails) + */ @Timed(name = "dw.query.cancel", absolute = true) @RequestMapping(path = "{queryId}/cancel", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) @@ -212,6 +259,9 @@ public VoidResponse cancel(@PathVariable String queryId, @AuthenticationPrincipa return queryManagementService.cancel(queryId, currentUser); } + /** + * @see QueryManagementService#adminCancel(String, ProxiedUserDetails) + */ @Timed(name = "dw.query.adminCancel", absolute = true) @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "{queryId}/adminCancel", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", @@ -220,6 +270,9 @@ public VoidResponse adminCancel(@PathVariable String queryId, @AuthenticationPri return queryManagementService.adminCancel(queryId, currentUser); } + /** + * @see QueryManagementService#close(String, ProxiedUserDetails) + */ @Timed(name = "dw.query.close", absolute = true) @RequestMapping(path = "{queryId}/close", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) @@ -227,6 +280,9 @@ public VoidResponse close(@PathVariable String queryId, @AuthenticationPrincipal return queryManagementService.close(queryId, currentUser); } + /** + * @see QueryManagementService#adminClose(String, ProxiedUserDetails) + */ @Timed(name = "dw.query.adminClose", absolute = true) @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "{queryId}/adminClose", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", @@ -235,6 +291,9 @@ public VoidResponse adminClose(@PathVariable String queryId, @AuthenticationPrin return queryManagementService.adminClose(queryId, currentUser); } + /** + * @see QueryManagementService#reset(String, ProxiedUserDetails) + */ @Timed(name = "dw.query.reset", absolute = true) @RequestMapping(path = "{queryId}/reset", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) @@ -242,6 +301,9 @@ public GenericResponse reset(@PathVariable String queryId, @Authenticati return queryManagementService.reset(queryId, currentUser); } + /** + * @see QueryManagementService#remove(String, ProxiedUserDetails) + */ @Timed(name = "dw.query.remove", absolute = true) @RequestMapping(path = "{queryId}/remove", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) @@ -249,6 +311,9 @@ public VoidResponse remove(@PathVariable String queryId, @AuthenticationPrincipa return queryManagementService.remove(queryId, currentUser); } + /** + * @see QueryManagementService#adminRemove(String, ProxiedUserDetails) + */ @Timed(name = "dw.query.adminRemove", absolute = true) @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "{queryId}/adminRemove", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", @@ -257,6 +322,9 @@ public VoidResponse adminRemove(@PathVariable String queryId, @AuthenticationPri return queryManagementService.adminRemove(queryId, currentUser); } + /** + * @see QueryManagementService#update(String, MultiValueMap, ProxiedUserDetails) + */ @Timed(name = "dw.query.update", absolute = true) @RequestMapping(path = "{queryId}/update", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) @@ -265,6 +333,9 @@ public GenericResponse update(@PathVariable String queryId, @RequestPara return queryManagementService.update(queryId, parameters, currentUser); } + /** + * @see QueryManagementService#duplicate(String, MultiValueMap, ProxiedUserDetails) + */ @Timed(name = "dw.query.duplicate", absolute = true) @RequestMapping(path = "{queryId}/duplicate", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) @@ -273,6 +344,9 @@ public GenericResponse duplicate(@PathVariable String queryId, @RequestP return queryManagementService.duplicate(queryId, parameters, currentUser); } + /** + * @see QueryManagementService#list(String, String, ProxiedUserDetails) + */ @Timed(name = "dw.query.list", absolute = true) @RequestMapping(path = "list", method = {RequestMethod.GET}, produces = {"text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) @@ -281,6 +355,9 @@ public QueryImplListResponse list(@RequestParam(required = false) String queryId return queryManagementService.list(queryId, queryName, currentUser); } + /** + * @see QueryManagementService#adminList(String, String, String, ProxiedUserDetails) + */ @Timed(name = "dw.query.adminList", absolute = true) @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "adminList", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", @@ -290,6 +367,9 @@ public QueryImplListResponse adminList(@RequestParam(required = false) String qu return queryManagementService.adminList(queryId, queryName, user, currentUser); } + /** + * @see QueryManagementService#list(String, String, ProxiedUserDetails) + */ @Timed(name = "dw.query.get", absolute = true) @RequestMapping(path = "{queryId}", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) @@ -297,6 +377,9 @@ public QueryImplListResponse get(@PathVariable String queryId, @AuthenticationPr return queryManagementService.list(queryId, null, currentUser); } + /** + * @see QueryManagementService#plan(String, ProxiedUserDetails) + */ @Timed(name = "dw.query.plan", absolute = true) @RequestMapping(path = "{queryId}/plan", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) @@ -304,6 +387,9 @@ public GenericResponse plan(@PathVariable String queryId, @Authenticatio return queryManagementService.plan(queryId, currentUser); } + /** + * @see QueryManagementService#predictions(String, ProxiedUserDetails) + */ @Timed(name = "dw.query.predictions", absolute = true) @RequestMapping(path = "{queryId}/predictions", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) @@ -311,6 +397,9 @@ public GenericResponse predictions(@PathVariable String queryId, @Authen return queryManagementService.predictions(queryId, currentUser); } + /** + * @see QueryManagementService#adminCancelAll(ProxiedUserDetails) + */ @Timed(name = "dw.query.adminCancelAll", absolute = true) @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "adminCancelAll", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", @@ -319,6 +408,9 @@ public VoidResponse adminCancelAll(@AuthenticationPrincipal ProxiedUserDetails c return queryManagementService.adminCancelAll(currentUser); } + /** + * @see QueryManagementService#adminCloseAll(ProxiedUserDetails) + */ @Timed(name = "dw.query.adminCloseAll", absolute = true) @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "adminCloseAll", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", @@ -327,6 +419,9 @@ public VoidResponse adminCloseAll(@AuthenticationPrincipal ProxiedUserDetails cu return queryManagementService.adminCloseAll(currentUser); } + /** + * @see QueryManagementService#adminRemoveAll(ProxiedUserDetails) + */ @Timed(name = "dw.query.adminRemoveAll", absolute = true) @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "adminRemoveAll", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", @@ -335,6 +430,9 @@ public VoidResponse adminRemoveAll(@AuthenticationPrincipal ProxiedUserDetails c return queryManagementService.adminRemoveAll(currentUser); } + /** + * @see StreamingService#createAndExecute(String, MultiValueMap, ProxiedUserDetails, StreamingResponseListener) + */ @Timed(name = "dw.query.createAndExecuteQuery", absolute = true) @RequestMapping(path = "{queryLogic}/createAndExecute", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) @@ -351,6 +449,9 @@ public ResponseEntity createAndExecute(@PathVariable String return createStreamingResponse(emitter, contentType); } + /** + * @see StreamingService#execute(String, ProxiedUserDetails, StreamingResponseListener) + */ @Timed(name = "dw.query.executeQuery", absolute = true) @RequestMapping(path = "{queryId}/execute", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) diff --git a/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 618ac195..fb438ec8 100644 --- a/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -155,7 +155,9 @@ public QueryManagementService(QueryProperties queryProperties, ApplicationEventP * Gets a list of descriptions for the configured query logics, sorted by query logic name. *

* The descriptions include things like the audit type, optional and required parameters, required roles, and response class. - * + * + * @param currentUser + * the user who called this method, not null * @return the query logic descriptions */ public QueryLogicResponse listQueryLogic(ProxiedUserDetails currentUser) { diff --git a/service/src/main/java/datawave/microservice/query/WebController.java b/service/src/main/java/datawave/microservice/query/WebController.java new file mode 100644 index 00000000..f12a73a3 --- /dev/null +++ b/service/src/main/java/datawave/microservice/query/WebController.java @@ -0,0 +1,31 @@ +package datawave.microservice.query; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.info.GitProperties; +import org.springframework.stereotype.Controller; +import org.springframework.ui.Model; +import org.springframework.web.bind.annotation.GetMapping; + +@Controller +public class WebController { + + @Autowired + private GitProperties gitProperties; + + @GetMapping("/") + public String index(Model model) { + model.addAttribute("gitBuildUserEmail", gitProperties.get("build.user.email")); + model.addAttribute("gitBranch", gitProperties.getBranch()); + model.addAttribute("gitCommitId", gitProperties.getCommitId()); + model.addAttribute("gitBuildTime", gitProperties.getInstant("build.time")); + + return "index"; + } + + @GetMapping("/query_help.html") + public String query_help(Model model) { + return "query_help"; + } + +} diff --git a/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java index f54141f2..0717cc5b 100644 --- a/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java +++ b/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java @@ -14,6 +14,8 @@ import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.web.context.annotation.RequestScope; +import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; @Configuration public class QueryServiceConfig { diff --git a/service/src/main/resources/static/css/dwdocs.css b/service/src/main/resources/static/css/dwdocs.css new file mode 100644 index 00000000..0027edae --- /dev/null +++ b/service/src/main/resources/static/css/dwdocs.css @@ -0,0 +1,17 @@ + +/* Text-level semantics + ========================================================================== */ + +.pageHeader { + background-color: #3c3c3c; +} +.images{ + padding: 5px; +} +.images .icons{ + +} +.images .menu{ + +} + diff --git a/service/src/main/resources/static/images/api_icon_finished.png b/service/src/main/resources/static/images/api_icon_finished.png new file mode 100644 index 0000000000000000000000000000000000000000..d9a231a79c151b8895a902c7287c9b4d5623e39d GIT binary patch literal 97067 zcmZ_01z1&Gw>C^kNrN=F>F(~95)kR`?(Xgek(3Y+kdW?>4y8dry1TpKTN{1e^Zf6B zj@RY_)>?C}IY-{(9up(p$x9+5;v+&pKp;y?iNA+{fa(Q*!NUSq-k7V^0^cB=-b;!? zlnoQ^0KXvEOKCVkK)l2Re?dZ|XW&9WK!3GRR(DpHd&_HNXTxYVJPiTC z@6HSSXk+4R0CKmnwsqol7a)Cpf*1H1e3^+9^!$jkl>n)_+&hq%oudhegOQz)nN$!F z1Oo9p8k_RI7k~42IPjYQskyVWJuefJo0}V>8ylmYqZtzm4-XF$Gbi^q-e_#C90I(W(pXH=3 zZee3$`@9z}7G{2?|Mkv)j})`BwsTanH!v~*3+#Ewzb^gv)6ehG_&?wCuRDK7@-u;L z`CqpE$8nyo0-hs?2F0U-<_EiR($4!NI+;ElDh(3hRIpp7QQiSU^^ zGE(hToM$bu4Z0{q`7EN1&EA}*mDM-9gFQ(294)BaaGOa@8v#q3+DHNZ05p2LJ(cq5 zNO$FL%OQl)oRFer3u#wZ+55o@LxZMbu3TRCSqhZRXZ$0B{3C9rTQ@^~k0WR1kpVDL z{xDMQY5qm*Pfy7yFXo11hSt8Za^$_A7^IyTrE@V+rPU!YrLC4q(=-?XLb5Ir=3t6vrYjb{ z>0hJFH$p!t9i&uBR464dHIfQ;p|0s^uD%d^q7QfaGcOqwR=kO(3 z`>lKaC$x6b;7M-!M#GhZ*ZrfL{x39r;7?i<@$g})E71=L36M0VFOXj-aQFnCnIPGq zdpLuofr|@?xE63?Xo~>~U`J0?(_6CjzDo9_ozTLDTZ02MJ7T}ge(2j1?M|8Uj3pBa@+Wt zMT|M?-W=`;OacUgF;{?xg6iQKLK;;bcA@Zf zM)ZbMc=d;z@}#;KCfDuwnIdlt+xwBWm8XJ-O>omhy359J*=q|#;|eQje*bnJKthU2 zhF+<9Rs3~aOJ(L2qKXFxo~3-B6ybHSyQ<6)UIH|k^Lcb{&LwmXL3G0@*&iDSRSjm1=X&Hqidu>A1 ze;b-ZWZp&`i$1>K0>NqhP5utSY-o!0{mqZpwUTCA>6ihpg_`Yt9M@f*{ABMW^X~5_ zo^@XrfXLMlB*_gUx9-02hL!Z~g)cTk%tkFORPm4+hDMcv;*kdIh)RKuP6Dck!rtT| zb)~0xjT}B-iTM^!yy8ZK(81U;ZnpVK)jMc3D5)DiZC7bw+)8mJ~DPTn8f3%5$`Us@vZI?H8maaY3U@P z$k9?DLL9XD-;xIqlKNMcTTXv`9klYJo0gwQbeZj|4#q=ptq$(Ql}8J@UoTRTS(Ct~ z)3h4gmpnpf1oDvfeONsW5vjYL%bMfde?ejbnkvNDKM09!FK@Xv<_MEDnOweP1BEAo~tVK+lT2HQgr* z{<_66p&hYOqsmZb>y!4F#1h|-z5ugqCRCoY&y57fJ@l()=j)*K^@2OB63x9WZ)Ke4 zy(;m!du!Y+mCYPa>Vtz8cs~@teroQ41O>b-&7n;ENKmYsnKJ`&Z+e+h)*qDu|6D~~ z)zvK&|Ju`CxG*TKI>Wz#=oOYn@rhS@tjX-M&+#{D3)OHbTQ0<`Wh8G6-++p14{%^% zQsAS@l!NiT-~UcQHtRan{;RoqdCi@&`&i#|1&#!Q*@v;$UZVWYszggyf-F`A7Ng$@ zs)AB);+-#h>5r+e4A(mS>{~&{BYo5iKfMW+l9-lQynjb#XWuJ&Wnr3LNQ3AAjhgjX z7?PaD_%D?^uT_7?kET44yO$afeUOFg_>6AzNXA>*nfX&&%&%W0e!aj$O!)pX>SO54 zYm;$;>mxc9j_{vH-D_KU*9=c0evI$pmg-r;eH@ekxnN;@W|k+>K1_#!mC@Jr>EV`B z!|5Ek618q#60VQyUqWKJS1V<@!e)Pp>G~=WzoLeL3xdSMX8f(=z{~mRMS?e@ARTN& zRR8|#_O=__iFa;HWfqUjtN}~X{xG3g5Fsa{+0efX#oG|mk`g#mBL zm;J2wne)D`A;IgU*>53*uYVArHY3aWbQB_Vni*C6X>mU2pb-pph|xYXFG8Eo2g6{H z=p$Fx8Ew(CTQ5mGDq+k}a-!l-nvH#39J9q1=w?wg+Of}sVa@jEKv!_P@^Ei>V8owK zy!z7p`_Q6=qRFW*e0+oO(nbN^kpM#~1BjWcrKyQ97^L7lMKq4c6ykCX_glYx%D{Re zIM)4YH@#^>DE}rV80F_%FxFIPZ!1_;!hJj-b#6%Hl1%b?r(a3>^wY?7VYhsy(FL_` zixPP30a)%K99p6@ND=BxO_iwFL+lBfNrc@K`cneqzE%UaY7U>KXh7Q{tC$O=r}icQ z?fMy^Qog;D*dShNJeElQ4x2cNhFZbUi{zE=QI2u08RprMJ;Zw&=~&tmFo zqRxwfiwJeLJ0MhOHJgN1zirRf4l7?^thy;++TW1aIu^FNf09bhTmxh5j6al@_Y`pX zEg)8Y;!OA(G1ptCnbq`UKO`GDcV~=#WH6+0Al8Ut$4qV1_CR4O;YMYf&_Q^;yawld zopmC+*W-qpC&M4AND&A zkN~@Eh$9Lboj8;QDF7cRNBQ5{qQ%FucN~tofh?IMV|r05R9{;bxYn0_a=R;mevM?W zWeSE@m{5IX4sKiZD5-49O{X6u~%=UDj=)&&5oZfS#o2cbhf zPO2Cozq+++X}TvqO`)n9I^>_VyhdMK_u;Jgd8F2R=DO(o$~C0|r@P0+#TE|%cncv^E%Nt>REjgimT$NCRj+z~;3ox#-Ppdx zJ?)Fnv2Ahn)YnlaRRhxhE!Z@Ig3-x8f20U+-7P%6xLbwr7?oiY)*qCIB9G?|5NT~4 z|J9ZA`4-`Y<7!J9+Y=Bqx0ZO!V%E^M({hH7yMD4_3dIK83vN&2gwZ7ebAFK3SGeN~ zmaKB`|K%)X_DlAVXJyebs9hNUasDfdl?W6H%zeoldH9{I>~>M_$9B&kE2dSBjhKCe zLx?0rz+P#n^yA{8-OA~f!@Na1;r8)}wtVBzZ;Sk9FWoQ~WAUUxeF(rJt`STxRxyvu z>0iGdztD_+P0Oa}v9@(6jjW~3?P^Y1ks1ag1+sy;w10a6qNHe=Ho|pz++iS?P*o=m zhhhB)I9q5IE;d~)%sn0LC9=nm2U;VpC&QS$ONI??tyzc{pnWOW0D-g+%&Sh@`Ikqh zFtt68ctfu#@(AbQ5RAzHom=x;(Vr1x9t*KpmcJ7*XAD)iBW8Teg>kFFJjCE~32U;` zWHs8VgQ+**e6HZ$LpiB^jHNxsx`(&L#yylvqI{1Fzb=o2dS#_hZx-Akf~;uE}Gmv`LKuV`A1Q|M{or z&sUa6SFSt4hkifV{WfTjbVu&=OfwadDQf`?t2B_}_1X~Xv36tQO^WM=Fg+!1bS|_| zv>u$x(FdzS>jl7bpdP#H&7wNYF_DP)rHN6&h$B91llY1F(UCbk< zznGYGXD(utRZw*F`F(^u_Baf**g+d~z!E>$IbmVxM4?!Kg&O(WbKD`&tLyO0yz6DM zcX*wqZ&Ru}6FRNl;O;nZhhW=f4a`R#KMAubpz*zp_!kb6y>iw?(>RkJckRxP(lY*+ z3Q&x|L^|!vym#>GM(9>qCb(hO8No6m4*Zy%-QdvKD5oFS>%cWh2E-=AvFNF%bxxuw z#wD&Bheq<>SG+?1-%hCo37-Y*=bOP5ik#=jnEx4L)Z$-5%c30lL zdsqje#6lWV!jxtfIy4bu#!aQpk;dsKnG9i0bGixGX6{cA@$;Gds zYD>HeX>=BXt@|z$CuKO3E)^z5eQVq`CtgOZn-M?H=mvGBs*?3}XHzbnRVWb@r3Q-f zQ|4kgRt~&Li-BQ{1t#mw#K5K|hAgR;l4^s*UQIhWy(mcACDUGa!j?y#M_n>oPCnS6Ipn%{IHuS=3C?KQj-Po^^~jpfyza4jyB) zdZBzdNQ#JVi}3Q`V>~3_&+M7L8=0*cFsYBG1cyHiDS!W`%E~TL38HF}`0~J*oG(vu zH7M{qCI@diTm~%bYA&*~d89yWV35}9p-;1lre4uxNNL;K<_{t!pwhC#@LnK3-`%zJ zb6E~rl=3Bco)woJ(l5xa?Q>3Yu3euR##~w*f+%!XGl$Seb>B_W|G?FAZWv=UPXXh5 zM2+FtLv~eMwBnkN{EW*`a!1x}V(T zdg|Yb=|oA3Udk++&#|a(W=JSO)t~8RLNJ)(VE`#lEHD$!g>1RJoPTgMns1rcp;(KU zkJy9Pr9s`f#tWNCpwJ?g*2Y67G^Z}#oV)}ZOuTR}TB@F&Yg2)TFT#qKyI)A4ux@$Ut%M7Q+{jf*$_{xv z3b5s!&x1zX7{}#ADwfV+*l7hGc=toguU3fd&-SO1Ty;RdTdtGw3@!A*8Ud2A@-r$C zw0kj6H^_sA|H5Iwf>vyzM(Z4E#QW)s4d2?P2`R#tRKT<&A)T& zIB*6g@d0Yl@eh#-+Jj$y{1zst^(dBa z5v6!N{zwav_rpim?%46^3sKC&h6yhM`6pm?RS}MUWiXZm-djeSO`qa(esWA41}72A z`D7%aKYT6YwfBSaNDG1a@CGQ6ET|jFxS*;D zyiHydG$G^6P0yc-ciRPV-1NC8x$21%wtaZ^-F0}#hH)Pv)s^f#-n`oqjN6p^N-d=n zZ9O9M116>kX%HT;$wZNaJ9~aLc8Hgu|r zxR~vP`-hOpp$pqPBmO^Et}-RFO$`TI@wBZw?`Wfu%$W;z8MZM-S0`821}ac{z*HMhbJ0&wm)JzAg!+ZFh^x@7JQ9Zb9N74WRyI*fMRngfp~f{l3)-xwwTeem@+X z5`OYX3xdHQ16&K2YBN$yzoYn6x}aboN=^T+_eUtXO-Vr-4r3__Kf2IsC2%kSIGeQg zK-eX`f}#avqk-)P!nYR*rs+8J8BVwhKdz!Q5Zu=ZR?sG{_Ey%SUJ=KGLRZZdxGxx-}~*OI0rW z(C!b8cknoDbHI~HVK0&#vI~k7<{XBzHQ~Ht%R4~_ZO-y$^v^ngKxn{hUw z!>E-L`LpF!y&-Ph5jXkOn;fUiBCA;@sD&XXyR;l$FU)Iz#`&Y!c>fxa?}3pNp%%I* zCm3pN`s($~nn6P;Rmihqzdc$s=3mmqHCF7hz!=BgeJ*;y%e}r%{gSG3fC;~5$B;hT z!b=%Y0(Ma_05AtiDz&#E?u~GqsjX`VNUPTr<)*$fdPLVC@GgYOAq5;mmZaMw+V44% zltc?W8}$^eizWGAEJ!M1@hKV;{@Ergyc7YM4KlB<5Qd%KxT*9Ug^hZP_gk_6DX1D; z$?@eYL72_*%b_>xhT7Cwf)TCg$2+W(aeJ2OEU_%meY)}1CFtqGLS=p#PD~(0Z&s&tR_Q%RO@`-UthqJJWXz2i=yHi=F)^htULX0MvZVbL z(Fb|`$DhSw917bbzfaIl+{zyEt972H#+~rpx<~qcc>_|<7%^MQKRhF#MeXz7@e809 z9HfXMYHeAdZ?V@HO6KGX%>FB)cv~p7sW?2{To(gTG!DAQ%G!EnlF1Zo6|Po1z^2cV zfPnZt%Ok~;$^7eel|}j$sPPv?lsMd2kpaqmi*0mb7?_3yf8tf4wo~0+ct@3YiL^SP zv?xu~*fEsaV$-ofBZvdRC$=-8jSk$-SrJ^qh!*fx@kYcqEn_pCl(V_bTSdz=haOYrfRK0|WL`&zG;; z<<`&LB}0?0vkN5o>`vAT;?}^{{3k3TQ9QQ;@wt~W*>v5{PTTz*7n&aG}4 zS$;=bwzmN|N}XMe=7Lk0^7`eCM{yR;wuQOzdyhy%ocO<>s2>+bcEOZ8S_8Z4ZuWgy6-Qh zBj@xXXkF<@LY?=1y}kI>ln$|&#k$T0sTqBCzCTiaH55YqUFbWtDuLLWZLG*u{#Ge< zo98482{j%ntrc2q?OYOD6EaRIGq;@_c5)-)kG*!!JO$zm>LPjUrbQNd*RM5**=UQ0!`3B>O{8e;zTv4MGFqJlc z38Twt`Bz*7#d~pn#pr42Yo%$=ESoHO#eJv`V}CTRAt^k|ZZY@n*s9AWQdUu} zoD&q#c}}T-M0mU%$x1lOzZ+(hrJ?m$8X9~5a#K1Obw2y_vP5fjSm<>vGPuSCe?SB9 z^ybt;ee5+$M!Y-d4v~-P29{>@U{XzXi9dFn2=iqqu?{sU$_On`1i|L(Wb!8Y3su@=z!T^b z2sN{pmYZQFJ+Pwg&W8&$Cua_MV%9LXMkCBr{%i+>6l9ZK6R@pQO*;%b9x8}$tF&fp z^qvvzGo1uuM$d0v{m7=-$?y5cnN6&mG>$}>sJ;(J$?f?WC~O}P+y3Sxf`GJvq^(iM zrEh2wOHo>GN(?2R@EhaK-g;2}kYso6A3|rg2YOTH9d}CArY?<3Y zm-|18=7O?MOO4@W3gC{+i>nbb;I6kb}!W*>J}w+eq(^F7kg?WCex0dTxy5Kr5D_+5>q~ zw!%_KpEZB#o5BC~Co8Zm4cE?$$UQwJ7x;o^?VuV5`5gg=S-#XCwy?0=e$xV|N`Fnp zA0`;W^yer4v{Th~M{j#V($}8OBUn!#d+XQM4c=peE1ulM4dq|6uuI}d6chfh4{-~f zBf6x0o3v1U9e#WylC$-|Rrs4@$;ATdErEyZ){jsR-`+XY=)*w*@HeVLgUTsX#cV)GO7JgHuvcIMbf$+uO ze{wVT;J_i%p4-1>#=Q9(=R8T8Vw!ZjqPdBw;#sxw*`5!sifHqXg%oM4u6P?W`eKBB z_*c*Ezvnz>0l0U0O^1DnxfAb~N!L8&`dP<9r04-|sB@O#2BoTI;o1_Ps7lwN!&Jc~ zO?0O>3%}ageBR>%Cl3DRNo*+da)pk7toHZVTmI9ztywr!nbLT|0BD7l4^%?OF1^%= znr3s#9Pf0H`^j#(YpJOh`#)(`7|H_lPD{@OKodjh@NGj{{ zkF)fK+~2=A>BR2%B^;2r|Ji#x8hXX7am|p-tofBa@&fJrim@mnLvNZrHtEmNi81+R z8dBQJJvd@A_8Bl9uSC!WD7HV99j+!gVNeA-k0es}_?A(y485M5teM|W9}Xq~@FKtQ z_8swW{HYfL>JIPchXW5a-~_juIgI|}uB-w6Z5{P`{mP`VRNP5$b|G@=$!*PF z3vL(-oqLgqAh{qpohg1!kZC2l^>gWO0roZiu^ECVPODU{fw@#(BSKL$|fY&;>jck!rw@BtTYb50CX4?Jgvse zy;U(v2O7hwr1DCQ!oakY#g=`#85Qq8@gQKI0%X%1%c(o-nWCv2{-&ZPP!{%0VKRgc z&HDi^%cHX_`XxP7i*Y^|ZDmpS7eXIxud)0r6y=s303H7=8GzFnLXd;LZj_;B4==^l z17vLk+4(muqNwUyjy?=;W1)ZU;g--ha^rK8*r68q8ey04i5kTs{SrBHRo(F)lMI*# zoUpa#hB-V$+maI{w1y*>NP8SFlN5++G*ASMT2GoP`zP3fk%e7Bq-zI9*9M=(iaiu{ zT+ST%V*f$D0L>^XJnL<7Cjs5?%Q4U#ZVZiaCt4m72+>%=F|!n&;~`CSzHw1cY>Fpi zbSVH#ZKUFj{6`DG=(nKxOTA51$uX2%5>V#O?Jg|Mj}xq3X;}Q`W^jUc$K_|@T-5C| zS_QZy==#~c`@c#6toSulE~NO&hUwq@<&v@uTlzAm@W%+`9`twn7QR787(>c!yNPJA z5chjH^2pscEDbh4{z>Yf0BzC={n{g5`1KiEm%xz?RBRPETPImFZ%3dIqzW_2ZXMJ+ z_!Q7D3Hf`1s*-9o0LWrD?mVlTTI*GKd4Gyjz%3nPA?7KVLr)&QPOD|^!d8NaE`~!x zJVKOs*FK2(mtp-_!L3c14Lvm?34iU3*%R*9M1gyaE#ZL0S@s83#|uAV_~*Q>2{4DD zg3qO?;HSp#qRDjlK?1-5!q`@?lk%^>)zX@x#k6M$@Gd003W3afl8jdtXxPXF6jS_F@3O{h7_mUAykC@_WowuE(*18gfvxhK3Lqf>?z zQD{--T6W#TLio^i(%*6l5JoP1xGLHZ#DIgOa_3=EL@tlG*M973y)n>$k%yJohh9U< zE&R$eY2jv{DrxrKrDHXH(=2lw4>(KlC#DNSCBTTYg}b65qvN#!WJawdFn*=J11Rf= z`NcwbMz&U)TKjF(8Y;HvGiK?hjGs@s{fVwp;r?LTJ4_pj@?;P9LMcFshHz&?3Kh0# zJt|>_TSASGr9Vl%vO@ZH`p`ibH20rew$22J1nI>m-g}h+uWpww=i;j)w+smmr_liZ zhF*J>33tIA5SgK!px?`s!Jgo^OPEZuJu1VN5DX1AW4Us(Q}y< zuL(ED7^4IbgzTSosi)Jhb{W5}T~13C?ljm6?tUL{o6!SYsiZRdKky%@0Q6De!wb-C zAfZqC7(c~f+K}S+P-KoFptXy^b_pZUn;$c9$hjn~xfaakygjRFvS5f+aEYB`{s7is zRk*)Jx}h>&Xg=ZqiFzz@h$xi@f0o#ZPq|LBYf|e;FG1GD-gL>;qL%^(CuebNEXq8- z4)3&4-dohKpYR2}8)m(ZS|>Z;UYEuS-kQ5C2f)$lOgFzPKTsF!ysB43ZM=GVjh*({ z;AOH-dI8?GF(}{*+E*PF-3#(^*nScyw15|1W8v3+>N9hr1Rmqk-yO|2o_)E!y-l1h z)yR81%YLFRr9=N%B}{0_0jc#X?|X84d2#W3Nn(#Lym$N6lhRa~H3&&_kzqkYU-t;E zzYDflcbmQ3U5rtH#gM`ec!9jJrJu3ZwyL&q-Nhc0`PoDH2~x$y zS8QV?-J-=mhOp3U>TRvNft>!Mzw6IIuwLU$U_clfDmE5cT3QYwA|fUR1}n#EdOjCi zZVN8cEafeykraeck&z##_iDe-<+2VG&lEP09@+#fO-|%U4xJZZfM%xUl_nIll^4znWO?&+;79v@ zzj$kXJ{hMRGy3sHt(z~>kiX{JRQ5~y@P*jD=LH7A{M1Kg+&w%z42JN9R$aZ@9M1F_ zlx1rDA|oS1p8{QpK@gfSd5BrLB?t5Umozp%+Sdd!Nm_V=i!Grijhx4@wzFZrzMMs^ z1VO^NpH{XvfLcGIZI2PoTBc;CM4F->tbQ0a2OnNy)HlqFl8IhgL1B7yk~bjRN~6|tGL*o!VMYJ?m6qrjc3h(&g)5)Kdm;3m zQ;2uEx@FzjO=V?=pRbnuo}BFIW`b)(t@#LVx(|W@-cnweJ_$3;0 zVEq))FdXy9!5H&!QS^Z<(Aj26{F-`WMO)Y&W~QiY{43JoN9}}WJ{;>arM1o*fDta> zj*i7%UsvaQJIr^)|1Cei$1xjC(fV#_lNYA&R{Ei*cC`mb4{JJdFQE`Uil64AD1Vf& z=^s&pdMRxAaFEF~dl@oM#sCnX=OO;>hQRyIBJ@P^{GBr)CYs&2ZONxuM}=Q+w80&u z077yI2dd249y8U$FdM7svet`@6umm$>*^jRo*OHpY|YoajJd8 z2jdT5ZCo3-CfE+w~fOOLL2*VcVvVpNW= zf?w0;axLQ{kSF+q10iE`DTW-`Y%dVY48W$8BAQmes(QZ7zsRF8XX#mUc3-WW;>2t8 zSBoD%E2Xg)FC6FVTpl(b(N@Mr#xaPoL+-?1%{z^FEZaJchoh4dIUs&>MJ?LZme4VFzBtcZ4Buut#$z*O$eLg0q%d-} zEdxU_Vc>wpr6@L&Y3*I<+eH$wWJO%bZ0!}2H7>iw77c^o$ZQ0*(zx!zfp^(sb51cZ zuwtJjKjBtap!Q`|o?<>tyNs?hx?esOdemXAU+L-dL*Y3H^0Ixehib2Nq46c?9~6zB%VlRc7? zZ#c^i(JGT$J{)FVCuuWSJ+xst3!Z~M@oUR;jy}QX&!rs zukqr#?2V_cTWhfS^q|l>i}UitRt!XW7hbJFiq3vC^k}zV46G!N*M`bf5wGSxnk1yu zlw^-{^7QR{k1Lts*R`b;j3*>Sg#PO0;sQlwds7!ZEe?F1OA0g%AMMYKEi_|*=9Xv7 zYYgiKWhk8z28*|ZCvsmU;U6wlAjSz|{t_mvUfpu&%s$4O>ya^jDiGVr?fqh~oQZFN zc`|7jC%A3^$>V!Vsj2Lze-vJb5FPwV`QA7mdQnO<@=jvlokvIROYB z23OeF=P=9r<5iPq`{x^cHX*z!E%NbVk2kW^eY4rhT=uQJ0WBAI?UCoj0j7e%F=-*z za}^+*g@$H8RR&SVC%jls0L_0?$4tCz3Y=*9K{lFaQlpS@Wu7&i7m}fL#=u!_XYcOA zEMifxHfrr&!^NTWkm5Ivl6I8!+|xTv@`vZs_|jU@wnY~eyhn7|75l2Eau4As4o_O2 z8Af~{ya3&1t>Lvfi7X}$<))hPBBIfg<_7YvF4E8@|8`TEMVq?%#C*J73bd6h7}5B^ zONm(m9W1^co!8<%WSyBu!6^*l{oj}* z2_4gSFzzAWUOID)93-|sH*Nccph&m(>qNiLi84J&A${&OWC`VtU8g|BG3J5@9ZdIz z0=p6Sfd+emIay(icVxAl>y5IR3b@I4D9_lbwm16r;9kr0>u^h8=T9)`^JjxZcOu=j zjw*cE$@o->9Z=Du{(Y`RzqD(>8Wsar^m)SNs(pLfP;;^|DS3+=49sNvUEJA{SW;ucL~_3yPO$Q4?6M(kw9(1q%)2y2NvFql<+J9UVujj&a1%V-)pHWe zpH+>Ud9c}H?*;Evco8EU%gL3hr6nZ6yvInJW<5ZZm=RYBWCVY1PHa=66d+C)Vd^L| zaM93$-V1I5bXMOE*-~MZ>EYa3x$^3QZ0DG>z~JyS*qk)JD&fnaCMX(9@=g}>wUTFGjTF_2%IE{ zsT*Gzmfvpl7}lTzBR^bfNG+tOi-|ga6zA6-NkW~5*3$%v``%_<%VwKs@K&F>2xVao zj4i19y62+@Jq-5s_=qk5<$O@k zGm)I9iF43F_5SCZ-n24)P9t8w3XPZVa}R< zcY+t4awe9dROgVQ_-^k*ulb)s%p8g|*5V@C^h1BTjS1JXj6u}8c%2b3h7?^u1?B9v z`#W7o$ADKTu;)nAy?j49`SBb;!eW;OGxFiNM}TphcYDs&3ryF0>B<(lTp zXNi3-?>?hK;ss4(_tbb^c1&=I@-F@86o#4+7p_S#Cog1;D!^aR76bxo%Btgg+`BzG z)Gme{{1nVuhA(h_`~TWj?qPR~s31*b-VAw_DO+Bg18GgF8DP;I43D+c<)ALi#UBMJuYY?S4bvC%x9${o8C}|J>lo>?zZIi|STfJu>gx98G8<}y& zR<7jC2IB#T(1`*nR5f?rA1ifg4p8tk=Q+vX-l-|?N13HkFcumYN-7P@Z!i)tr~i>j zJ#=+AzXhA))ccQvPDCq4^n?3hh*h$b#)owIFrDcZg=-?8Yw3qJQd1WacqpR9bkEed ziO7f)A}of_0AkPntr7Wi{VvQ8RkxuOOw^;p|9*wN5;yEyFDKg_TfjHq5JYH}e}3BM zOl=uf*3w4p57k_}X2A|NVmN2E9#U%DdG4f_cdQ7DTQ}h;a zAWq13LXEd;deItzQ*SS$R~?wxn$*8{ILd@Pxl$cPd{0*xE%z=Q?cuG4TL<>BVMT$)3lj^J-DRemI3hGfPXD5yd0 z^kt$SFVcXioL4xZ$&I^3LDLRWkM2+$56Q1l6r{h?PuQz{kqvu#OacwMB=^tJ_V23f zZ~gqGy1H8G@MV?0&LXADO4et%AQ79sSRzNp^&H0FbC4h&tdwxguRxXF{DQKFZ$|~D z4sqB@vmV*av0pcw^VJF=k+^|CQrgD*w>8_^qp#mzRd&D;VtNy6}2< zyi%sd7$OFAY&Tc~rX={1=#E{YIqwMSm^!Xdtfxa2Lb+kuWpE??6T@N*)PwU7vO9=?*-Vg{lcND2P)0>&f<{=nC=vNE*UDn@qCxk}JjzA>p!kKr>3UF+SOKC^piRw{LPtGf~=H-(54U(5sm9q8bjSg z`y;|j;`cE3`Yn?Ub$B^Lj$36<^)32%&=!$cVA}f~#L$n~GfKz9#PEKcj zD~t9}i}W!c!#>n6dhXRSXw~s^+s-%a#TOFIp@_|zSA2sxLn%G#yxL3^v*$p8M6FR7 z9R-eVDjX7xL8c?k?z_`#_#FIHvQ`a!ZXNNt6&kI`x#;M~>6iV%uB^!r__U2wFX z<-#vPq=Ma73X1^a(^Q!4dwW;jdYKNl^om{rQI)%d*c{>&z7&`NDvrRUf)%=I99gJ& zdc4z%>u#-xtNa}n9#d~3bi>Hmu13sj%|Z_)hEu}a@H-|WA`D@L;teLMxcIB!Hy6!E z-@R5skqLTzuUEokuG@ea5Umd!JQI>glKr)_9d(Gkk2@^e9xj#8%yh0FPru(Cdd*cB zbm_h@tHc<%I1G59q0NAlkqt_A4d?X0O*jL2h_{&3jtUcfz_usCzVrV6p{lCtU>U=6 zCqE_2`*Oxi!S8H{Yf|#H&zJQax!+`R^q0lO#WF;W{Uo~+*^lSRBi@^ALSIWuz0pHX zR&I$fzjIml-o9DK*+fD>`}*yQp(80u1oaknzLQB)dBR(5>EnLH4;h2#8CB+r8g}Q+ zRKc%uHcn8VD3ZUpRP+7b|#Nt@rON2q6^=az3#YT$) z14T1}N`XJo?>T>=?Unp4QDH?DAX;c?X@BK}5f`Hf-P=Q>I+Y8vF9lw6P$%(6OHsGi z(r>}t4sQpWFR~bR!|%<5}S7MBVWYX26jd@nX$$p{w zU3YQC@@%G>piz>B)*G~yuPV{^4f8q7J6%cy28Gk3cN#`uu~@rkcDtQaVohUtJkSR= z5aqg2(zgr2ZedQ9>)B}p2|ZpLMddIjNf5sgclXyi3WBdQ*$(qRuEIe#M>9;$*C|ud z2%A(os2&ka*5|w3&I_urel!7m==K%;ke%T)DQv6ZnB*gfaYY`!1%-b<;$>*K5s%;M zS>}W&o(qmzHmyB%yUSeoZQcFNh@bCOecQu-EhxU*szoD+j@*Xx>EkRFP1NWAs;&aaG+_F!-5~ubh7kw#;_QA(wWCsnZ z=&f>UKB==^M@TK~IP?D50hh@ks`l=st7-Ji1aRjDPZWYntr0aw1lolip8~z7M?v7? z*N46GHs6oxv<3_W@wk{IR3e_aarhiELs`3* z`ojI@K^!Tn&7Q=YnWN&P!BqFOCNa{wiUZoJB{#htP77c}Dc^5*OUO1ljMjQ%5LY#R zoCU$_SC$!Q86P41F3;Ppn7B#9h1B{YZEHZ=vOyv*Ud8VnO~^CudGJ)E;3FeKT`sij z51lO9BH+L8s5tOG@{BxlNSyp9&nwv~_(r1efyOcHc5|%VI-x`4qz1;hcUJ)1xm#+p z2RtqI!ycv`AH8cQYXWEv-l>s`&~4sfH{Ght0|k{*LH}tyLSepcJz$-p0t*K_)E$PI zh}g6>NGsAKL#&RG&D`4(eD0q6i;1`L6oS!p9Y4~FSM19frM&{!EC*>Lb|3=4V#N2N zB2PG&N7IhlBj16EFlcdyM)=wfA23Qt<`NBM$>1H#Z#;(6*r*_u)@IDHdVWp?*F1Hd zbjy6afV}VKr5EfBF+X-;KW5Fyj9w;57C%u+8l(8R{9602-0xwpWVP_&cCVbyS-I*u z98?{`Tqkx(JeK_RcLnEin*b@O#by(zesbvddxSjpe^>5)&5QkDvX>x9%mZgD+9sii zhoTw@mx1UVspm*lXBD)e99#v@dm^2Dq(StK$wSVMde;@+w_Nv_=5eyzw{{tbFZ3yU zD#XjdGr~c~$zfXubM;YtQN<-CUI-bly9B1HIN#rt3#_cV{L4`>6z!A+M2$TV6@vmyznQn_W$fcb){7?61A>$UdXPZOG`R~O; z4v0xI_9;RnHp?(U{!UX~tpl}8WjIy+(76rT;a;QQP6A%`05jr%!0&u*xK&#cI|O_D zA^CgT^;>LNM}o_gq*auQ-*6j3Skq_X;lB~R58;+p(~fC~8R>%)!wqldC;XS%On9&h zeA}K(!q+4_THxD|D+vd*C?eNjmWRu)1pFtCUlo?R!=zQ&TTUd(Tci=zZ>g!ITX8rw zo)T&v-r*p68}H{BjM-gTZW=u9w!>9gfIGeY{7~lC>x*Q$*Xr!Hq>(-ETa@X0PAxqu za(|bx#7@w~vXp*G=X0}V>psyAH*%mubGTCqXH=6vh&9$kbx-_@?7)BoyX1-3I~kj?H#wvWB;Z4d*A%_$o@=|? zF8|mn6|WY|1CZYwP8sq|`Hy`_fE`yK<@-*}`tGqO(13R;x5=Dsd@#VAB7OeDh_BRs zBa0|_7R5sey&#L5bdze;FgX0#eO{;%=hZP6nP(C;WniW|_0YKkIKeaM@XI#Q$gxQt zHKo=E9NZRyMKmo8Or?M)&EeadbmK!T5psEKG)iRLSJOotB|&yFs@xY7l%e zIF2p#>s{%%JxwvLRD zs(+(%CO$O}^*TLXHv_g9xDE`c!msTxB~jT<%MG}rE3YJz?5u_Ope>*CMt)EN?_}>? zmq`>{l+{iIH>g*pq0ko%R*^+(WE;{S31^}nDgR2DRj+UrP>#NAIwgJjt?=P?Th2f2 z=v8uLa5IS@ucGE(K71t)(q5+%-ZjHUv4}eIG7hU zyX(C^7!QBJBnyMTK$1c(g(%G+h^R|B(ExBw^WCa|)!|-d!7J2f&ezhbU@%$d6j%DA6ji8#lD_OByv6a}IFIxGX7!;DLl#O=+e{AoMF%^i zfRW0=>?95q|5P=LQ}WDYV&|w?X@v|Kt!O$+Qh!U~oe-3q45v~09ZtNj+GyUPJCo3< z;QIROkP*tNqpB?Fs8Yz+wGfI@c&tGqf+-4SceiH!@NeFEDR4EMtCbuPWbhU0!^_f@ zG}V%XIA2rlNa1|d{SwH9^buiyk=utP{|n&z7d4rhGPahRMm2K(ndPJ6@(Q61_~`^` zAM^oG48w#Hom1I{{GW{SqPgsBSQ)Xsr%qqx6dqRr@gS5&@DZ4GJbCK@k?~|2 zL(_U@(ha+q)xwX|I^Y@<_VXzvb^AGU)uvh+#n~6IJ6q}6LJl*O4-SF0q|t_+|9mO- z8%`Yz-3CAHZ!?7$Fd#8M^9~>5=O&BefnIEw1k#2vY6+q*FYC!(ZORc;g zb)i(SxN-H>8vZQz6xO!PFwfZp9ct845S*`Mc>YVp z?TwrgX?Q8Tm{@ZQd%OfxoZ18~9#fn)3Oh2@D-f@i8y?A}aIT@CPvXG$o)lwxdQ8TN zSGnbiJVW^di!5|LrRBxDv(aKiuVa}H*X zf@(n~k^iO*(~z4+MyXTSVU)b=EudJ2mG$+l{MF`o7h7?f1x}35&9SD$dXl6j-$-E! ziU@5OZI0ni+id`#0prlZs53$`+x8(rK|W*Pd^LVGwiAE!#~c^MsCZ+iKR~FHLKE$Z z;m;X^)}nK6$YJ*EuFMe2Fo4xckI3KA3qhPQyz&7#Jdz$xeS#F{Cr=318>{0}2znkXS;d>#MfxI zSPrAvw(9&}{eNRc^A3VXhIG49F&ph^eDo6>q)<1XCWa2jNgZ|~UV~E-Eyx#NtI$oL z*ZPg#?CM3n@*hPPLrSbfNk0rS%K{H!abU3YF%{jZ(~IZ5TgLyn$aebH;2S$l$6x*9 zZ=4?@RiJ$eP1n!xIQ3c$P^TF!jLEKE8M8h*NQ4qlp2~jdNYX>%y4a`@<#kKt- zzr}f{-g9oBe6#>lN~`far`ot$d;X@`2*=OF$1B307H6Hr#`tnNk&ZQ zm`UA>hNpKUw4PwZuW@CpS#d2IM_W*|1IfVoYtt+hy40e0KV>K<<==15r*1iTN3G*P zd9{_PnJ=RD{5WP+K(u2|qC4|0NA8J+!q;7Z2Sgq_WXn~8gvs3t+A9l=J78YlLBSBm zCnPl6K`cT&(eZiqsm5n=9U5|8RC{MwpK*Z%v(O*4k%Tmg5x}ADp|;*PpGa5LcglH( z6R9(29zOf1N(*dY45I0WrDP}|IOq>jSi)1FY$&d-fu&yPEbr(3D|v35Vy{_bVf^ys z1Rv*cAcAx^*))G07CwLSO8jC6u3#UE47yLD;2B>H+2hkfNsR|Tb!h$o3XpP5V-!Xu z9ko-*(=EQBUWGA6_DF6Q`<5tex3)4REzaN?OZK{TQpSR*zVbekn>FyvDEV${dWT~N_+_YoC4Wk4>c88|1!7>j&k#l{@6?t2?qfU#i!hV%x37ozBsr;vL#3Z)2vpVvI& zM6*3zJ-;s#K7`0klOA=XR*70K&}P|M3+Yl%S<9T*oZ*I(VV(|uLi^T@a{7KD=0H%L30o%zmC-Koe6E!>h_SH*Wc@~Y?4cHm(0qV>Tbn$1Y_+RfRTtg+dBn274>|V0>EwnArWv|TU`?352l-71xUym@xJE*wT*8F93b`O4&o z%NZ$3u@eWKq!zk^er-0<6UE#^%tLw_&hKs2$+D*V+!xzvj+y$dfR>*tFIhsF zM9Fy05zUMGQ(_dr&^ch>OP_HSQh02AD=ZQ;808{?q7F}}^DuO3CZ}de^QO4P-4g%pK9qe@oGyEFi6u!@>s~`pP{Bj zlWSYb`55@Wp?)gF?T{ldGB75cpCSXEAm3V3g6vaBc zz_xDLqOcT%Z5laDsN!d_gzvCV7~$WKOzzlE)o5xJl@++)4qS=H?qobqdAC{_0iM{P znbMqDzTWOzF#sYR8{8X9TxvrA>JQ;6LJ`A#{fC^t=*e;NT8RX==y>!@=4H*ri`lGw@ z5sGoZl;$SQ!|pTs^0_MYd?UBqpC6xJ>X-F73VpWz@l*IAXwU7ZUH+=8&N`Q40vVxovSYc+5|V{wUKm zHBy6LFAH$BCj~MZiheQE7d>Y`qbOH0Zu=9mI+QuQeJ0b>&QlRe+!HYV!u!PeQJl9! zBMGr1&2}x}cu(5KeiRjd7IHw^dJ|u@0CFbeC_iHE&hNfp%CV zpBLJ~p(U;ihPap7Q;$e|Z<+l2R^$ULC=J9wS-;M6uv4OEBxCrJ?8?q>W4h5XPN2`J z@`%&I+(VB*3*g|Ki22wV-aSselV{cB_ob=6n!Dmv`=B3#3~GX=1<=Uow&a8he5C|# zjXa{@V+oJM&8wq@>IAN0xrgr6Wh36_zbGgv&mjYX1qfhP*eDiS`$Nm^v*B!M$8qxW zNzaJWb0h*LR({I@L?|fp;PsbWhesAk;*=ADPDDAoA6UX6(7RI#-rytO9w4Pe&cD{^ zW0!ng0_g+B*Ss_OqdyFM5 zvvWHkY9Sx0?zE8H+5m7nEnM9@&FcCuJeB_it>|n0NT5jZP81Rxu?ITyu_q|Id7|h<@wgmIL1&QmWXr7fkTZzyerUw>;3|QkDmf_bh}A9=!`KL$FRBKT=UOZG(duwbTqA6e#V^ewvsX;y*Wd+_&OY0lKTTHg zW41#}@VVZT^)V1NtAHZ5hexMYB_@leMRSXxWy5t(12k=<(CZDOVxkpkEMNs5PrRDx z)VfTHqs$nKGT`7ZZasIz{B@ay#Ha(3RO;wTg9 z1R|_?a9VdQpcne`d%H-_o4e3=6}Y#7z{_EN`;_2_c)QdF0NAW*&Xsv#>wP>PZoy(i~6*fz~QM4rhd{goZK1{7I(qS{2Oer$d z;qWbWEkPCUzfHD{F`QN=tXVTGn&7Xe)71nn}8Q z{cd&xc>MzU#e-xUgn9IqTy@*Xk_2FJqm3q27bp3(Cg@elc@OkW-uSAe)V4R|M&#YL z|C>!Q%wdy?ICcSiPnAkm90a7H_F6RK2VO}C*;9$(Rkw?(gwHFgu&14F%CP-0h<3NY zfBsa|k}0qHB-i`fa#QY&{|q_N#$=o$lik$s7v`p?y)Fmi@zutaDg8>-MFI{>f#c7J zvj7je`*#jtdblqyJc!_LBx@cn32dh*@8;`l$aI*l2t_)l*$fvSy|*jXh=SuS@%OH| zWKSeH<Pm90 zhdUdMG+*rfHFs+5RC^AbIe+axamb9)WvbKIA~~;*nYC?l|EfhjeEzfB$%o*q;r5cL zy&7@rCA?aSrVYFVJp0#-lkcQD3iA+W#psVCKS7be{#*e%;fqHrJ-`0xq=2?x$LG(4 zuxh`A@7JI|dctMCDw9AbskZ&Y7+R%6ow3m3OUZ4pX-ul~+|w_`8So0phv3T7VS5d` zH(T_iPrW-1#eWy@$Mxla&ywPVcgPfG1`u<}WW)fB^glmj$eP{$GWXxZ&|k!Vwl@>3(4n?h%s7&q*RE@-BkI}u-=Xv)ixSWKPU7PmL4Qbw3Ko%E z^Szbx-TLPGjEk#$%DYPsC*x7?ZN(rM(x-lIB_bkH3vlc3{*8P5@+G3W7OLFu`1vUe zgf%~9tS(9(d&ADOsxG@^bYv|WX%Q>jbnHrn>? ztPRJ0Qh1sCA#ueSy>9k}IWUh+9b^mSfV(q;c|4|@;JGD^5+%Z}vlIZfco|6!!~Ni( z`e^_2VgtRfPtz=aN=xe)(1SU;5Pj1`P|EcJ(Su{lcU5lWh;UJ_-j9W7eG$0ee;L2y z7Z`1%A!5|Mc=R&if0YFoo_RP)&j}Uv8x5r|<+GDeP3-v%Xe}LhDTs18HC|L;%Pzs3T|0`GmeCK({wFZ+70D&FLkI_DO4c=(DpKbA4>nYVV4rmR&zac4_WiTA-uOLLh8r9^7E#)i_@%P;v zY;yOHqYZ7W9ALUZjb=iaE{5Fy3EDVBKdYs3&5gF`b&>x`i7sf9&2$fpnl8zRu{Rv% zdG@>F!1qu3S+shAp?1 zyJ*bh1oRn4R2ft6S=~tqZTpwv75s7PE!%97MW#Kblw8nASWn)%;SVEENhpQlU1^uN zovi(TnxMAOAPibMT8TRbJU!ye^5NoL{BJE6lE%v2PxV}=( zW!>Z>#Oa4*DdjXk?=UvZa*=RvPrihf^0Q6^ghyu^`FtImCmN@k#+Ul|nOcywV7)>B ze3)3x{pdTea5-fFCR6z7+5;nO>-3i7%AmC5H01X`kXT zXHS<2cK8p<_b0BYCR`>0)NNKQTG$nUj%(hRcamJ|V-gzLeLfwJ>K&l+%W_veN@ zwRwYVeQ9~w@xc6o<%Ud7*VN&mJK*3`qp3t++=9?}1dTfAC&ordZWfm+(268Jh3x4j zyqZY$PGk9f;i7a}GAe%4?V>r8cMCvF8P|v{u(UjTkRv{%Fu0X1e!>1n8m%G%w08O2@UcMGE0wTuW;+p~ zq|Jbh(X5YcZ9mLZ)2PcGevq}40D&q~*Aq+l#$=q|zu|KV0`ok=umCtU*2NxFqd#Ib?z0^E*Qhjv^oCQZI0^ns=C^gbr+td*+22;ECY=e@<07d+yX!SqtW6c#)QAen zfUa2Qg+E<14G$VVGM!S7z#bH}8q6{rorFLzX$mJH>I~BM@}F7$C3`%=OyXQhe(J7E z2P$#3d{HVG6M>`!v&S!ujlp(-02g5pdH^I&0c(~?aPSx2QhAL2DCrE(<*supKk4il z{SfrV=dP@3Hd(tq&@`UZD~+UaN)evajVvfx%QK`9`=xFU|3{cq-0?~?sFZjQ`K6hG zapun-vuE~WcG;NcGir>rfV~V%>D0o@GWoHzha#(MA;$FYYdI7{&RC(LDBMvI*1@Z3 zC1Mk7pz-cq_{zyUi)^gsSc$qUSf=HTQLVSMY@w9fowIjCd-0yHRgZeAu8BGkcIYZ- z)?XdLF0~9TacO)bpGQVQmGY7)0+N99)X^N0&u9KZ*SmPrxxEy==z+zIqk^J^c$S)^ z)^eOe5%lN+L%=d#-W+x}J(UW0O%`{IPBM_OUcT!Nj->U*F`rH$IfPI&?L--SO)Fp% z!pxKK=sa0xHz``F{gqKB+I0uP%D$eG;~n^$ZiFCVVd4D(5w|?YPpUL>3VpO$_r772 zwxm4oUUNL7yg?Ks+P*0y8GGJL?*6Au!+oJyA^a8z%R4t+5>h5l#WerREA2qa&<@TE z2O0=zG@ss4A;8!~ALKU6duh;V5|-wE@~X3)Q}sQC$WJwD1eKhQuoJhMh*mNbJohu6 zvuCNmPRU4%^LTzi{XcDj9~Fm!l*8Q1{(;J4$4;Vsi5o;?H?t}Saj zJ><@gnUi9;@6X$}Z_RjkcslHt@phtEOMTME9I%8QtO^)Ir+vjrhT3+1T|%L{i$e zIi;BDuioqtmriy@Y;;XbfXrH!_S_UVc|M?&6`@7Epkh9(Hb81vBzGyW+V@BF&nj`; zh}u}+Ko5yIuD9{)iXJ#bOCE!cUw~HO%z#7;_CZv(=M{sMwYBFpT`EPE2gUDTh|1D$ zw5fKSRJS-BTO4bVh}$3$Z@4lt-Gb2V*xkg$gcu=75{OM&#~g%j6V5ky4)T5)XOGR0 z_Xnif4PwAwQ?$i5Y@tq_P?CadizBEmQdzZ0Xz!J+=1B@UPD1yw)ZYlDP?(J~t?G1w z)g^{4-($p>&Tf<)G=n+@@SeT5#OO=fu}!LrF}c14!d=Hwd%qeMJpsg=r(zoSDanCU zmN{pW?52#3W|}$#XB55iwrZFUc68bAKMu&}39G*hI$xlTwxG_3XvwykP3t3{-#SI? z53He2wML_tiERcJoMrmi_LQxN(U;ip6l?yS`OI=_^mg@sfdwGHHjw^}66V;so&o;i zvuWsEN0KG)T`U9Wn(nyFF--To3LK53z?K=o=zP;~*7*_vMw~8bJ+waUD$!St?uCtY zT*3bEMK_7(0DzM4W-~wAyDEKP+*C^IOvB-rnd8(26K|*#7k4ji?MV5R2(MUlOh`+jMW)Xf67l5 zV&OB!RvLz&U8TmizR>*Ac<9x$Ih{j4s z1z>gl*(dit$l9M1{@mXPy;^ekc?tl=U^tIN47SIi2{1!`Y9(;si^01^>)ZoqZ-?c} zk2+$aHkzQ3s(3XW99#^73KGpHqgpuLQE}NlIk1U3CjN@s(`GIvYT7p_<$-CT9rL}O zEgeUi|4N_FE(3smM$=5AF01e6M&mhv!wQy)de@Uo6&F-Jj4Hm+FxjvYa;_*5$Xdx;zb6Dz7B{y6a}4og08I^8v|?SA7SZW>U6-{^fm=c*_b=yQ6-S9HHOChHQ*waa=1R(#M`36 zIyBQ@*MHAU7Qau}NhFyA=MUf2>h}DynV0O+(BIYd2R(z25G-aT-bxjHWp)vpf3u3B zRrT;$;$vT5o5K#j)8U4ei=mQXGBR8X+bTIUvix%b)28byqD{i<-}~XxyH7As6+mhv zDBjcFTXPRivWA1V!caupvdVC=emeB=#?^4zYoO6#XjC0w zX!u1gfFkj^PkuJ_--oQc6>%d^%Cb%Ub`vu$GfOMSz{4a(G7W5SnZUaR%*rd5w)z4m z(W}`3D$W%xJ8OH%FKG^%Y8{VWfUC8yh8> zUnZoj_)X)v4Ntw(_u}7STZPWDXiZP#2EV-jXP$t4g~Hpr;E|eX>(0PLe9WLrB`_B- z&N?34%t)w^%$U@~G34EwQJqYR@MefXePJzCX#g|LnuNA%0%1OJ?`c#IGEdEUGJYApGb} zcK>SA)ihuXC;Kr-O#I|*cz|Z(k9VVb-1S_*;Cm;wppRa}`U>nc3^A)N7N=pmoM_h~ z#)0)6wsu*2vR7-dJXXP*xrQMfpycrq2ISXq#^=iN0nh2BL&O^O>42_DCk}ERPcp(}*k?pP^ zPcmwOacpymP>_34QAaS2jOxEXZ;Hf{V3>TG>Qmfxs;a1M2u&0+v3sjk9D4ZWZ?W(9 zYpK_X+bTIW+NDd`8Ga4wz8L7c;`)%PCw}kpL}IrL6Mgn2(CxX$SfNTtp4eIgL4mCS zXgO$Xi7L)|F~)HsFZ>$FjLV)65z84D2WMauI#EjJ-)eYaqrD|J&#?>fEH~t>5KovJ zMY^$_rwdn?B^&0i;RZK`+XS2&Su%%y$r83BsJ$_*f#xs%*nU_`ho6cfpRnQJph+a4 zFzj6G^Kh)&8ej?SN3O?H-7}6NsfonlO4e8mY)3KEc!G?h#v8j%2d$-dyd0!Kge7bY z@)~Eb)VZ0bD2F z!U%g=HdE&M@|4`eEq5luAS@}#XX(aBjqaL-sY>x%;SPYPJ_wjxR_mwAsp3k>$JHUt zv6R393vX6qNC4yt<+C2Bz?CN5@ z*IPvlL_a4sQREi6<{FH>lNe4>5Hb5R%VSZ0PV9a9qRHycT0p4_bA{K~((;`kIurJ87D#K90K=;> z?Ub~ItE$v4q_pfhj$bh)m~l2uQ85iCQwz%so9=7eiK8Vk?z;WUHN)Y%TDLH`xdzin z(0uPmJ08RN?X7QaSavI>8Z&0}3QJTXCu7+@XF@x7*b`48QBev}rZ~?P>@_+7E5gBr_W~1J>^O(t`uT3bDa-IsiWOe~ASqgzWl8qoTA2+P_ zVy$4FUmYudtQHheTm%qu^t2`67#xZBAfU;Q@;aG6(tq7vD$nF-?N zqFQXk805l+?Yqrv2jgGuSInbg5S5GrI_axdMF_h;bVK#qz7W@|++X$!(v7jCK}xddb(> zTWR>LE;Xx8H6s-I;J1&-B@=0Y+A@#0+p~UYhzhNaK>EJ+{dI-H&Q-;&QmiYv%lc^B zyO+6Q?jsq?h*A8OkN>^zMWSDKOCNrJ?*io;T$J_pftxu0k=`dLTtowL%#4>8`k8B0 zt=}d0EQ%k-3If7+A4zjBQJFAv(L;KJjJd$7U_JOP^(AYLUAEm@y=lFc-(fKuV8s}m zOe3s+w4#@HlcxFwY*Z>sZ?>+K=R+H)?JL$&Im0|I@q!`}vtq;oGV-Z`s<1q$@CbA1roE%t!KPydCQ5a!J47$_3y>R3f zR4fW>uG~+wA-UQv3H1=kLD*k4lWunI{Zv#WfV+jsR?{PYzO?S84H(`YB=W~LB>K92 z5j~JLpJXeSw#l@{iB)51>v#U5g7{cL=GoTS9B7cPEYM;5`#bBQ)J=YYp$a*nqPH`; zPegHRm7?y*trfL3TP}>W4ZlS;WIVE^s`UN8B!l#4qtVr z{v$lz4s}({a^|B^!rhV;=ypmLv1-Qzt|f1b<0!;@At?w7tCJ^bb=eG_KM23hs(!gD z+EyN3dJ=kS5?ZDc9xxbdvaQ?YCM)UIXW9{^ZDRX{q5oi~%(DEc;++}xW)NO7C>LSz zvj6IMGb3O)%v2mm0>oNh4N^rIo0?vuG^QHXz|a*6%mqI49wY=q_OeR$o_r!lExxo9 zZa`t}M7hMhgP*X~Ab}1%a`UqP;V$?%{&lAU`OBo*Zj>Os6Daz)fP2cMrXW!3Q(R@4 zD#SIa*X}FPw+5<$T12FJ{(b)3ft4uZ<87JlZ{4%l%RC|nmOSPeCkq*@n*JM(?w0Uy z$(0ZrRJ8nG!9+4geV-Z-*_751f`uJum1aBb)k zNQ1!lO&NzuuE;IOu0g{?Elo6~3$9w<^$y#30-@I`mTYHFP%Y5gWU5?EWUbzw9Q*u8 zXu=i&mnPh7i!3na&pik>5`SRL7RMb^hyu{)zOZXdDO|?-?(fT5a<^052J#8{?u`NW z4W9`$Zcp%eRPRC*w2UoIT1-Z~`(H=iLZ?`;5*q_ukAY2l_jVjP2T>gz7Y>*-DHFbK zoFc+y)qUid@yx=1BD3n^?&E;<#~2b^h1A^nH^iIbdVI@44@ob@?r&46Tc6`@1hGmU zBeM7pR^%I(N$+<{?{_;FJ^SA^ju6I#PgQu!&u^aVGM(7%vFUP(cVqDjc)MSjGA?!pnZZj zA#yW?L7VlA0o7&&T18!P1>WGU=)9a(os3?}rkfMZLg)CbL(3*)3k|y0+C)dUT9w^c zc1K~X+PUEce$8`y9P&q$2M*HnW0-hh2c0UAG+bsejE^CqqiPK4Ai%H+qi3_hOa)pB9=V80x<`3G;2__fRa9hHMgk z5WJW6`x=88`45<@Wihf_Y3pI1DOo`={6rqu*GqQy6u}+;qxrrIkrX1#py>0IYh|>k zk>F9_AB|a&oW$)HOSRp^MBAS7OE1-Z-NK#VSIBrkYpbN?QeBGv><`qH-?|#l00bxs zT-wn5d0YLI>S*%0ss-Z8tM&N0EkgJA*BH5W_*_n+BBP+04r>F`-Ct*j@ z{klH`9`~$jgB2Q9UUrS~`Zh3@Tn`$pvhZt#!=cPlTb;dxb5KyCX6flIsmH7I3^%Fu zYOU!HTMM}u$u~{-O`=#43kP}Pzy+N|8jZt+6-=m|cqTs`HBxx59$#>l6yet`E+;banWMZWC9Z*`M? zE0I*WA@}TMNWKx8a!x zYpb-P2x%8z%o7=WjSN*&7M(*DPHZ5d=LYC)sWevh|6v% zEMkV&7M?v9>i<~9S>(flB2OG+RyfMY_0;|j;&8Fkuyau!k$op&_s<}k7w+T|f8~j% zl|=opc66r~C59(eS=*};f4@?MH`Wp(-^dd6J^zwz^wpYQ=Dfo3U`~haD$l8{tYBxk zd&c;s0oT^cvxS%!kn7`6+;QYyQKSg=xHqP#8YnX0kW7E8dBJDz#@HBw)HfT7Ru--+ zgF~_NiqO?lZABI4a0}8g7(;f-n+6)MU@i!9>DQ<7DH0#v)C_ybj)fZsTF6auh7d;@ zXLzS?4rx0RgjwrF0{Q9E_>|XSM>r8OfP^yCzpQ6Hu%$nm)j zi7dzMm@QgwVo8>Mtkv{#N-){gD>9fr#|!HY7c(_FIe)L?Qv+Hv!(gr`c=p>Vs*MPp zt4gc=TF-#XBbA^{YrwD;*f(ro-}rOvmxfmGIAQks-?dRhaUF^M@x4GW*x*(WALEt> z2aNS1L@1&AuxoF+qwl;SLjQbaIJOf*xr*|lTt}S$OjIF;MS&!V7S%%P%%X)6`!Ml9 zFrUG~`VG5Y$?ddW#L1gIq$9IBTh2xX|MZhC=7ETW&M5MZfiSq=KHngIf}D@%Y!&z$ zN0(yW316&RjPD=HjbQ_KvcL@O9ijoLN@!wGM! zdLyID$uDh;mW)t4U5CcKtr91Zd*%`TOu6>r&D8il(&(misgeY{Ty@&NiRT?*@)A~c z?vQ0*R-vs{D7?iU_VN03+jAgip&Dg%-S}KSWw-84E5+ephHMHO4qC4w%KQwegBNvN zBk4R9_Ezpd-W~eFA#p5TNI~DAMrJ%+;Gi~d4LM`vE%J=O_t`}dQpjpp!-(XnRx`(r zc?SDyvah7~4{#l|m|)3+*~5Je!D>NQd>IGpa3?MI6DFLqG!z`JL&qUDS`8#grk(cb z?d46W8D{?L1Vg$a9dA9h_%UzU(H1?v-^Vga5(ynCDM^zp?6Afb33iDDpPqyiNKuYG z2|1!wdUMYl9>fh5UB(5%>uNLWeetY8fWhqVV>*J(uC{NEh3e)tk6w&KZ{SEV1ACN9dYW7gg35A2 zFZnFO_cC>h;2Yi8?M?XW!MVKg#@~ zLSNeYN?}^EeBa$#EOBE@ay$@LA={)zakcA-e29p5ZP$ynM$QD4*I+sA=kzWxie+ey z;ZAY8CUmb|E~o1A=zI}gU}yZL6LB0Ox&0#5D5!fmTvAA8HY-ao$d;9 zsQN{?BUowgu(=eD@jB%LWkylKxZoY#jkGc3fo5C~Yhz}rMNHYJ-W_781X+*xOC3gJ zr8QVsJ1)rB%nGe%oRsqbRM39_y|eA4-QG;#eQ=f5OTV42$)LkATq6Egg;(lLG)m+1 zzPO*!bn|wf5>GQ5AeEZw}Za! z_9}{RNkQAJmuJ^~1IQh88X0%stI&84R_MuJ= z-1(0(sC0pLS+B*Y%6r(Y$X+f7kx#gJ<(+BIINBog#(uuQB$IAPclOw~J{KXrrtWH( zZC9h6aL1l6@F4C7%wo`sR_wteUVfDc7pWQGzWo)GlqBP=IhEkxp^7qZj^BQX=&B#` zSMNqd92edFdkpWZP+9tAAsu7q`Eh-oVYioxE)>N3jsix#!-0um+nN!!feM@}FRpTa z5-)g(u7yZ~nGnF8iMLK}uoDluX|TlkP#)V}gqr3aKgV6hLieJc@a%>nUipG!)xH z2Z<1RCL3?h%8H8WqvtbyXd17YAmMf?BLf}wXJ+F49r8h*=}6AUo1Q*d>#;p%Ad$z` z+>O^nYU>Gzzg^+@b@YJ!I>j?qM=;lppslrgDs)tAX4y=lYO+N`;@1kdutY$vDYT(u zr3=E2|CSE@RC#^(r#ElD=#(E!)MXmv|8RGCboI?y)))V;2$91u_D@$TJn^XWzg85i z*!XC@$|&=o@WIBB5CfR^|DOeLWSNQZ2gRs)9e6-fwx6{Eqc<4BU8ydk4T@KGv8x!K{9u$QIQ|BZe z8PfLcg@-=}&e|53W*kYw=^*s?@Tt$^kcB=a33df`iG$QciCnwvM}>ro;8WeBU=gcd zfQ2@N!tn8oBa}K$c}a$ClRHm4Bo(DmMOV-&epL-Lz*wFE9|qBSbnRAEb}aeei$0%E zU_93jwqYFXiwhqIMvGJn{`~oK-exg_%ChSVuU;@_1&TRFv3L!c^XJK7a+>FYCH+A* z62ep;)kGDNS_nCT#?MayqzKHdC7X;f)IK6Bm^u8+DBJswQi)4c-Ftc~RX^XtZb@Ap zM5xEOyxRaz=gXKghk|ozdQH5Zkj!V=7nsn+CSHep9J#G5qHC2-vw6v)1~ER}o?1<& z8DW5-f2KyN$rnhBHexBWYH&!8>?vmBqRD^;+@AC|sJG*7T*+jvbKi;r%F(&j+Fd2S z#355%{nBJU`o|GKz%&BfQG+rr8z^f~g`n7JIZ9{856}AyF6eTKbC*O`{=&4=V(-_S z2l`ICsMdZztU2$=?lSNT0%xv_R6bFIDsjlxDD0`^uL#@9tIv!BH^Z4M4N_9Mc27-N zSy&5*Ex*i|W9A~D#*mJItp%;kg?H#DHjKgK?vQI-Fb`}js-s$3J*Ad>h_Gnz?0ZoL zke`c0m*5{+dl$057=;J~unw-u@6~*qwSN_bQz1(aVSS!T{htXNNvT%7Vf3tkt2Zt@ z+!QLyM{6DS1IWHue+oX`YW3D*lz$k^w_q9x_j)l%_oLu2CgA~CiJ^_~Zle-^WUnpz_>3gPX&LfQ>w!TZ{0H%Qa12X^FcHB@F)iizT-~H~h6&n4h zT+iY(we!Je=Y0n1-s5ZeX>XEH8t1}CJ?IY$16!8TsJ_wXTSr_!`6iy+2wJcmDjjCF z%%T8d^+BPRhbBNFJ|nC-qZ}52J1%7Osf>DeO))RE(^&kdvfwy;^`?!5+sQ7CY8YP< zBGdpvFFg1jmspR7=7ce=VaUY}bfWaz^4w72(x6wHFB9lpm$Sku7c7GKe@W%qFRdNF zAZ3E-LzLdqF>bT_Xm1ug-!PES%{9dw6V(vx)f-r*&_G@QKEQu+`c|mwg4i69ajSTl z>1VWAmvFO8C0_vaH5JR-J|9IUf>)CwUQ6|SZhWhs0B!sqrZUm z5Qe23%zqg9)_C|^oexnPx#&W2s-*Y4i%Ie>M#Mv|0<}UDcb6l^hhb6An~7yOnFcJ zyou5W@KIGA^Gl2?30X?wQ$p5}w{#qJ@Gx~BfPqC0M5F%U%fjJzsOVxj+GL%c3Z1*hNo@f{_A6@SD9hlh6CWV$% zh%*K`cpX8WI+=o~*f>7B<&hOZW4B z0rZNruT@{v)2<-q@c2U;RKD#r^xIP_goAZ)p`8=KbpQWAgb>Bng;2V8ek+O1x64U zx+El}C8ea1knR?Q0qG8rZs|sep+lsbfuTF3eJ{`Z`+xA8;^E$N&slrzwbr)PRAw{$ z&T74xS{MpI>M3;?oFjz;K4avuM6S%RWGQLPsLL9Oa=>++A^6ewVt=kyEriAht~TQN z-alGYkSS)CoE5DS1hk}9y_X}RCd$nyD$o38jYEEV;*CMKK$v{N(P<7ZjaE_7yQe*Y z=JxId(87PZ*&zMuFv2q6GxPOMX+)wqTf1uJAH~J`unKvzyMuymUTjb6^reZoqAU*6 z6vK6lpiIeJ#a9G@)OI*M#ird4x<>)tn=hC|64#Ya#J_f>jiaAhyWaiA;(&wUMoH$4 zY1$ynO;4r~4h8#?yHi)Ew!D%O#h)?Sc>bDB_Tu!35xMkLy~0*?u46?(p0wOMu1`@7 zXIjUvUNu=IBOYu5-?TgcdCU*DbvBduv0PejUD>OT-Upf>-lm=-%$i)zq+ioVv0YW9 z#BpO}LT#+|Q-}I=66hnH^K7$WMWIA+HUEVHf)zf=D*GI$bC@Yj2J@pSK)8N5uW>j)1tZ0SQ%4W3xA!RaRD&w}VV zp~&ty+&REKMt*vQTMhaWq|ochM&ISl)Gu?#B~nSo7>OCwAp@3%&3vmez;sy@@H}2k z0r0g51CU`oO%+-?V74;F>&K$`s<7f@zTR%i4$$ixBxc55>b{4P-YeSN*v2oggh5NJ z8VPV8a8l~$ro>9+&G7BKS)b_tSwHlAqYPuJR<9sDKRYio?);Zj^t7_&;{2vzDQ&Yz z=*gCRf;k6VN#LPYupH!}%;5Jt$|=Lemw%s`U^8{snah@v;pSZ17f=Be103ePgcH8( zR+m%U`iHtsy6<@}o~SE54VElqs-~l02CS}O{Oa$Z4--eeTOxM17kfqe`uZ+|(FFCm~6P=D1y<}fP1U}G+d zqPMY$hLtuh$LhG-V|lO(N}?LeQLCcJqjtw$kEb-V2N{djg7?Wd3kcD}XCVhQM< z$UUgrAdC^}p5@|ue=(yL^sd+2A8UHBrsLc7xJ(AQ_9iVLob3X%d{vkv){>%%sP|Tm zy)CKg)?F59d^oa|w{*NL-{80$sEChpLWpQSa>LZQ4vP;=Li-HGG*u>@*`+69)=X9e z8(%_*5&T)lyn+|}^W?hpYjpyxEpo3}`oycgwZH!hxDX`%86qq^R#+jyl7vjWfVxig zoR|1}saanWEkm32;MiIkn1{8$W=tJ+R4{U4SoP8#249m}6W*lyKp`Il`uH4D=u!@x zJJHY~3ao{&)y4~QLnLb5N6LN98PhDsxlr=0#S+9wh%N(N*%Fnk4Q71PCMnndaax8rc!x8B! z$DgUg2si+Yy}C9YjE!a`G{jVr{?2)Fj3@h)hoD+{#L*@CiIR|Coepee{^sah2ZALO z2WnM?KKxGU;3x5q6tvq+P4IB4rYrQ);caZf-fE&6tRXs?vPPI@dd%ws%(@E=>LT_9 zN-oWF3Lfca3sJKy?spK5cbhDl{>!_?Ao*H_n1(T} z_bF4du$_1*G+NK)un`Vi@+1VMplqUfRf+&yAL`x6Vnxk!*~fhA zZ__Q|!#D`(uK`f9gBSvP!}J@h3kYwtC>8<3%^!;`eZr=0t=|!lpr4dA*%1-`1oIjZ zCND-Ci+fR|gTay*P7+QUw9!@S4DZ{4*sd8v*OAT~w`nd!8J-Hk53ys*bPlLhclXC< z^9!HdDc`3>SV>u)CTY_B8%fP{la zh_f2xgwt9o3DeDE5(VKm%w!v%B$*hCZc<)lTh~Bg(<&||b@%R@@QiFPW|wH0IeqA5 zFkM6v!o*xMI}a4wiypFZCG;#mATe*iS zykGPFK6xNq7u|y4hhqVCy_posU(BVaKmIL*QVxV*Vq)g3?+uxy>Nw$tn>I2ii94!< zi1$!ADO7>fV`57yDlVj@97Da8T?{v--J1)RvE(sz>&vQ`8eg%vGxw>7J48Z8{~8n` zEVY3b4kGC*Ef6Bp= zfKaP`O**~}%W{|KGr59i{AN9$pL4)BH#Z|jft0BHYG13dY_WG`qgh@VXu5FK&+dJ7_yr}4{ z4eZeq)%w20zmRb!>}24Ylo?s0(eUuaInd7(X=rH#tQ{X?{CaZPkx976a+{4P7&H?I z`X^`)TM(rWxcfjc(DN#3jAerk*Yqh=4Rx3!7gW`Wz04t;h=7o63MC9U<7{z$9x!Qcc^GJ*B)hjzC~YQhBWVO@UmDSXXvEr>HPO&`Nn>n;c77O-*OMCJ4#CUz z-b~W%2)%4rrzY;=KQ-nG0~I}#daRc=EKVEJ8JPJ_p>2kw$};pVNvmV7>^x;B0V?6u z&nUvO^ZAab08KZvd->kyVmK<}pODp{+5(?6w4uyP&DKTjHjNbrGQxqMVSdif;}AOH ztR3MZDlj=v6TS%yyKtx~FSkc(zy~ydU*B5}#gC!0)V^18$_O0t0>~p#eWHa}=!>DX z8m3RfyX>=W_1>#!Y~vkip>T&2fQR@eNL{0{d2);kaKeHcdQumYL4x|oh91v1bE}b_ z-{-QLS^%|^LriksJ{dqJ9pbE_N;MiUoPa$U#QlkaKn%j>iz8}40nNi}Wy`<`Un*Ou z9dQQP@{Mr(8mxJlh!AXaAtgSdvL4ijj#_7!i+D1t0_!xB34Ft@A@Hj_H%NTh^xOFc zf#6~_p>&X`28jXpp>kMCz-zNM*Dcixzy>QGt}~v*s+dN^gZp+MO~fD*S`TxYazw24 zI}AAIPyI=z^6~pw(tsWI2Y10UlKheKBI!c^_B7HxkQ^A0-NvxtxkZg|yt!XgjJ2$E zSu`&*Q;@91tpJ((4TN(Gz_nn|>}Td99A?&e%#YGXHk{le#QV0V_=cM<+W>Aa$^k@) z(6s`z`}PyGQL+#CE9DJ3KXhfJf^%Wt93!o^c=Vofr?Y5R0 z9Gq!@)GbO<7HS+hda4qK;P5P)^^)6cTQPo_w#@NT9jdzSg{uJ#hcm*O^G0ge;EEW! zKS(iNG6XSILmWdYAY#9jyYLe_gGM^mnT45S=V)^A%f5YM1P14DgegiqU>|(n6Cg$0 z**m!Q)eQyJ3cD;S^d408z2Ek14PU#9`VR>G*NSsd&Oi`sgVxr1TPdS<4%3nZ@7)hK z;OV5Nt|0k2zqG_qA$uK?OV>6Gf7r@lHIy*YWif?Se())VMl{Ne*gS3H&pI%};EDC@ zL8sr8w(gWOBYr&Gp1_`VL~gs~AVzRmh)ifaGuXD7;a2vQ>RP17MmZr!SGPBtN`$SC zzDoXcjht$7up|%hXEwNj6wE|^s|jdAZc$U5`MXVQn1J4Lv?iH{fV%BSGR2Eu?{y%0 znZeaitKVINn8@i9C?PdSD8@KQfEHjTmQg!Ck^9&0tX?NMb+mW zLo|NrYm=Tcl}60)6={`VYdiYn%qLZ}M0>62A9M#ES*B^7uZZ|PeZ+-(BbkOesVHH} zhsH88675(ccVgRQ44|Q`@Dz>YvEj|oU2euo|1_JA<74yX`oKSz)O*WCU_@h?nN7dE zPuVumQ_%D&BX-Hex+}nu%H<&jmZpY!EA`YP7@sm52sN>C1=9m}U=kIXRYMHMHIRwP z9xg48Yb*>{TSE%T%~YiVHHiD)4H#9tOZeZK1kFIW;rrOjk$yFO%E0T?00bI{12CNS zI+A3=vwwKdt23Q%vj$nN0}J{l1uFsN7wP|4GrocfBqAneWpo|aD;P_3(=DFkSm2}G zzSozeOl~@x3grZ*WHsl>L~!5_59zx^cPy#LJdXBia9Fq`o?nXY$oZ}}%#I>G7>w@W zw$W0oAZQkyLgQn$e^IXdu9wAJ3JfQa#2%=E_#8(%(Q(S7b5aQb=+u{)6NeM+49Dx(x}zZr&5 zpBaO{C#TMJ3ed7@%_Jd0V!jh2Cc1Bh;8X$AugHbAtz~_78e*sv6j0G%i&H^+KX8{K zYv9>b*yzU3`q+hR;tRm<r&9~w4ce}baqFy)tWJTz|L+Mru%oC;*adT(baY(HBKyi{CVm+6*| zlSpz}K}W5td#Zzv7=-JSLSZvCOdf8lE#+@0%|!owMDJs)?5u}XyV~mqY^*-FYqQ4# zzNb?XN%;Zl)9D~kXUL{18@%;{&alDnJP>y69|KI9GVF~BgWmXyzj2_kuy?&|`x63i z`0CnAw(DXg7;r0&W8;9N(`YAV=z4x^2IFyGeFFHv3ksl32n31}br^xJBPfn_g9@xb zjIjtqxq-HPTu?Q`FAx&4=#S%=rJ;?2C|U?$)y-As8+Z6N1M|$GoG)MVJ46GQ0w!zb zBE?6L-Tn{v^){nO=W+j4I&L*cezlA?z{-#_1`;OP4$_G7FNgd6BxHkYTj`7%IOVNX zpb5a|N`dwg+R<93`24YJ+k-KFf4<9pSPnw{bx*l;zcX%?*nmE<0ux#t-*V`KBVKY0 zkWSQ)M`Kl10eK?PU;UUtI1lhI=+>@u^Qzo(oj?&37)TZo_E*%vip=P9%_N&x4<1DI z=RiCcW>7Yl!$Pa zhM|7JkOJcoOES!af3TNk@IWbbUReDZQubjiJFC7F5?Lcqq}O0S$8H@^T~SeC0|38M zZSmJLwPX=)z;pNv*|iwT7JKGAmHh3DWeyij4DgS*5|LIgLgk8Do)E>xSm`XgjD#gH zbmM!8Q}q9#KJ~t3(c*-siqC(^wskIY8AkPDhMVb3Af}j}V4&)Pt7+>EgE}11ljVW2 zYHeyjPF3x^4CamN#R;6D3>Xqv0=TT~cmUg*)IZ~msM^gRZ{*gh8<5>}J)RG~=HO-7 z*2{+a#?Sw|Ds7}ui0}#f7f=e;m_B9u@P1klDLZZ88A-dd_9MoXz2t=Rcj6WIhP&LY zeA<-DFi+REb`cd)DUyzQL=MbMfe$gI&J_LY;%!%gWK(fDh&}G=_WHW|R+CCE|s|apH7WrC-(VGKs-)>zHQ%%Imi;BF%wdM=#@~=N$ z0ENcRYZM})7MFz>!aCeq|3EUVh#YONy^ly`@A5Fh4=+c@oJrw&w2wTt*s)HH=e^iG z(o;(oDvsGSj@0XS_U2J!S0Pq-jw>L?NRf}BNlL0+`%2~dBNniJ)_Y&r zUYZ!!WjHSQ&dv-B5kJfOiz6$~;JC73iraYoQ6_$m zlY6$o44wkR7bb!lftFufa7)soq56-YDZ4iFIG<4G%qAfbsddgJrTlz zs{Y$#W5GpJltJ+`5P7+PKjlezxFwS>AM1Y(G zRfH6cqp&#d0EAD|M{G!+-J1dMZKR2i%2 z{=N!2s;C>HrYvUbbMI2f1dqdUWD^nfVE9okhX`XrAXl2}AM@M?qTk2J+n|iMk5y$vmdBC5{x3;Bm2tAZ!{$RzViuN(RqJT_s{QMf84CFme8 z%Nd9cod;4uvaAho?u8T~el)C_Z~C9llG@@$YnIv*)va}4K{{-MPTG_FrCK9=1GN7t z{AbuwmIi+VC4e-6jt`+@hW7OoB~Rv^0vj>DsU-b?^Old3{;s26_`Al0w@)fS-AJk` zlyJ^Qk1~HgfOw9OL&v^(?08YvKvl7p<0HUvSQz6u6Vpuue_x1cUjhE8x=unGezrX< z;Jld3L_?B+Lamw{WPIk{CXi41WrQ+6TRppN6t;LJ{3}SromOqoeCWn}HR?c7Na)`p z`UPDH(4>F#V*#xhWS}S;Jn93*!WbH~urUgJ4=P$H|5fUt*O(B-JOFwA@7P&Vu;W)p z^@F2l+0{`sXJhpUgh7ZI9+IH_edAHf#V(&qo#4lxoA?a@6FGcyhwqAe?#j+1l!aL> zrq!)mq2FtE&A4jg=vWAiBsOP^|4zL2gI}x;Dzf8FJ01(#ziPf zk^@X7tQRR1>8Yd>EEz>&&Be`47|e`ZE0_)dpf%`Q%bTQE3urfX=1V1yVFmS*M4ID{#zHa#|7Z# zOakziexKs2lkEXu!zAe%^>lVM+1z^@t|qh=TF%^eZsmJ*Y)be`1j{qHlUgEOb3Ep8 z$DpX%abD^}ol>bdZ{Hm`>DcorYS6LJK9)Go9!s3fEA#VOr*wu{NZF~KX$As(2kaE2 zP?$sghl+a;=Vw)(^42d2frc2hQD2F3fVWZp3pW)&pJ|bjN{Wh+4A045tE-)<0#hr{ zk9G1Sllf{Jo=oHV)kA6-uvV`jLnhCL*=denN_O@^gukg(uYyFl$m?{Y*LsV8-9O&rqkd7@2jx~nRoV=Z;rY7|W;2XF%|*>0!#HuB|BSN+_ywB^)sx!F+;gC2Ak?f+@M*BfS^Hm8I?H1oyg7d z^+F5$O*C+6H4&i3A?~+l=OuCdJo`OEeqMs@VL9FS)G0i3PE~M(gSN36eVulCJqvqi zTx_#aqa$lcs`Z?@*6+&2_odcXP&sp+bjV?0ucy>83LX*f2};IP*XP}q$6~mj3nP6k z>zhQP7HKWuQGHs^ZQLfru$QJiWc&x~KJHr<(xCExHon4xU z_~}n7FsO;&TZ=BDZXZrHW}I?+u7;QEZE&=uQRT`Y=8qAlBBpR7-u8Jvh+65@Buaop zzq7xQ5nVjhuHWb0*RT^aqnR&nscC86M3uhT+d-{1pi<-({bxHhDCVziEdF%ddR166 zLewV!;%cR76=)ffRdfb_IJ^#OYRF~)~emc7nB0x$i%N@TvEiUtFm zdj?d#fb?TGo&jVPbVVbOY~W`^XEX{0rkw@KaDr{H1nSPnTr$^PpUvT+oqz*CQWZTx7C(Qdzn{ zAiHDm^&lVM`yak2hwo06oyCHnFRRPThhtrTvA5EmAe7#4!uS+eCM|8jMfX34&>_5H z{N+k?KZ6#OZe!k-I`SdzQLC!#(6Z+9-M6*>V^iG0funXj!VurLr%&UkWS~0LQ2*yt zxW?DN!vn&x#<4b2H>g&={$O9dHpA}^p=6FX_b%(g&;-N7T|n#I6shFs8A)9V{muee zJ~m1MU?+^fY-0T6-EFIpQBhGMJIh?a!3xH-oaQY7OC5X(6dwm~1Jjgr)sbI#5v!Ds z;o&^Sfc`T8Rj~tf+fD@*B1Fe9Q19%%njpNx@uyacek&h-aOH+*r|GDJQLvLb{D+sb@}wGVl{-H6(>{dfflcgeq~B&dS!1%3uYt>yM8-X zNz0eRs{-=_!+FbaIN{IY5E&&PvIyz-Xx7qYp`eVQ1UISNF%s+W5eLp;yg^`~S!0&( z>A>!i$I2@l!<-qe0Sc5g{yQ30=tAfZ858TKujo=bK%4h+>tV0+p^PNz#7#ZgF3&nP zcmQ)nwV0;%Z-ruUTpa_su(d98b2Ti&EvwO7M&2cID|+dV7n1qMqqBnhHA+t`-MldI zPqper^i5Cf!q|ti{Q-%*XSDw3IulF3I4Y4ePpu1?$`=Za+Kmq)$@kEdH-z>p)^GiYW`QGFTZ z&&K3mju+|O>3@&^Z{jhAU;>cs+;O#8W}<5tt}HJbMjl6j4|stoODA24snsXs=z+`*xm^kFur(d_sVaIrXqY=7kv z1SxbToKe_Vq2yYdf52DtrJVKy#WEV3Dxi5ea_}*OjqT9}{gOkENcd_W9A{3u%KV0S zTLdFLSL1i(lmTGu>IS6S9|}C8IznbTEaJ_H--m!@Ds&%*v_B5RK7@xdlR@4)5e;+9 z-X_m6WO>!7E=~`JGx2wEBQ705lFx@>4*I7mry(^D)NI~R2Hiy0hnSDpuQ zrTS#>X;Ua&&wQz<@#43kjccY?mQm$$1NQ-J#%B0-cyE|T`kRgXLBu;i+V*;nVAGS3 z6*b636m{c@>w6~0#w>nzIUbr{6bkFDb=X!osi%Xd?P8PjyPdHnH4=BW%u-Qp`}mEA z``a?X6<;N>lOD28xl=LqL>W)hn$53-a~C$B`hynIn|YDP$3kpP`jU|EYmW7 z$3`VJe*kje9yN7UfTA;hu?9Is5QK!&!RkNZ{&)$p@l@lyTUiNrX13_!4ZNcEh27m)tY zEzRl$h>t$8B6WN1wBGQ68-MI_PzQ1u9^6nqPw^t<%_ARd2-uUPbw#vb9(f51IGa@T zn<2LV;cKD?MZS+?#_#`9repIa+Nv}UFwCMI3gv-TR5nuu;7%)KG-24N4lq_}_OGa3 zHo`0#(cslU6szETSwnH4Ew)0$Ail_3Fhp7=hDuE(m4ma2)9=*njf$lYeT|D|MpH ztAe=%1}rbxh}ndZ8YSo4RdO8|o~aE<=m@}4Pa-{KdyuIgDByVkH58P*edq+F^Htn<9WZ9E{3%%ds`a(~BQE3qwSHr$x(-1Js%RrmWsm|HiBA`kUPW5J@@ zFEh86>^K%=Xid}549ud-;qEFb2ZWL()c;&bu`{n%UAYFIQKkBL|HPz<@D^I=lG*-& z;=?BJXpS6x%(%%valn}*VQQG%0KQXd`j2a8LIa!mI&Z}y$qy(fC@Rjals(M8(G;D? z=CYj!;U3OETGNF+$HioRXEwdK`71@W?cnrU;^81XV@5(;*SSTaF^uf3Dd{@X-CEDY z)-w~<>+7(Cjm*7Fop>TYVZ1STe2~qRd0Sf$DaMnuz`zdfca$nR)bRJlyLKr=c*ZMr zl*n$G42oB>4PNbgp36jS%lB6`eD5C?#~21`9!`I+AWjHSAl$t%Up+>zK4L*mB0L7d zeUgae-ih-R#R@pmg3O+jCrymVu3t15rzd0}hWC{7f#b6k;YQPn_~e}>$K%#!{B0Bq zD)6D-PUnjxOpN~D;w0SP3-z-zwcJnLRlaqpbB1T4(R&xB|DoVI@OYp>Ec!8;d0({G zlMXi?faQB7eo$2KiTyh|Dh^g@b2GrR(0!%ukaoNcNujts;N}o)KE4Y$KN2*Vnm*05 zrd%w3T1!{-^LczvAcL*~>6IQ5p(cT*NPga6ET2fiA344N{7*NysK)uo_sarRd;EQ` zVg!wSY4W~2K`eaxP}WA&l(XP%V=Q>u-OC`&d_=j7-4+q`bw+aY34sjrpSK@P)iH6u zHd7`9Nq(7dZbfyw znC#Z2xM=R*Akf)s!(afre#Z{NYRf&v(_M~xCiB4izAI3Ph)&pot*?n;QWKrMS2B=8 zZ0crO#}-nFyU_%2t8Z_U!SFaC67#yXJ=;5@^euDzHBdS=e7ydj;^TLqB;IFd+J3zN zPdU9Y>SF=>*|>1Cg3tR>fgnNt(fRB^)>!wx3}O0D$rzPd(I-yai`tDJ?oP$jVoL2w ze();-8WkxpHI5`y+c*TrN1NU-ssVfNC%a;_5y)NvVvygieKj$P^5>+pva?LWzWV8K zdDDrvXr_D!hzJ|X9imZ>x-pw{U-)G#T}w=m1m~%Fcw|3H8xwC4?_$K}bMoK27$nsP zWn#uM5P^NX)-76>txw|yMC}p>W1Nm`RgOLT4R^W2$!SU8cE|lLadG?$;mO@x#KmvW zn47COlHafqtEf&m>)MkJUI1(49O;cfo}X1?#Z4z(sRoH5psf&4%IaDhDUp3kbHbxC z25|hN@628E10Tydu#6LW{{3i8-R~QjFL2@z^X?bVs@95@lc`#@FJ|2~WVAPqtnl%7 z)t85@G|HrIdgJ@`W?tzdfmgL$boS|uc}fwqLOfe3t5GwvS-Dko7@je?dp8WIPH1XP(RuN$|_o(R-&s)5zbb^ zYJKTOz4$R@)HhSynCIXU+M9+L*+ZUEA8A=-jq1xAQJ(p`WsM! z^yGqfCsg%ubL>smEfxiM=PU~_6wRcCOMQgL3;4>;n3sE{HXi)mJ2g|Z(q;9_gfh@s z8SU~Tea~A|?NrQRf=A|}Ax4IEo(S-eGg5STPq<=qtSxpA=>VMe|M%Mt4}l)xs4xMd z{KPxM=DcO{H#Y&oAskuBC9Zul2eb5s`HR;ny8{%KwT65T}XH{W_ zG+UTM8h-5jGS-?0qIKMzhxna0wjF&8a3TEAPaBjVj77AcFu|{i#<(#HOx!BXp1|Sn zQlavjxPg!@7g<_bN=NhEe6rNp6y>mxP>T>0Z49em$24wTRE*}s^C?kL!`Zm`qFmY7 zS+vKL@YimW${&V)?0Pp-@6i>`D71C%EXW-`;tUArWw-tU;Qi}XwoTqkYS)qsFfpK| z(ozC4K<$7Ejry>*bw_`%*rP7l)12qE-(4Qc2~a?Y4@`e!9W>+Nc4;d5JY zQ0w>?s})~3IUBmYB}8r;;^NzS)b7yPiX#A*fyCmDa1`d3B96O|7VQ58lzHKhuZJZh zc-0fp@b_2qJT=P{b*dF+#fE($3JN!~SBeh#MrHNz>MKFcT5TRdd$Y!m_WS9}ca1(F zM&WdJvmoG-VV?)B9QXv>GH{=H-=j;yPK>@n6Xgl zCCb={6~Bs`$3_#qd7o~?PY*Nm%)-e8M6X3jFxA<+k;_Yi1*;_(H5(-}#txC`jdg;{ zen7y&Q6)N*K3dh2S#CHdNGBQ_`jmB#TPRE8wJx}eK5X4M_xtxyWr2z}IOTKCz^ZH@iM&U(M;1N+vl32_HkfHVxdSQ)&1zrf&l?sMPG`))=E z&?b*>D1zZ@U!~`!{cU2wamE<)K!EYJb(mGu(;MDCvE*eM1)ECQt~3NqE&DuYAC<-n zfj&RymfCl3_wYnr>y?{fq!MTCy@VT0lgOsmVx5PVzHD$oX(&C~rl%17B5R&Y5*Ut7 zR`neSt-wK(%%%SOmMNvTw^xMsHjCK2nafPND-!W5#(v7uX$2*GJrVs|<$r91JS1Eq}S!T?$X}&=8&=DPQmuol@s)JD+S& zf22NX5Ysn0@38i|)an$vi~w6uf1n5-0Z8f93ASqJmT$1N+ zB21+xaL33qa%pG}=X<^bPC?h*+}sdtjOP?%ti%l=z_gG+5ekH!#MDcpavSqz!}9BQ zjcM{+n{#xOO4msKf&T7H6UIHHaOzBjm$OMjnDv31tsOsTeS~KjICv&@b0>RR>yS1} zhwPoQ3#2brti9wG;G#aIot6P{8^FgHa?JAYuH|DeNCHZP9vTijJS-Pq4i^yKqaI^o zGzHL?Ws+FiTU+yf*ujJtOPO&Ml8?S0$W{f60*>S!4%f!cGt*D<$_wy=vdNngDKei? zTmAnp!>Zp#!SYo!suvgz(8R4JpcVX0s%7@z`&{>|&p&{g1H-?8@ioWZbAW{fOr?WP z|4q!rYCyg0<@^%$E2<`Af!R%s2u&3zgy!0FMH|btfEQyOOmhY4=Obj+I%q$=dPL*{s zn;nrCdS_}Isd{GuMSCTn*0awDfGTDjJZWjgeXnyum#Pii?I|H3h7e{ZfL+J01GtGA zCX`-J6Jz7-St>Z;JJq7-U(Q+BE1M-c9vppyL@W!O;Ox2vYCXo0>$DlSm#sqWJZVbF z36($gcU9KkSIX42oXfu4KOH-7vOcJvHfqd)Z@byi@Fh?CsVZMA>p-$9kF#WgIBPj^i#0tLn2n83Y>oYsFotUy_JiyI!!oT572#=B&Eu zoWPSflrIy{^t&%w~Z-yXM=kS z+P9zmOHFJTbXB4#F$^P6+FCs!aZ_#=$+&+}(Fj45{?P<{W0`aq^V+kT=$upH{)Ye(3-|XMDXZZ)oC>#b$UOq!H<8pDn%$iQD>o?F_1wVKLN@dpZLnrk&yKHFy!30oV zj_Y?rv=#ecgZcDmuyer>~8OBPJn zLV?<}ph+hNm$NrVn%%!eY&M%nqm4*=@$-s)-1&#XhpfC5o$aj@QrVJ;8lcB`%B8nzZU?xZV86!Hqb-SzFEdxP>U$y)>m-(J zef|rwbN9fAgO`|Z_NUjA=uAMFiI&5Q53o6PVKY(|W*+A6p%Ds>)5Zxz*_!(EXMD?p?$5erMn(nzNy=dwJ-&mCs6B8**_! zn=MlC%#i|WcPTJ8MlreI@4_aA#C5V_a$Q@EHahBxz9X`?QHWqIQar$S8-<65H$;u9 z;ag5@&d2|Qo&Z%IMjN$WIf5YT6{0Erq4poBM1R3V3hc_7NJ=x^Tuv79nHX*XmG=7MaRcz%V0wPf>#q>O-RvNv5| zTB$EH$LIlhuj(~Vx0N`}E>7(ww7I0&=9u41C_H*NYkj-_yGVrl>rL7My`A5|;Dj^s z?o*~whj$LN-ALFqQ#{-m>E@cgTyAhy=P0sSG3i_~ll2>z3pzTP?osE*1y5hrceju| zPbK}oEP#L_4PUIpbqU{>b)S5D3WWk#39rdJW2K@jNr^c90l8C8gygfsd`6JQpFe;2 zoAUCc+BAA1Ob!d(*PF7d&Ft;W_qCsSq-*rpc2n84cyiOaW+UCQB8)PZi&AR7PuOH7 z15kziD0|&)&U@e->9%(=artiYve%vQLTMTz%+PLeJF=%EdM@5xci6ZfQ1q&5p+JRs zG`+}efn6<2l_%>r3n{Srw$ZlX1>k-h`Ux>gT3<;|T7S|1fzBUfJp-~!j(?PbfJndr zYpbok#E9Vcyv4a-k>|;D!GUrBRr05xJ9hs>)^+CTDH3L8uM^2{YiR4H0F?XXd()!B z3pbwaf4mGDeFildic+Rue)30pW=*$k2#2ULEbl%XCC0ugfM$(Oa*uk0fI`mzap631 z%6~MuGjg4Z_|_}0WW_BrV^DNtlT)wk*XCAo##huhWWX?^>AmN_=ZthXYiag%rT1RS zoNlZ9yI0_r1601Q!T{&7Bs`gpxTq0ccFfVtH1e(UEw@Z-RM`vHuvb2_XX_neeB+FY zHUE~~I5H>BjLi542M&<>wmg7(=Nm!9pu^nfxudn7^ev?r`D=|3BuI|mYFDcrEwc=A znL&QX-|=jZcCX=$@t@V7oTK{~Q|rhVYwbMkH|^j`2R!UQz~L@z%u{h&vEaUw`Fp!= z8ephWygPxl^@&s%{M`{HjTB_C6BExq^?I>=K3xDcX&=+CgvNN~0DsT@cGwwt;Tc<_ zRCUI8#8J0(!9-W8eZT{O^_iw)L>V3(NaIOa47VB@(rZOtK%?K!dJBX)30D42F5pCt^C)= zf;ck~NkCqt075P9xd8XoP5lK4IZ0xoq1HfDL8FVpzoj6s-O3jmI;z*@xL@P22^4*BGI>XPsz#oP2c$*uS&g*MnpiYVx2aczW($uHn5PU_n4d27T7lfL&f4@{Wm zdsDsKTlT`gk)Jh`Gt5+gXa~X9oEiX%2$qz_A@7QCeg?CDO$Cb-w{VFKeLp-A0upl6 z>ecf~TH{}Ei0i#rj7FeRkr9e*&P+a};yS-&TrzI|^mfWedrEem_8`^zHj^X0(mZoe z?L%Ah3*^Oe;#X_&LPVX)2^&YRWq*_fL`JPR782MkMC=l@#$cM(_<>kdPyM+6% z*H!3h^r(^U+u=CQJM#8YSA&90d{+eVHkqq zWyZa1Mfb*8f}^+d9~F{pOx`{U>V@qP1s$&(dBv*1!GY666vS9HGqjUi201=7BQVeVp>FrK%E()m21$x$s@%(hx}yKwUli7xiPnhVp9Z;jdKl>@?cT*|)! zWOBoF`e=&;fS6rI;1M?A&_a(r_v>B_BG(dkFQ-n}jM5nd>DF!noImk3^dCD;3TNCl zobe|rvF(7j;6&K|&!hAUw;GR|;}Fhl416)k*D0jML8U%OJOf%iLkuAA#O* zjYS*HOro>Mkz%OBW2mKn6yGiXn}fwzN&58nLNEHdS`l`3cG~{_9mM8;b7s!#x`Ghi zZBkZd_#$|^9DQ4AcZUoH=n!wz*{*@4f^M z7x5EIobR(oi0_(C%ry4(jYqXl?Uq#AuEe1loW0Dwb?k&0YIT!^soztPWLiIGcJ{wM9g4l4 zq9}?@`x|pJ-Cip>XhD;iT}eF0gY!uqMNIBbm#NQMH3j+B8`RXZ7uPX2}&z z!kq`+Q&u6)Yt|Ci0@}wwaJRxKx9USCIeg?@lOuuf$97N-dqT`aj$I$?vdM(6J#BdUpU+OrU=pW zXU{ulG~uLelv?lOLu$B^UzYilP<9T38)(ug1Ac~?w$p3@3ILrLg`2SZCx*Mx5t_F* zbX=$p7-;{yR5m)(Ny6Toot{o*NeyvZXyXZ)A<&4XK9IW*G4eB=PZ$bM)G<3HlzB|R z9$^KLMaWo=){9nRl#-J!F@r~u`U2$GaTRct?04!}>ed##Qj|scKAVgPM>_zX=caRk z*}i5+`$4G~N_tlWlwnt&WJWe<{1kP}p`H2dYD=2BlLX>|w2V%Qj8JadoPX|p=7mt9 z>3dNbE;t{--uGAtgO#{3+_hvf{cpxhS`M_@0sn(`mY3d;0KgkmC?WG>5%ED@7oiN3 zJ8eyeT5xNxjM&D?y|ckKw^j0Q_{u}8p&6B}+y+JF*)$@qUdWb(gT}&oC37vDfELFV zE~5soPHK}vgc<-Ekit9n!mv$OqnQvT`mM0jCuX3ouBRpvv2d!-BE%t>&O9NDEQ4 z^FMtl7v11K*Agn%)vQpUL>$ALS&Oa834wTd3U=MS&I&my7O+9HVeUa?Y2?wtj2$Bw zqg^`K9*&t^x6-_mYcKJJ9K+Vob7ZA?sNoKb`2jbhyRfIEicYW4ZZd62w|D#WI6k)f zU${BM8GfJQK@*Ms8@h}X8>F$oDqy!`t#P{)g@BIgR_ulW`IdkUB2(Qsx8XJJJ8w){Nn=Y*x2>&{n>dKCXtG^Q1h0Kb4D-tpM~le_V1EOO-o*m5tVb2HiU<-)y2h= zP}qLP!rH88Ng3E8QdWDR&a-p9KY8a9Z+M%n>gSe9O?J^CqEiNQ@S@S?#h$Ny=5^NZ zS$L-Qs)t<($Ivs%FQDx%p8Q4UOc)h45J#u64m+u5Gs;vwH!}E?Z|n#cxLj3hsHF>G zlpHxIt?&V~=QFCY2qvXJ6Jp~bPy%m2=?GA~^ZX2d)8Jnf-Ckv^E$3rU#19Sz@KN{o z&l^;P1V0;t*u7Yd1JyYJzWoa=+;YUO0%&^+=$$0hicRKL*HwV?zYT8PpY=E7k zV2-ie>guv|NfUU}2;+=@g7>xb>idA`iL%mCy-QWpoPKAgbBfd(y}jk1lKSO4167ok#z;qQ_0 zV5zKt5^y*XK&dD;86N~#KVQj8Fd%rO2lPA_m~`;@mBZ!>zkmO}WDu)1oXgzWVd-p- zf(vtSD5K_o69qVp`EeeuBR~InwmQG($3PK}!QO8f*)J~73!Pq-QfB%-{a*_{oVY9O zL)Z!HmWm$W*>%+v3Yv(xi53v13s5892kcB=-O(L+$qQ+Y$t@T92&P&Ac4%Y_h)d~i z*puZ#V^x^_=0{UZU`SS$UWu$qFOi=RkI;FkcGqUyhm8*67tV8lYxjM6b+!C)pr#8j z5SH$5SxUK-*Gv5h%!|V;WgLP3MY_sVd`Jaeo$81gQm%yZg_=Ag`qnTM^dc05Eeqe5 zhe($xKnPhZzV;1dh#5DH-TiTWy#msQbl5EatE4m%-#P$Lp58q}kYEmRCQ@2bMXw$h zpu@kj;!-ndbC@rulATKq>wMGSdTC1UV?hZam%`EuU^VdXsBdOYT=DuCM!-Y!4i#kmwu~~;6ab4Xwo0Xd z7zG0jD9eYu1cPQ|;REsz=PHMF3IFb4Uo41loAzY{sO}UhQwPvqv7y2_l)8Q#smP5= zY9nzNDnM=^D~dl~KccL=TS>L^P+5!+(nJeF!pebsx|?3Z%&kJrS>&uaeZX(Pp_tVG zs`0`ZkPeYCe`@hWL=kzDKO*ps(6(zk#-4zQb@$p-!_t>wWEW{ct1%BNk zkXwMiR;vrd7w7v(I>En!-eai)e$IbEY(sb9h766)E3@3Rx|W&9mqKmFAkVQ`w)xM1 z)uvs(6I=~68w1HR>Ujaq~43+7>GdnL8Ql>=N$b&m*EC5!`c@Y+d zUv>oK>ns5ok>)iP-DufFn0lrOnC*IRlnYEs=+Il7yvLvOloMU~qlE+xyF9-hI8^-a zTRfIUA$lsFC5hLWGPM;KGDew(e3A>wX-8Fusqs6(D_$3Pwi1Eogn|FgRtI86#fmub z7*f`|CjpoQwtRr6j01Q|MY&uYFGLVvgwREX_{3>c&X+pMe(<_A1|3gLSuJtO{MpIm zkyMh+muze$iHL*CAOW4ddw?yP1H_oKx+=6vr)0dn=I1wC)K$1w5+vciDnKfrk#S~{ zIo9RRy>!LbO}mX)6!A2BHwz0VIG4|eF*cg2wybPZzP7J)YjF(!TA z=}FQy)`Jnhb>KH&+WD;X9W%5q`wK;TEhqQLA~a#fkci*mmilc}hDFbX84y%Kgav+L zF(_;GFbzf80E&%PhlR@WOLGl*Nr!nZA*W?Q*NHuO4&(tLjRm$gyvPIsrXs^Dg5K{e1SMcV=Xx{I`Lf|*^KHWu z5^Ilqvvfg0LyN$Q_4NNvFd5*logKTJgZF9~rlX49Fy=q;K`&rDA}l73>Lk=eChv6| zwINm}k}Rm`vGj^puSvA>Hk;QwqLBqpdrviHl)ia3%C1#bQlhKFRo5RIs?M!RksDPu z5%|-61Q;;Se==!=M@2KM?EN_aVsZCQ{TupFv2W&)Mz0UQy{1vcia0?sRHOh!WrlY=)K{ezzyP33txT)lyYQt#AEHvOJu)i4 z_57FrT?iSvpisfaZqA+zAg_n25wR3b7!3tRi|XxxEX?<^E8y|QI3>ww==(p2;QVyC zlvnNe16~hD9<p&gWHtJ) z4TWejXvC&1!<*|g&igs~&hZ?w9DyOgQlp!k;Jme&S@GK?&h{_rJ-4*n|;MsH}IGA^7qre<{)?9{3=+nT@?bMje{mvI`R>i<&Ey~FvS*FL4A)ulI1x))@V})c6-IIFVS18RBF}nwmih`8g(#{G42W?u zNfL{yc^&R{QnYz?1t)1JBObWR&b~r0C8djzNF+DkYj%Y@RaJ-cZBgFR2Idy#3LE(m zIS*RdO-gvDu?K#{w%24^-1eB_vIbepdQHNBrNm08RFROVtF1m09Omk9`{Ap5WX;|= zknj0MA0QV^gHgog*H==`B!K_&jgvHL!L7g%5zjDc0dS=%$8lv67Tl_FApCM^i+s(( za`5ZXaY!*j@Ku6mR$+~;hlPcONkE3**T`Y}#+G;1nW8@)EhU+IeGLY;@;-$(KgCFc zK{4}%RBO$U;1wrYmzVviokwI;S@Xet(;&Caaapm~TWjA+1xiOgOSS$?Ruq2Zuv3Vm z60Hd9^KPPXg8-n~?ECNX0~kSL_aci4xEk~%PnwQ2@_p#}zUCmweO#lSH)?#hE8qzY zmO{f)>8|_?%ptASd7Uhm>&Hw{P?6c;Q}HVT&M#a_ubxas>eCjgkD|JgCPxNy7d5zT z0{d3Vi+;RC86OB2j4(&0n|S^8Jt>O>Zug0K9Ggx~u} zNKLis6avD!EsR4bV+u1pdPv~XEaPwmnq6@d&(7+{<6#8U^4bV%p9qkaONRO;g5qLpG^RAl65 z5&M@A=HIxeaU;|O*sm*Ko6AxqeLhpnhVskiuMW>uJWMqk2lDS9iPNWwvP^)TI&jnC z-(e`!1_JR7^DxgeiXjJB}bpyh{oR8bTH}w#YZer*?zpOf|`6s`glDSFTHc zsj(~9O4VcMAY?gEy5+YO%#&2y;Nh+yAj zUz6E;-4RW(74D4X>9s97);j?*_aLDL5Ac+@ix z+f`t9uRd3pw;-vCi3V{sCE9t^aAK5)4PX*NG_4$RkoaV({|EJ%P727~B0<{~e>%yx z&e?+*FNRiR#AqOOgxYoErPXn**bQaV;QCy%sXA~{a)n4HRbfQ4sI(^r+BHwq&s(DO zCDXI52-l}L)M3H0FrxLZ{s5X{D^-Wp&rfMdB2dK%YoEz=<=|{~D4UUDA_Va|4S#st z904QQrM)cH*)ra9hbz-7y|;)n@&Rxf&zP(!I)uwdH6_7Q;xTdxC z?@N)T$64hpAv9{py!dj-6x-y~PaL&#pQa*F%(dfN7M^-KW^q?%ZCJ@Pk7bnGSn$5E z93+SYNI1}*lxCP>^T^$KL5w4fVMS6Wn#@$$t#!Wr@8PBsdYtvtKHBM999l0oXBZa2 zH+dS&PN+H7Y`2DdMSI0M9T|^9U!X&D%}kf=SK+<_p!;^HV!#2$xepG;hWtGJX;-A2 zLwUXYXJt0t>pbZlLnY`a8)*qfJZWrve*7E02a(1|Ee4Hj(#mA<{Q3wccEnK(7EGqe zemJh1$lfKMq&bNHG7AX%DVfkdP!|#?$O#0&Sjx$Z&iv6Xm^Csxi5hV~3L!fh`Y08K zj)hB98r51rk7gLbz;4voXNC*WdL)*AwdjoC}xA;P_>G0!I!Uue`R`GTNmf2!H1Jz2;3g`PFgn`=UFm^=PyPU;U zvpDIm{eTVe1C}onT`~WtyM>m@QVe+08^b3-!fhuiR24*XKGyJND0w88niCwuKou97 zwEc~IXVT_Qc%D@d8E8R_U;-P(Bjd&*cO}m+kIK}ubzy`Npz#rr603JJ)qXLZU!K~fsEP`O-7j5Z({hwRu+1fOB) zmk7PTO0<3&(x!SW!uuqLrA-R4K0|PfpiLe-fhemsI+-{ZQ1(iy_;4V8E!CjXyln&% zW8jF!)63A!0j%1uQy_ol92ouT8&?I1hv0|)5%~_+>6^0y)$3YN{`gqKsB)Y4y?g2> zGqqA)lH+t}1kHTx#XLj5kX9xYbo|rpXX;l~YC=gL9&ayS;2v$po{!Ko{d95-q6grZ zVT^QAp=d=ZNTh1WbAEaoWUB`m&uk*kw(5gazR9xE*5k@XiWd)sgNXxhHq9vFUaD9R z+Fr5T0P$S^asbCHO!U{`T+lr`?|NO2)==hO5TBTwS1oP@Ov1v#%xcgCqCDMd zg{_wT>{kSL;k3eV(feONeZ*B0atg{$_9`nW!_?F5vleWPXsyb}h0aYyop=-=%eFCk zk1~`oWt993GYJj^{!pPGTsuol*yB86MW{fOe9JnNd9uC9v&VlKg{7&e*rnAo1cy%jl*SZxO8TwWb&E)TiGnKd)4t!+1e4w> zmcMrpB&z>`vZ+fhMH3HAQPy8o2T~X)AEzIH)YwW?GJq$v#{eHzK{Kk}s_FV_m(8m` z><9T7>gPLcEh`O=7qEY_)j6o?G9>Zj30jCkF1pokn@KH{38ypWrKUyA^Q8i_(TO|+ZXn=e0u&-nzCEgAuGQ>PgGW$Y=i}k1_|+=c~2?{|fa+yCnVk8$COXQ3!C>OAZPD7#C*t*ImUA%_3en^=7VYg*#p3c)1xCuTlr`8V($RPS zf)tCu5aB3{2t761I219?ma?0OLD-OI`^eQhdfjW)(XN5sRLr=m6UUS zB;f0rE~&hLQ#iH#{l={V^@&`biq8R^J3k8@ssPcyNJFamx_jHSDAb040NBTYkcTUr zH9t@3d4{@rqZ*+vOX{Q(nQ*dLvyGQDx5A4d1n$((xeLFkkC7l1qlXX5FS66nT(( z%ru?o`1jzCNqZ(g|7ujBXOI8i?SPE*m(U2B`Gc0w!GT=RLAVy=&noHfbpW|ch z*ZK$>6wZ7u-`Fs@l`<9qWaxz?0uwcW$T8UD!m~``TS6AyOBFiBD;% zBECkF`&7dwYc6*C&=AVp51(A=lrA45FQyA)HAtl8SN&0u0KoXvHL$nQ3Jy(YrQr)+dmewXt= z2u@|*`WE3uSOG+{vPdhI$?c{9o-flH^c0-|*aI2{ZS{utJ1HQwS7H79Fp3 zv!5k)KE9(t7=n6PVFeNIvaK{B*?vQX)|a57U{H#ZAD^1@v_q}ea_g#{Ngn^{Ra;R@ zeD+~x5pVB?GS_~*KJfYLz_Ht_OBCGxwRDAqdDHX&F+VkqDiSoUs&9Jlm@|3R0Fr`Q zF_$*|2YTotoi3%-V_@%v_eP*dC^QDBSnT!m>bf_S)I*dZ< z4~8T{2Tm3Eji9+8S}!)Hn(%x_Fy-MW+wI?}N#TaG)JW|7MOIZ+p397TwwcZ4s|~jd z6}{sv$U3I>`N-T{+EkYu8^U}W)w0Dh8@6hdDDibDa4J*s?)H599U#;{76c`jnwQ1t zB=gIGxkxrW!y{}N@t>*H>v(!0p5@k1U)eV{lD&SX^arjC`w!FwkHJk72@ zz7gmPX~SwWL6ODw$asCz(l>oXZ!V)N0GI`XGZ_>B?`K&u5;Pm9tKe4b)=fkilD(sD zZ_lB?)1cxjLbIZHul^`3B*SoxOEka4pc=)5?zLBs0pf99i&3Nw-Ii8l7?1%8^YRjw zSmTxz!-X0q;zgkv-lomg;$*-K1)N{KdZuq>x?N3GXa6zzVqmvKx+89xd-$=YmSqXw zu?M@!0J##miSwyB5b7wB6Q1ROOD9dpULa|U^xv8eUmH#dTgLnD3Z{wzn7eEbg7o%;A&U`2Yu??s|aLBWqDsZhbD1d@Oiga%`y zHK&vEeTq0@##_*x0>E*<^u?%v+Hq9BWsbrZ$|HN2kx{zaW90c0z^`seGQ9b&n~b(%_jTsYE-Rd=D1^F0pQR#dQZrRb=;cB0jozNyAt z9|C}-ClKdwJ(~Q@IX^fV7H10f=*E(C$;5?o!Q!uMtDwVzo(*)OQ!RsLpanJYS;zqi zB%*(yLbFds6B_wF(Lm#viRa{xFcYk;Jkgh|t{h2r{2)HD@1DAb;$DB+5ogvLhy( zjxtu?)45KbkeTNpjIKZczE+1}CnBi<9WD?no^bbTgf}fb9K&!c^j86pK=kCC{Z0tP zKtUH^nh^H1exSw{hgB_9)|e^y2>7eKz=$@UA9-tc_kbHg9pIZgCokt9SIkCi{vltK zg++-Z3$WnW^MJwDR!u3}1BKiDaGpplrVF3HCoQ0<0shlM<1Mpi` z)*tNAmGdQQ{E#crs^DJZyjqEJD*ifJTpfnPxOV`_&Jua=eV5;nt$r?ISIeSMyVZ=y zb4E|3(kca#dWHPP&2jVs9$uIS7F5sBN+t(XLzw&7w6hHWgX(kqV)JciAZWLZi2q`y zhFK20N|GDLMa3KML=yNFzXR#{gn}p5#xbLtn>V=k&J` zrzn(r8yGs{#oHSQ=xZG{xVHC+9$t|4Qwp8+HY52~#bhM@yEjD8@JHs;Eu)XGaUzxB zB!Nf*ZY^kMw(lBQ=8W2Ss9mcRVj!*Yyi~8P!CL)iosU>laMK@fX)EedX@17n{6F9u zfKt)WizR944+ELY-T|Sl$yTh=$G0NrSEq|L3cko_wqM#g(zHX`2%X49)P$|IW~s+* zVa)yVZxhQ9w>drD1!y{#ynndJp>=io@S!RL`}K>Ml$4aT=KwYlN(XLbM1Dbmp>=k+ z9TS*4YjxFtAmy;Vh(B(|@8g}u!t>ZQxkRU&Kjk0LGLgg*A4vH0o>t;4o}m0)Nn)gKn!m4g2DMlIM?hjuJ^VXo^dr$GGfmdy#`2mA>cB6O0 zr`XsnuT-S52jZ@y?HAd2G6ZbL*_YWo3Ks7&0Jmlh4FllU{$syYyqDag$6_ePqOu^m z7rf`zC^zndt3o1TW+cm?gx1xT<>Jj=x}F9Bwm1#0NBHkR(jXuCfV3 zcQ>aAA3B>!FBCQuzJ*N=ckZpuUYBIf2%C2MMMGbxocbzgV^yLt^b>4zJ`X2B^ASD6 z8kFPKS~<~;l0a?Hjgur{oUUG|@!&)hly71&%Z@^T2Qn&{P@k#J(l-E z)!O$s3`;@f{s6(TST=OX9A6h&?F6*Z+em8w_N3wVjcMG?*YU-tO-n=D$(*$4y8=-+=Kr^fJL zOdQ!Ne{+Ptm1y)h(37Z8-I9klG?v+)1AS>q#GcLMU3H)pA<32w_htA zBcdA1nVK zXg5_oekim(qq-=q=fWkXcb^sEtX=a_Z`4(BgTbYi9l~nzHmadJJD>788hQzuVVoj{ z^nY&%vM%ihW?YA@6Qt%{0MpdY?igL>#>_Y+%F2W!jy6J3b>MAb2t^rpG1G2Oyl_4A z?gl?ib7{Ytr&DxR+iwW*myN2g-?FK$h*j+NAKd+D-hp-=!>Rp69J37OM!sh#*oJA+ zLyyxeR#Mj9-B>Eb8~FqmGQ-EEmruG|KAniu?}SP(GK6`;ti zAhD~{f>+q*OwuF6WX)QCs6*9Y)@fF!3JOB_v<|6(ch`_dGb-h-cqrXUgZ1574HwgE zk8?AXLZD`q1(>mWd481eYPRva@JBx(PVp7b5K&^`FDIAQL&6F|FUDp51w59-)+Ft$ zVm4V4P)iPOIgI$WET0N_r$^cU7~5P61_v&>16TE zZ5`kAG?v}TT&X*^4($y?)oQ4qyOILau_h5EB_r-kamFgAb|2|-$~1;DJJGYPg#Lsp z!#fd19hg|96U)_R6$Vj}q9fs08AiVLdaSa+pU7VJ&laT&KLkR^9RLo7?D_Myx@|R$h^X zjMP_fK?se$?omAUTZx&pw-I1sMVw7jn^|(d%eyT}+b9JJN5v+|vo7?%z1(OYf)-sj z^of1tsU*D0lRtj+mM|mahn>8z?2wqx>&U&wLPNFIvLCl3IGSa*I<8-dU21$j)NG`ZS#O!t6H}{CgggGih6Y z>?^KyBue%s2L&dZ&~gDJQs^b++c?I4_kNHr`SNxhfk#tA7g0QifXggwL$C zdYfb7*IKZA^^AenUW<(<;vmLWa>p zi%P8Q7vIF-f+n{9#LZOg6{7FqUw}~4T0XtxKsG=`KL4;^wZCDL>&Lq-Dfxmz5 z68Q3NP7UT~`u^>&;1)2Vv?wFY+ovDW$OsU4p*jBVa(Fmq`Cr&Y%uIl z%(Peo6oAtU<`0O4+_5~YZ@UGVGZ-sT$t^(0`8!7IE7oSZlPuLGZ6vfBwl8&^zE z{mWn60ae4Ug3~m8Ny9GCss!T%JA9WguU1$u0;jIX#&Z%6JKgQ9w&=uH8Ri)-jv;A8 zC}d?!pl)_<_Cv8CD((9LV`9wiUpe$$bmbp>gvgk_oa0 z?Y%0e`yNfkmfq;F@RPplE7naWDF_c@l!UwaewKMqi6EKETgbS+vG}~NdAS44{1t&T zQ8WL~*V&`mACiH-OBV`iqBUW3+cRJEL8wAYjlz0-nS1uj)J(CG2RT_Fz-C$r=*xYY zHZp|o`zBkxEER_4X##cJ0mPfy3b7dZj!`UPLT3z!6K!Isa6p%-r8h-2-?B!iED6)T|2A!~01Qmf>QRF5|tJCZRvR2|3Ic(}sC`*yq}OcUc<5}7dN)DK)1$h$f9MBSyI~RQYca!`>y`V5e=rg5z znZ-tg7v9G)R_SflZk{KRfbG-({&fesuTa#U&6-h&9eE5eo~<(IAmw&^4QhSl8Yg~E zi7!v-YZL%2y)v)*JnZD*c~!~k9}pwK0v|{4G1+33B_jzNw6Fp#9d8AQ<(49Ra@`%) zK)ZL&xq?HS=sbuY!lFi8je#ag|2tVNA?UrS_6bbNkWS3)4VUWq?|SWEocU!OF5NAZ z7ic#y%)Dg?;P?cytnY-moc@XquJ1avaZ7RP-dBdiV4cvzj%Ju zx~{3^q=os0Rt}s+JUIxJX$gGN(=t>+!zM45-C8Y_w*5UDN#W(CPIbmRJC*`^HK(tk zr>0%woTEdW>?E3oG%Pd1G=*GCf0W}ln1EtS$wo>_V95<)FKzy?h-N-FevsVhpr$%E zdksK_V8-0`{I`}xfv!P~m=Ni>B-*NMsC9pFRmUHWV>JMF11+-xg-F6-p#@WcI+X-E z2z!it@pre?6@)y=WZhi5K%76J$g-zY)|}n>axE*#_8l*eNrHC!ef~=}xk@iZWr^Yl zRe)~^!6NP=0P)tA?q}@yl)Vy**l+G!ad#9A@IYUY%v{ROwrbK_eP4+Uu~8M~OhGyq z=g=j`Wh31;AwbD%qO>$U1k=!nt@Sy#BBT!__p6)vNrL(-DejN{$x}B#q|!i#CaTR; za|6msq5T@HynQKI1H=$z%E8@5oL4Cur`RMDaH`&J=G|Ww$w*~<`SO?A)jsm*vpL$k zw3JyqoJ9ZRus=VSy#~ij0`~vzEZFl36PUj=~57TpxNV$6TAe05&8Y0N$@9uaiPE+MLdh zg8vrTT^-H|{A|BjU#Dpc%SO5a?Oki_h78)D5rnj*7C-jcbjGF8+T>>By9wu$U!nue zo_GC*&vXY^$*cu(p!kCnYKx7^V4j#qOz_#QjVL9@`u4|Z?sR(D6vrQ#I6HFY0Tyoc zc)7teILDZNmV|ee@TkBE&`~$K0o};azRS^(La?x~DCC05Xiv(&fgB=Rt|NCcqOZ^o zbY$ZPoL3r8SQCl+Kg44})cRc*)1Pm;0rkVPnn*h-d27R1f-tsS)Ijm z5nj;5GcEgWkAHmG)>X=jBLX-bvf}Tg1RoGKd->{q^ylTBL6r~w8ajrg$h?ep&=~uy z5zF)~i{7fYBZt7T((8sxn`$!S%P>E{w`~H=-~nc3^Wa3c+aChXySGape8+U-eKWl- zs{9yiR3!Q#&d+jUL=n8l$62L*a3)FBnzR!#sVUKZ!(9t>;Ya1}N5EN>4cM+q3J*H! zL~F0%JhTCl-rT`m%gH2_$@q$l*O+edCC_bj?` zt8vE#4~un65nSt6zzoAo2q=U;QBuV6a3h++CSrx-5~)bP^x#keaN$r6SffF_3_7%6 z{}vw`_0!_N14Ijl*mc$8okssc4T?p*v;b}SC$Pu{bPEZbWE<_Xtk$Hv*MD99SQbU2 zNJTy`?s{nm5f;A`7gKh5kJNfjhxr!*A?0$eEGuDj2)$9@a78)S2Gr? zFbIyo(-;Q``GTIWiOJGrQMTXVBb%E{sri9P^p_;Tjp7qbH5lM#;kM?va58%a4r&uF zX>Q8h;9i}o>ilbyl|qwd+VIdYwlDmN(p{f8AHTQpE5Cf~SWUWEXfOnTr|Sur#enbn z3wg11ezBXS7Z-d19=_I618qtR?wRvah=cRLX%1MDopt($UGJ^TWAy_-4nbjUPd#h0 z%n6C;zW^E(D@=fkEND*+BRF$+sg~+1=|lAQmwwHm^FOW;PLO%1}8*)oGV;86A*qkt9 zyg+k4ylM%&^TO;n3J1{ZP2Wf#u7#@GDr$>NI*A+2rgfE@rmI%(+Gm(S?}xxAnM$#? zaon$CqJ}r_oVB8(j2x{Ah6_|5n;0P|bn5qBYP%UsJ5Eeu@iFS_io2pB?%zsjL?UT` z-2tXUe?2|E)`9!d0+o`Xo(BJ5yx0C_ii?Y6qJcJ4^?bLx6qLv9Njd}_!V$k@Xa4j4 zew%&(dGmg1zVS9T6ZHx}pzDKDLT>uDfuD#m4gtK1Z)oMWa2Ga8(C0Zz0e9s9ZFiiJ zoqm$?*AiyTFV4@{XH2hfakmspXg({OP3L}P7dX6HXFI`e;e%$Z7dWB(6MZ$ghcwH` zzP?`w@_ANsvyelj;?;6~i4BQ0j*-V4m(X&`43|g5RF57ju#M6AbG+UU^^(&6_X2Q@f1TidjEQd;?>@xx9F=QgbA1*w=33q+2c`5^GFosj)q+6_#M&@R$$nkL0eOB*eUAwqGl)4c>cGWPg=!U_7C%$aX5qG+UOKl;n%odKr_-)&zTTmP|^Ok^zZLs zs!3KfVdBwW0x7BC+5NhC%8;gwpu6FpYpomb56A58U-d!eRv9C%$N4G&w3lz>RZwn- zg4S(kqum=Qo_81*`~`~A7V6%VqTt064)%`&wf9N0I2b=Vl~t4qyieEh%C$E<52kmO zHlKa)pOq``!2He0xvu|gsy^2w-=ct2S3k{|T~&nCfX{eK8j_y{ z9pFI$hJz?SgZ8AWb@m6D5i8G#Tv3A&V2{dTSYiGLG#)I~LKH>9fOCoJq4Rh8*GTY$N0~CNm5A{r0NON)SXUsWzq{%QkQc8sr{q^Cf=z6X>4ush^0L5aVzVU~kIMeVgn*bbQY8{BWgZ^iloY z&y<@)cNOtu0`_8&Gj%47?T%;F^`b9M#q0%=ab}fC&{DD0gB`JeACQy{v?8!^ggVaH zHyBeuIrjHv=ylA?K-C8nuwY0;(XF~4e2Ix>1f!7u=~|5RB+7fk^7_@>dV=gQXRLfr*5B~0;}I+%9X-Ui}w#c z^TOV0_);pF-DhQz}fTe<$I%0jUf9cSn|amEBwkH#c?cY(o$2u%+t4zQmVUaNAaiSplDhyoYLI|1Y`WW5cp>jgK z^*=b9q8Ce=E8*LH(PT-@HkdNYidl{_AMl7X3`6|gRKJc;9UIW|-c-UguxW>N~3bq4}Hc}R{3m4BkSWof#_+L5=L zzr3q+h|gXvy?2}}!xk^xNdkqbDCt2S@O*jcKi@VO$iMO^0B$u^IMZ3~1S+Hh51lAV z&(W=3HBrq^H5r?Ly6>xf69F)XB>40c|4J0SbI!;3!l!Oun0|a$^r0sfMX{RARDij! zezb3!##NR7vk#r)1*)J*JDHLSo>)LnWqaTwYIha zz0>WlPrq6IIV-onAqPJCVr*1N8hyUZVVHN~wibAxl8juXKTA@7Rf@CalnJbn2G#~j z21fcWg5MZH+-ZQ;BTn5n8TmlD_nTvxZSh<~`tN6Y{$y*5;?e;F%y(58g72m;a89nx z%MC-uUkG#QM`62D6()e`dwsh)Suh#UMCXr3$v@BlO<(*V+G$Ge&V@=Q(A?y7UyXy^d=w_d#DtG+;?`+orjp24d#}Bks;vup&As7H< zLGWIH_1R~uwefFFKdS?|4$+FL|4<@54Ge(rCI>()9c>Wpm^%w$rxYh**>@v(@U#wCB;P!OT7CTmR%GNRt*&b_oh4;X@Tz zjBxduoYztxpB1;`#hUlp&!2C%=zQ?KhwFs9?@>w)2AD9)rEB?VVoJ!4Y>S?AI^s+9 zsp5)ukmy7`_Zjs9l&+J&4DUs!PjoCh1fq)zbs~#R1(#frOb*o)(zt}}8`ERF;k8$B z(|Cn>I+IKtje>O*RZD%ILc~NvPAN%jj%g@TAoJ_=oLgsW< zjZe*7$UWd7(-ZHlJh06B2o$CE$y0H#{PjQx3@kWn?@%9@?XZL4>-dPKaTnHaCbUqM}{-P+88}A z?TU2=_|Ug&wqtDp$Fq%c7@B8D(}Vn8A)jiEejE$8;d=f%`gu+Jmc)e7R+L9QH)Y_c zi@EY&`B;;wgOO=7Y-q=D)AVpzVsjOg{o^B|#1X(a+dPe6^-o3y0bSM9Yut0t-RYZP z{_<87Uq$lPe!=!#l}}V z*XN>G0fcM}MU3ti@ziaz23>USqzf3Gf1161uKNCa%!GeWA6U$AaGGYxu->j7bx^?N}^%~dp_n(e_PCVMWce&+t~ zZtW=f7OL69P{{!YU<&ZIu#o0`!XE*v$(J+^y75P18JPc^>0i*khJgsfeR2?vjYD<) zIuh!OJ zgrs*%ynG0s#N}6Ln~Gd6cuuhk^pL&ds08-r*4zEfW32nMZ*eoUC&blWupp7Ynv>Y| zHCff3vQWupX`)p!_tvYQq0*y!isB9w(pI0KpdqYtv&22sO@J4U;1=$-4&*#mg2>5B z(s4Y9zMT3m}z=rYkL$HOr!j~Lk_qlv#nEl#jQ=272EKr6U|pjW-13` zBQFwe8+hY3qM$2bKR^btY`+`F8Y%T|6gS);kExP6%3WcgBifU1P*J5S6*}KH`~k_O zdZX9YEjkh1{7y5t{7jU5$x47MD$TMt0-}@H`C$!7c$?U%`1Zh18|~dTZ{*WLzfp}) zN3~P&!?#O2auLzCf+yUfgy&%SU}5-#h4Sa9Fvfswpv3|sKza|jELO`A^s4Z4(OHeq zEwdVqJ_msH?k$0)SYZY7)y}D_*6kSEO8&4(fTgh~L;3eg;o?1q>O<;9PW*I7`6Tn` z^Vr>YG6+qJnwtm9BGK1daK|GTqtkZ!M<|2pzz}>VCv2bZ54ZNX!;mH_Q-?Zm=4Y^D z2q%FBLheg-xpE0Pb>h>2;>s|VM(mfG55sAXW9b0Qm=$%Pm{k1&LVWmZcGjRYdv1`! zvcK1K-5w>v7R?MJMNt7iq^l0p=;=d0ftmky1S)ILQ17u5RmTf zmJUfN0qKUJLt2myrMu%jc<=wdF4pn`ixYeAv*T9}X9T@d74S|_8NpD>#}p339ptdX!{uM0^Rpvx3V6sv zf3)k5uS6EL01TBE$IqwGasV;!vt6i_F$JLqz0;gAOK z!VJx$0n%6zDO@IUNk#9wSps zW2`4HplGe;+&2SaiIroZVv*Jz`XI-vS>hHDJZ4W};znjB#rxTcH#uucqj!z7WsX>r z4$97kWFghlR?c0aRu9&hZ2HPxcyH?&M@HB^RxCoO(eG8FFwhA;j}A=#E-|n;x?K|& zLlzr2Y8`&B53Jpii16UhOUpr%rB#%5Wa@(Wbp-(D&L%5P`_{{qH#yH?8LX0ERc~S; z;+a6I&n_Zjw1!Bz5Ds=MUsY(QwC0>)`-M84)~d&d>$~Atp0_&0;f=5(!;`YX1xRkb2hm{^75J3%v8%*S)AMDPijNeaO->ak&NG;+HSHuVEVW%LDq)n)Fz5~L^2 z`!-TX56~Rg#I6h8e3RI?5|~Z;h_qsh>jk|1f}hp!)Z;ssCUi_PO)Y(+pKB**yI7MZ z4Fu2qTutjoo~9Tf;dko?2Mr{+azCGxD^_Dh&QtqB{|>?*%SJU();wsO1wBg*k7uo| zt(_kG3>o4fiMh2)YD-T_Iw$?Gmv#O}s==m=ODK=?l7wG;R3YvlQB8Nevx*|PgakSR z{rWCi(1{Ips+IAO;E283o4T3m`^w3f(Im8_kP-o!Vmd7!bpScrs*X+q9I%O?InJj5 zR8`@Rer)czuS-9)VM z?FFnypMLP=c4SN?)FlJQ11w+UvNKk8fdG9Yv5c~Qk1l^i+X^}P&V*I(#T&f6xjk1>rF=;F&1| z_>m&Cez>L&i8nP7_>7o!O4ZGy9nOv)TCGAeMt4PCvVbgE=XLvYnt3&@>Y95zSV^(w z{rDjJ^3khtRU}qm56IWc?SmtR@3m)jhsqfZyG~IG2b2IHfS4tuR6CL&WBza+ehBST zN_mM}#7}}_iNB|eFc2!Fz@-2|qoqE-FVa`^le zS7Jw?zJ~&3wNs$7AvS9}O(dMd$pjF|a1+756^qhxx|q>@@m%EZdW#i)M-OShnR!g% z_N<4E#_a^X)zjE=;+Q_n$SYi;X(d8GibT~Q!)BROCpG5r%Z27` zEHTBcs0=t=8~Bq|5D*cS^%4i^Q5qtiO{*?N{ghbW%eqK~r|EtzN>Lj?j3AtPJ>x$I zFC9$k?w=W(Iti?e;MMR~gUT;!-9E>e`MG#QE@13odb@}y9x>h(ErJcoN3q|>y>fn(9}vDDZ6GjCQ{ zAZXF*Z1wU+Xg4gxDGwAu4#>SSth!tnSowE5*!cwjrlETRQm!NeoSub8*}ee#YM z#*d@>Z&2(sZ&`Llo7fMH6Ce`_sK>$^N+fhzCdaAV`;j`d?VKUM2X~0H?f51QSF^y_ z-aZg_^GIBC-hnp#rdDX@nl(rF<#nT@Uuq8-7 zCcey3w6@1${w7k}-g1>2?zLOf05f9Tc?>pd?Tg#6HQu`c%MG?Zw=e2V&nvkf4{m34 zMm)kir!-@q6Qu9=v!h(qYwz2S>G9tcO<+tLst%Jf!f~1pWxbxWtax>ze|Rxd{+@~W zlI!cN_;8rgx?*{Qt|jHlaqbWKWsEGuqex{rDiTQfpsAQzD|JNl{lXa}S(~Ii1-lxj zq_d-r*#yry-$mfg*BK&L7cp*H5!xDT7e&mhQJyQH~#9C%>3t$*KcM& z8v)y6fcf@e?Sd1Gx_kXu=6BV>H}yQY!|x^@))xFsr_pX!L-f%^qQkXBjQy7zeRWC% zx6}A3?7~(KC(lYVad0&O0;DBMXc5bUNlU;pVxyGcX@mL+Tr{sCr^xbLI=L$fuEW#P zn>+tlgBZX%4-TtwN20AIe`l#Ij+?gftQQ57rP_ASW^|OOBV}~(1oB`oaOyQ5cm%#d z7ae(|kh-mRr;NH}eGkkN6C8lX=P-hvG=UYRkUzfg`UFOua}6Xvm+(22Ip-7bMHRh# zPFs1Ni=ln{m_u=P@{^5K&c~z?c&70KNP}q6GN z>|E)Leb2XuK=s={?O(!ApJepZWR{&Tw+D)N2PsC7N=7Qk*PsX9Gd|xy{5v?eQG7)h zrMqk3(~1e*AQ>k%`HSFwB&>t_3&(*EYriKH;c5E6!wBtUWpv23k?m`qEdHC>;0fHp zS#Q~XK!m%9AG@&eMbf$g5CqjsW~`I6J}V?zYw^4YQi9ys-&Y*$@7opub8=PQ`=scF zDmq)im2@YGR?>=}{O>xk-NnN=4FV7M`?HP-=C1z^Yq}uLnk~cwtHyLBoIK|d#=hko zWnm+Gl36l=7*Mr6@j1Q@1EA|A)1MPH0>le&S9SUHnfqfX4UlK9q4{O+x^yJ!`CoR| zHwon0w9=TcrKWFmIy(6hAV-np$SbcJ#5phu6VRehKiw2?)?xtZ(Kc_X+{@a!G%*O7)BLPB{6MJbJfjRTc{^$b!6&Q%250b?V()jQA1h?>H}Lc(gTObLNxGbtL*IyE%{ zXEA5-4Y!5tk1lc>O|05v9A+cP{D#S@eiXVS+zdmd)A#+kh7r$M0oAC|be&!V_9OI- z5<;uY?H+60!Pqsbpi9WW^7HTOZ~;{RIM*}Mq2-{#yg)X3HwxRG(Ijs^CWTs(4Z{w! z^lhqf1;@N+j`x?&+ICbu68s%qZf_&xX;VEdI)~4iG6boy&E)+esKDbNUFt893qa&# zrFE^X!hh2(vOVAwIr#ziG-6=;{&O93NlJK@zWo9T4MPC?UUKZglKZNhp5X5`CmuB) zy<#*VRINM^n<70XcWB{0(nY8}X378kJqDs;)B=HS{f0W6>Bmk@;B@(}Gd4wJxz<&p zlK_6H+K$93oaR>&K)>K|H3o(UKcp%*$a3bO|p!Q&Fr1S9{$r`+^N7XHJa@kh?9MVZC$H zV0zi_MasZi=o7&Ui5_wCI}J_bNg0y}I2R&pJtrKPu^A8pz04u*SE#r5BQIe+=V9Lh z#i3~{C7VeV5CEzGP~--wTZ2@8u4P`%z{S8#l)hq&e5EBfJNP|3W4~?E)RyObq0@r5 z#*~jWmukeb7x6u=W0x#0QjWpgSe8(SnD^~DaR?Uq*Gyp#jkdfW`<3ta{*BHcHejx3 z*6^Bo%}3ezaJl%opgtT-Z8r7uXE*I_(9Yz+mrOUFuir(6&pYm5-Bo=4 z`mWX93{}5p+qw$b(qIC(54P`C!Ie{y-*Ei~LoN-3LI@5`RN&(ARDX=(KV1i8o!r>P{gkXd6 zzzGiApz*&~1U77{051dFPE?N%yO19*dtLMT%OzTYrBU>6nF-dNZTCu-C>HxMtmqW` z^jaOd{x7k_?6QfMBs|0R-@~nKK~cbh#u^1@KXLx!KyDdp>zZl637WLQqZT7Z=)Bau zI=ub-3BCj87V9zgq52R0!4J(PxROFNmxx%Hj44dJGI$KpE(_f_m~1pHU!Syo&iE-p zE*>?G){mtMPgVIWmi0*fZ!2g5a(ur8g=4k)h0znT?A>W-xLaC>zUC&t(;GP?|Q?-qk#}lh6dT3~< zI^x~N!E-JVQ=f?A+g^93zI{t#w`{YAUyd$2Ivo)QofFm|WDJuL2iIuAGr4qdJG z=mUkV5L75D^ZjSB1jB(G>2?f+?Fh*wVm>8O0A}J#a~4>X3q?%c(tP??KtO{o8admL zQz)JQ0av0PBgOYnocT-uU?`GtY7)S71Y-$&xGN<%D}tM3ynI|I?)VetLWAxULD{Gp zRIf2&Be~(+!PzW206&Nny(z z+z&|3Ws?s`uWUV4d`sSOGnMmLhmhqqbE?+P)|w0`cXJ6W3YOk#Xqd@mn)$dEc+=V@ zn`IM|r7EqiESFMY|7o)n|H?||q->mh`*mHT^4zZeKr31#fAIl`%*J$jZhBx}vHF2J z`1dRR2XSzhJ!Ft4I5>D`G+X5J$g?#P%D_-ortm_irq69O)c4Js^wR?B2YG+}c2c_{ z^mpv6tkG}lEY-)pIJ8^!te{f`v8d;ILKVQM??LXbC%C%UnE>?0T)y_9QS@|jn_#mf znw(D*F<|NJ&t$=_tAvceG@dc$$D-@)FAmGTi*`R;8JK}EL`P*E9U?ka6WlNUM%>eJ z91i0m)F0jnX3n&5#3*ErYw&J;d$XbFv#z-ufs~>aweXl>y`_4bzMAvNxle|_{OFP` z+3I<#B8<@3btc<^3A@qbC3o~ZpsEl4I<=zL_8IS^) zyoQ;XSyD+!Nu2iNq`iV^kl$lIj9Ba~aVtY*%V9Hn9#IE233dQPC-0uA63OR{V$Z}W zDT&N?PoU$jmH-*Ufs}#(0}-?jsQD|%y#2sI#SYi(9YA@|t1iNQtS1-%mPhSl)mdA&duIquq+++hFkf|$il>iti}RRtrEzoRVNW*Sb% z+ku?`uv=eG_3eC(c>zH5Jf4X?-Mb)xG!CNhM@WE}l1ntEz9UJ>M0uDmygeT>dW-G zA*2-A{$7N#6O8MoWw=zu;+`3sv9zCU$Y+Jm5<7i(__z)E{RXj9lW2-5mwP(*7* ztu!1l&j_nnamE%tlN`NR-e&QLA9-!9I1)(eLvbF|cA=S3H+B*b!2pP~i692m@QYk3 zoHzB>$+zI>$jI`C`-mIB(x60_i*FWZ!1D`XAAX2X*U)MArxLq2MV5n@GXYCoryi?WfwPP|OL^cI znsI0p|FiF{`Nl&6ZNw)}T(amfbUE+CJWD2ej)qeg{C#EQShJ=U-7-kz*?Bw6xFuieFK ze|Xp-U_%kI;=YR$)pB&OHn^Y{=9>X%d52M}`-wOh(bXfw!yFIkO|Zn{cL>6Lx4+9< z`rg68!J#xx^Q4^nO~$ik*S!Yh84;vK5J+r>S3S&X+br|URZGJVUxDc;WaP6HOC>d9 zrT-IPdjdyord=Vi^l6PrL|g%Qb_2Z>b4)+aYc7h*Nk%^jyNq^jyGk>cht`JGP_Z(~ zv5a;k;^Y$_>!x=CX39YUgv!H|Mx}8OFa!i_X-Z50ULiE+kXg34n@<9L*1FJLXtnUR z1`E{bxXH5GNqpNda38gVkE70rimY8{=8hxeeZ3YMiQ7i&1jr|><)EH%>DAe{FP_j4 zzD4_~Jd*fC?=X7z>KeJuocbf8yp3#j?GYHQqSNNDiru(kI+ZZQDL%1DbF_bj2anQB z;9>LKUL04cKm2w$HEKNp5 z35X-R(yT3~Y_Kb#5S#L393DDaCIn_qz@*SA6;N0$R-5NcfthWdMPQR{rds5V|Embk z+(CQf4+$d1Wg}8UI+Hv) zHv+$yHV)^wm|KN;=wzN$=uh4gZF_rr(JG^o!g~a~K$pefn>wW=^R}P;)5+x~0{Yk? zK-JJb0b{K369)`*bisV%0Le()WW92RbtzjCZ-i{KEP_`mPG&#`gwQ{E}vFSFsdmzb#1ofziijZ3;rtKp7!Y4C%0p92fqFeCYt8 zK;jAJ{Q}#F)w4Rdz}qGOacZ%iv55hyW44|v7;yiW|R8jRo2UcDBbZ(1d-v|-bNDzeb_GQ9JUt6FE|3F zkZX>TVgSuI4TYm3YyIHrT9QIxu;5aWXomA_E18dOaO~|EbrEU1kC1-(YA5*LVq#I$f`2*Sc zDFMt;8VINN637Lzvm_FP!5kQE81>i6()Hiux`NgbsUi&yRfqA+X?eWg{90LUzl-Lx zhMU`}P)A<&bqa4`jJ!hlN56fG@RN=47TkQJF2Ep>TTFsZTA=FwqR$2%D$?iUaSfk} zX&X!Fh8#dqY~K8W;2p|Wd=3pAYWKK1@8d3%A%O2z4`O@zQwX~?6*!?O4;%2c2!so# zdR3sT2ET*7yRaERhO>mz zNHF*k1z-k2c}U^2ca*h6|0cdq3V-IvZ*Fe(ka4KI1W-9Q0JHClASq~;yN3iqKGWt-BN(4_)8r`o8DI z2T27(lWnUB<|}t&-*v)^!a4JRne>9D_0#<_DsSbqfX0Iwj=>m$4E{hpC34)AwI#{|h@|1x`PvDpO~uo~E} zABRU!sW!)X3$YIl$UFh4&r+#fjWsDoQSA%-1lyppq`#MEIup0p4%P& zJGp)18i#Nn@KnE6DZSL=;%pgEE|+nBVSZoC${`~;?YP@+r8=lC6EgD|{KzF~kX&i=mblH{PbZtqn7*`8jaxu>H;+fKn`B zT|y+fiNNuW6NTCG1msmxp(!yYeTab|7~_(`yMNftqfr&4Eu_MgIeCFbo_N{Z(bGCS6N@@6y~>O$R?m5n%T z>>@k(Z(_{AZY5HID#vpDR^nDTCvIj9w=3_h+lo!l*U=(Fu~!|N^#8?=W&sB~Oe=Jr zE~`gwPF$cJZu45sUnj;DIK`gSSxu27L<71+-m6?6W7-^TIiZS`3Q*LwhX9fQmH?XQ zgmqGu1gxYWP1GO7nXrNoL#NWvV*}8cCIN3(npG?xx8MTfeF?Ma^nX8QLfU?RGb6R6wW)5A@G-MJf#7i+arCx-@hH6N zmk-?I5$`1=fd2kkIIXo%vdJ-`{(Dmco1ew}GqIstNuSum^zRzwXHr!YJVE8>HP4Y` zpehQNhd$#Oa|784ETvCWi4aKw!awCSM(!{afRG9(#W0q}-*eD+e3a`6>>5b9XVw$!yNj&7K5NY<5~BHh~w2ZwUx` z(@HvR?v?x=gE%D{(*OLuX58@5U;&Q*I)KZ`%j*wjA7Zr-mETM?vSApuN7?IK`axg< zDJ3wMZn;{mx+il5zW5irJuz;JC&xCf8v7mWEv{INv{v z4p|Ojd-NufXjp}6c%f#Sg(OZXa1sKbF1(~S8gB1lkIgk^&wIn6s#R8>pN`ow7=a-q zs$Q?1&Q%oNqk?^$OA}4(1Sm~iE|^JlYpcdE0jCQHewfA&1u{c(i(ZQDx{U~q*lDWP zjw;@1q#yX#94o*&iaFTe--~d!9OB6VHAqA`>J)@mA{$ib#-557>LDHyC02*D!;Xi+ z1~4FOG#iEe)V!193dmp506)H_SAh=H>4Om&Xa-p@CC+jfNk5)~a}vcC7t+#t*_3Sjqw_NI8AoB_@lV&Krjhbi}4)5*J`WCwuE zv?3gu|9*Y=vE&?CNgxNLr~Oh6D}~~N&{K-tD%}5kW6qA)gm+4Wd=ztmgl|n?HC_H3WX)5JwdxWNI?Ic%NB$wOvbv~i&S%o2yR&Sbk zx5f$4OG4c=KToOCK77wdMs(2e&vLls-z}F&EkfU0%d5E)A=Eh53 z>GD-?3EdTk_M7eBVVX-=cTr*Mm_JnP^L!6C;;RL+6 zlQ+(&-BH=KirR_P%C7s06sf+xif;rn%J!Q&A9>0L87F>v@~~yx@1Kz z1!wL>%%9|;4GSm7Rnk?D8uvzBTY1nYJjGLGJ+@z~x?PUbEV7x!F9v&K($nyzfUsyd zSb+{MLw|3^yR=<;JNB@od_TAsh{%{~x0eSkRF7+W67#&(40@uZ!;=~toh!+Zp8H>2 zV;{Xp%`BR8uW$XEXTMDFif**fOt~mKk604=7VcaFqSY#ICkCI_3isCCF%i9YK(MRW zb)@lqKHVJ5n=ucF|M#j{0wjffrc|LmQrsSw<&<0m+i$P2XVdJvBZkbnGn_)`$O+yF?z zldJ!(4Uo|eKtDn$p(GhMOn@j=ZGJF3$p2knnx-~G_K5OJi+6#RZ*o$8(}o=IB{+-je)BazS>H6Id$YMQcBJ{Bvd7 z<*&+Yp#!Gw}jfA;f3^S3YcNYAX3TWkNZF`!9AJwrwp7>Eb)N`fme3>ji1-hx=cxbSANI zi76y1pS(SSe(hu@JFxh}R~7{w9GEP{#@DC@>a+{j?#ul|{vP|PR`{38n_RLmWZ=?0!X88heuB6Kau`-{4pT7=f)4X%8c~}zJ$l=?740MTW<#f z8v8NVbIK`W`E@1dibQNt)W5`qgm5{M$!k5KF91-g&!x@tqM3NF*xbObhBYDLDE`Vs zB%!NXsWaS~OM&a)g?gAh7^QaD3Pm1ikjQDR3ZKP=$dzueuu!)rge!n5H{spJdVtjP)o~MASIWa8o=+Ypn>ut;x zubGQden1z;IYlN%*;fwwy-;Pce`2(J_U>6@SP>0J6~J6#N`{J%Ov{Msfa~+DEI&gP z4*3yS(RZ2%)K=s=abZCN5-TtN0_e{p;^u}uL9xjL?{&SYwl{KlIY}OO3-WL%pse} z05**v7kdoAW@U~l_T4uEIL7SlQf&R}T6^WWW(k;y1cYEX(p^t@Sx88o)9V@>VS*yvLAP zIkzX%9wl~f8jJ4d?gko*?HtS|V#}j;6iNX`MUNC-7^E;Uzb6_FBHc{E;aIj+Afq@& zYw&kYlH`T2KjR;erWq=&I1wsC#F0foMV(Q_6E3s6H%Nd-@Qd4w8sZdN(`?gH$dXMU zJ;;^pNsvw_u>d4pSzl&N^sHH&v^JqLGJeL~mQOb`RX*DO5_BOZY~xDVQ}O%`%vM~z z%giRa*tQ-jE}~L_C&M9I8(0M~-%|X@g{T--ivq4tgeraU-nE~f(vJlbH`+tSC?LJT zcd|9Gxj3>V;(fgibVXsO6n`N&Nlkp@Gj~Ler(Q>q;?sgR#K~S}{On=qzSrAe>Ezk+ zAw<8xI#d*^{I{1qReOxDMCD0E&hmdIizWTzPQjR{eo67Dy6ZGL(V+$}!yKUl$b_S) zDgLO`z4uuIP=a$a2J^R2YEQM_DrTbEW_XzD;qcPlXe9Cbc*V}EatX>$oZ-M~2_2Be zl9v}wDY$9@!}34B##(t5psy?OV89E{?ipF?EW37Z8+vwy8Rm!^MrJ&lcCEx!k-ew) zc9|ng5fP1_a9x-cDMnaq!{p91t((BaAll^&WQ(`27b#?rV$IPr_8~MRVi{iX6&GGm(Pf6m`S}Q)ErPNXKu)Xi;oD$~NW3K&o zt+!xQV$kvNx>N;;^i|N;M~c3qctFKLXtwOM$@Q_5S$08Pm`R`tG)x5(r2o~thLCLh z3vI+Yj&v&*L@pd)C|{oj1-*xQSXCQBn-n-#6reMqa6&I>Ii(?*P)8`IP)IAz4F|`T z-%F^KWCXmtIsUO(rs>+O-JM(Fv}Jk6sJkL_*&k!V$S- z1UXPGgQAI`yHL!uHRD_NXkXM0vVoP_i+<4OXT(3O=!)ZGfmQoz%Wa`-VIHE&3FeB2 z1}$(h>hgo5GJ|4i&8B(d2~MlEZ%Sz-P~*ZKmf;X7lIP$jg(^f9UfmsgxUCB%{VQ>t5{;fYbq|l-P|v1W$XP%& z#~(s-GoN3iH*klGp;5O-AD+-9F&6p3rIl_T9+ zqo3^3btXM7LyRf@2Ud#MPbCfM6cL6)jouAX_SheU`AC$c0!YL%eG4f*?F&U=!wu1$ z*c~qf1g!puCaG6_u4VyXI;NnKfjxA_8PyL8SP(cM*aZKEYpr+XepiB`)C3^Y0+8XI z8J(G&nF|yXKgS1>j$+K##zFc)zJP1xW4Goqrzp5`jfYO-an3zE=~+1T%t#0NsYEE8 zrkxvKCEYuO!A+ZCM@a6MG@PYwrui!UyC&Z!XDq(bV`CFC9{74e@KT_Z|mFAhb?XeAF8cbw$uXGb}p|B(>CB97Ep625ko9NGEy7%y+NIIplF~pYCBl}jXb_N z)yvO?=43vXWYqj~4u=o9qa6Q*K7bAdYPDt_d+DGy+4NfFG4~s4z?6jhqdQ*{Sy6(P z3t+d_6&t@PA7Rx_yPeWwHyuV(oU@MY|HP9(CxGD- zghV3Emcm>DB5J;p?YW1WEuk*Gi|?cp$A0%iXlDbRZ!>69ZBxU)>doH%BsgfcRB`yg z^@Gpg-L>CE!XVXcU(QS4fp^)@m3RPl`4Ba3{;xOB#{}ZTH16Vu+ejIg*<@Fs9E+6r zhWJC2oA{wL_Iqx2t>NwflCxDAX{{Ci9LWO2@RpgX4=+{QdjI1K`S*4Hf|rj9rrR|?VPp^cc{@`;SW#hc! z8r@cE`Zvf-BjCa*6IH@4tUi_AuULW#&x{K45|9x6-~@1*VizhM^PXh4R6bw?S$fQa z&8a#76~Np6tOvpk^S?um{N>83;0@P8t$%}}T3np5C!d7x)baaS)-1d#Oc_=kZxGi9 z1R2S6j(Ql9%6_KN%@Ans4Pqz#hAa5qR3^$}zRcEIuCf}jGZ<+#ZLWKSa1L0ekU%x9unwDu*f3ot2P&cP}+>MT?7XmT%P_Z64 z3G=Gd;me70oS)8;chyRJ|Ct}T5n6S3gVLa?Pc4(=s2fP($&Z;^D*;9%IhZrT7+D|uLE3|9xQU);NeKtPKpQ*bRP}Zge8Ej(nr!`7f4=8G3fZgogjps zb34yfnqXx|!`7ID@%nWn4#Zv_I{g3_q&GUdzM2_qP|8C z$02{)l|OIsj}!m>!P`;9jxc7y4JwX|7>`LEi#lc$ch@BXpn zC38D*Bzr7B5y!i%5z-GomB`i-*;Eee2}@GA30?E7oQpPAo}zU&XX{Carz zX`9dGL%YjZIzT6!eUTI-(bL6)z~ebWa^(b2d?=JjK63cO)Ac{_5m9i8lHmE`ynz2^ zRoCDIlzaIWUdQVk&Yrd!K-k|LJen@F(r?tZffv@8u=Z6^*vy`(oF%QmpO79N58|H( z`Y4tsCwx~7fBr;iJs5`w0#vA180Mp|eRpILdQ`-1NWWPfI|(D#6aCDvoX1EB7J6xE zTCr$u<+=*vSIKs?CwEPo*)!UrfT=w{9}_Bq{=`MBn+5^n>?T1^_({HN2;Py;yGN)x-Xq@R32i5fQyC5x{(H7rz(voAWQTC63qGQp~0mkK%;cn>0-CT z4-2R(+gR>nH-i?4coPYd_XD8^__ecLo_l``Ww{*Po!@8>{2vhaF_Yn#_k=G#**c3P$1vWfFs27+U#j1gKgg zh_;)AJ<-ak;to@FEnmKDJHz;i!fvN4wXpU*EnXPVbVN$O(pbv+`SlfRG;FA0jR@F< z98?q6*<&V%Iw$uVe!2d}s${I6si{ahkD%fR(6}QI2f27;lL)x~?pXKk6FV=a8)eNe zqt8Y2Vu`e;DIM7%`F)&SeF(7YI@~`~FJmxLtiBcSsQz+Gkt02UQck383s3G5A{#ai ziXaA{_+-sZ>^{wQ!uc>hcQpbew<_GTu(1wM(7YQ-<23UVszoUAG1H#5ZGBrcky&k} zUSVW%5ou1@9}VJCgsWU=kER@PSq@(+sE?Ty9yB=7dGx*0(20@9d1pt!9Y ztrM!G0GAxOljn3AIoi7vT+zd5HjfNeRL&cKvT#GXJrV_ zqrGhq>Seb0_jyK`&xeATqQu^D{#k7ME`t*i z!Y*!dmmAhj?XK(;qP;dnEMotBM>SMX!)M{_1F*-F@a>-hhsWt+QF_nMpOK7v$(PPO zBf_(dh3$UTeIwDv*=~%JB$Ew?^*hZwVV!9{m!R#hn-9T7ayayZ>ex$ z6g>STlnnOk{Sx~N-{mr$hnUx+UX7K&Glr9gw_Ug3MAd!5v4_UtB6#a>k`<5ChfM0r z+K?5yhhQ;?n+m7F;*4Lv5}c)}wwLeqhuiS)8{NtgK29|LcS{^J@p-Fp^n*09J2qK~ zna#mO3$HZ{W7MPHEk9O`yZ4CuJV8bsAFsC9eQ%1KHu~F|K3^T;-7h1$=~KAS`A320 zt0baO^54CkD}JA!#!;377wjY!&gA>WI^f4f{}ZzXKF^RjiE75^#{Jj{=D*qQ9s=h- z7j6X)k1Fv%a8L~-Fx-~|d7ijNMn>_I8ofJmGWaDj9Za*DOy{q=fy{|p91ekD?$-bm zNO>D9B-*Uv$M_nbF1J2(7UKSab4Z}b%eo`PDdznjHK2v1()mNq_j!oKDQtA@RTiPc z$D-C5AD0h7p{78OddNUPPA^Xa7UVd{n$Knp9K6r#rd05=va;e8iGF38i{$8i&YRF= zf|KgNuJD*vcXBROmHdP{BlGD!43r&qg z$b}To6Sj0|PrubcE7>8vR$WSFQ&u%e6Cm+#@bZBm=XHAxZQvjx(AwJZ%pJO{TnbL2+ud>@wwC=`bIt%mLQLcrW1@GqV?b@T z0#TtCwZ<+?BB{Tv7K?ZyXW3NZFGDK<11 zET&nfaHWW8!${RZ;phu6$OlpQGh`^_EwEdi{q`r0b8xHp5m2RAG-jQ-aMNrqjQDRR z2}Khp*b7GNUvmM8!q#1ww&zmJLE7U*SZzR=lAWh*zK-;Erwv*Eqxc(`XNE+@r1>WK z6!J~%{MX@Pxp1H$K=AkuKJ~*NxxO9ID*>nNbc^LqmPa);{JK(;A>dVIR1A*Yjt(UCe?z z2oZaCFvF+e6>Yz_^^*>N{qzR|%J;m|4v-^&eVbQEqE+cdLpNNYqWTnfU_RXLW4db* zG`azNNYwE8AI`5tsQ0pt_ICQ1R~VkG5j1{%Kxv#lptv|aw$MI@aB0dd=f#sL!42#5 z9nSS5vZyc2LP+$sPpH`n5oG_DE?2kVRrh5%xDP+eC7Hc-fwaWKE4%#5o1J%l&)RL3Sz23sSqbFz({! z!bLLdP!$8tEUU91WSUN>wCj`4`&9BiGfW_-HB#VJv>I(w5_&`(|O{(A-BB9IZR z?s6U^HU5b8Fd~13JSLR~lROR;f76Yqm=$x@0;Jm_A((ApnulcYkZV!oP7~#v8q~4@f0Li~+N`w)S*CiJ^!u*snLGKN?r@u=`)oGL*&1dAa$Pl1hAEk6gPS4O)EcJ7OK{9#$^ zr2l(waRd~6y`Q@s14O%=VLZfb47xnCpmrZe!AF&BIJu}q3qGczVkSq{7z6h*lIFjN zWb4<(uB?xxQs$(R|JQcn2!Uvpqq+4`1nNu8#-A}&V8*4mxg09dEB@2l+_;xPeKM%? zUJ%C#VjlEJU*SO4(O7bV5$gZnZXAAov*`M-R{hUP;JXkz>}P>FB-Y>mK3fnzHScEn z727n0W9X+sn!a-Ww0d9F(ASTwwn1Mj(k6eefd?V~YXt;`YD=RxMRRPPNP_F`7MQ1q z2H~$#W{@x0Ise-BPC(Hdd-)X{?FVU5%Id~t?%6EJUA^KD9=0xQq{Cf=rJg^ue?RE) zezV8`>pXhejDLSe+bMVRm-cT84pnp~>}11n-p#)y{2KWXp-D2E z91@cF-$9p+=f~8JZMH8ux66D3^&&J5DxR1aTjF}{N7tXu5QVs)Npa0}&MBa6ffKs;h$DNz%UwNrklC1zgArUWRLDY10R@Tyb&%~+g)hFasp6aIG2v|bB?BR-y^?g-h`s`_L zH%D!_POAKWpHUngh)jP-6`$+pQECf;M=|UE-(l3GUvw#sJl^c`cf6a-95HCvMEfAskR}10u28q{fuR`%q+3Kv zTBQV}yFnzRq#G1by8F9>`hGti9_Alr&pEr+UhCXDmww?u$vwr8+JE?e7hni$dvZW+ z%a+ruO}(yDRZg(^yfAIZyuOPco(fbTJQX3n#r;&4)t%g|?^EUqexYW`QB-;{4zN$& z|FVMw|7$?acB3|vy>`ik|KSx;x8b-0>)lZIlqyAQ=Uq~`?h(6$PJ2-AmPnlC6>Kp; ztml6Ge+9t-*=R93nP!b%eW)})`RjrC$(5nD1<48u5Q;mjuxnLa=Oy|Nzyr=yo@N9x zb&B?CL-->9z8e-Fyq$MZs3{U@Wh#Jta}YS{vuPtJ*6@)0RG5Q5C^ijo!&4N{BKej!1-MFn^ zBo~?!NWB-a@wf}}Uji;O;QdS*5w;dN{mOb_vM|}J$On8c13@+~jqOP{!2OD@Q&Vu*9oTZX(V(e*3&IZRtwm=l{cCyaeL> z(W0kP0MEIlB6g@o9fs+iM47{DiPZ$P&mV{wm8BvAei068?Mr0D`DG;jA|!r$;kL4{ z`M(`x4ahpZlZu9qo)a?-R8LzD=g4wx+ctb4@TWqVUAoWv;*s0F_MvJuHNF<=%ihszqY4k-? zI3&lk>om`1H)_but-blvQ6BaGTZzD{3yfj?uzOA;lNObik2%#*X8LIC$cTs%wvH?S zDu;Xrg(nUNnIL;!=hR_=_CcJOOPxMQkb8IZHjXgmKXgEWV%_4YEdY_sut+I!5xXUx zdg|hNJ>f?f!nj!LCv!}|qSCj7-j9E8D2g+xsm4Jl;zW`i%3TjyzqC*1_imA7O_lnC zsleYbJl^zPYJ2hpK{B{lpwik)RAb37`!LbGs~{R zcS2fmy3?hPTQ!LuuP4yW9*8_|{STy%3Ew;IX$t21qPG_1^7_mkzei4?_l{a5XGQ-Ze{J~?0Bx$dfUW~{Qz7Z<>$WhK9T-E8ch599k=d0KGrv&HupN@{P( z){XoQfzVF`loew$K21S+r?d8COo2_r#n%1lBEpdvGta=0))jBdJsZ980-Gm zNc=FE+lWho<>Be}OV$Rl$C9rMrVR+2LS3}-bozfP-CfOaY1s(<=z3s^c~q?b!`--& zK)5@%-u|uq0=36%W8RM*gTFnbPYP5kp{w`ke!mfRh4@&h7nxEeUQmFp=PoJ`)*l!^ z@Gl({Cygv#WlcNSNRSW|l)+@1T>PR%0natj0-pP~Bz^GaH+>pQgBmkV;Q2VjvuKtQ ztsm||aEa&kGl6c_Qp>wJ2|NIUN0xlp5`OA-ve|zU>OAa-WGs!?)_)^TCj}7@(9dAU zx|3@_y==yKhKN6|kD15}dRl9#_5gqBi}%@h8n@>NOI2wXd3fxKKNTrDo4Lv8Bfd#y zO7wp-iWy$`q36ayZj@EUG$F~`9M++zdvT2fRD7|NdfSv~PM52kAK_G_&naqWmfREW zCmXml5tvw|b$ETU7Wvzw)+kU9ox&u$YsXWp_`q63*i6Rskv#Y9F}%JTQ5!CdL*zwS zb-tv2_>>e}F9R=)r{%$Y&yn{Uj}%SZANr^tQ@+-k^IyA@LM^fR(W~MW2cx8#F~_X$ z*Cnd^mT|9R+MFYWcODqm$J7NB_lsTkxH_8mD>u9ctfFjZNjcw6AN5jwAc<1`+j-?6 zoG#oq`MtbVaW~3x3>Tn7Lfy+^ry}e!w@;SvYr<^ap&KEuQcJn09V_N;MiGdLNleHb zOGtpTwfYanZJ}r+TSZR(znQ*%HL8$?$VstznpWM(ty5&L$OhO#)bA2BRY4PAK`*18 zrEI?|tKT0Bvy?)tP?ML2t3+TlnYL37`s50yQLp|B7bPUMSi!w+&bu~wWrl2h`tfW} z=odS$OYSi0Jj`Gq?5F}yOW3*pP^li%N->tSQZq_3pS93KZ>kuAzjk10!N;1MTt^{9 zDt>i*(MS|5O)RjtADK2&eed;eAtoblnI5zYs=H(9H85A32UqSQEdYz~c7xr+HMp3IhcIJM&@1V$fP#Y1L9WvIfJ9 z*>GJq`;94`KsOIq`GWqyQ+ykQz`@}t2=n==7g0N90a;^mqPcSs_J(7~oRS3|P!TdB zr*eevU5hWtXrv^$lEvsL@T0kaGd+m0#i+!Oj}&~RJ> zE!$#8Rog*&k@z*mL~H_@YH&xLMfwBZZdaeeq7s{`E5bRSx3u#*mxhK}m}z#^t;C(N zBmXMQ3uLFd9+48=Bp;}AIIylO=@*0MJ>a|(>(VB$MZMlB-QCSW((SN6W1RS#Qpih6z+>*nSu}k&g6-fJ1OJBr|`XU?NH)=A~ExNa(Ebx+ZioQeEdWkKhp2WxW*F&EJVdIu}flcE!T>ArR*y zVN%|^ZgC#jITY{Q;@nxj6mSNVY^WX3;k-fT&sVkg1t5!34aUn}X69{Jdq#5UJCy&E z4lcmh0y$4K0Dwce!8MoFi}CX=90E{G8kH({rtHa_+JKgO#u1?vG*0v{1ZjxRiwqkv zIexhR+tdblAZ1jJ(=eYjBaM1O2Wm{QaslHta2+CS)^;XW2Jrz7??vg_m;!Ga_YWtG z9_XrNi7+%&U$Q)!XtZ+l+&%lx<*YU!)fs<}a+?<-*N# z<62EcG=evz_jabONd-$v=3B&zxETN7{eEIwM+(#$xWNNvTTHnW!d0Y;QfGOQIf-K) zsl8HrK!DC6xB^&IYyzI3gz_DIx!Nz_UZFZ?q-V|@1eZ>~Q_3R@qx~dwCxzQuUXr&^ zPH%n0cJwwpeT^uHP7OC7QqjSRlLn|25AcR~nBwgKvD6X0|2I0JgTrxV=*7)j`ooQO zKDi(n`I~|J9m1u6_P{5QLjR|n&txq|8Y)%dwwf^+)+$9d56jBCZDaUf@bp-!5)vpy z0KHS?G*yHKR9kb_AC(M8!I))spW11I9K--D@og{iziumsiV>x?%bY-EF2*V@PVX63 zssu&i%FJx>3)6}x1_3gs(Y)A?z6C{5&;HVTK+3w8Ak5+KN)~YO4+D)j)>-CFc)51O*7EA(iDpgb7Y>p zxkr=#H$VGCL3M$Y-;yI;Ekws{S^YWV6x>WX18Hhxh09W#ofL=kQqYEc056FyhSVD( zn7LXERbVij*hsxQO`(@XRo$;*`0oV8+J1rm#VPu)08>>aw<2yL%EVXvu#lu;@~KLw zP6)?1EnXT1U%ZFr)&uD{G_?0X(D4VLu>?K{Hd-xwq_nO*9h%3kjmU7Iw7h(x&M*SRck; zh>&Td{fNR8|l-2`v>>`hry^w91? zcF;|p3HR7*a{P&{4a8y>q5-F zu}Cm11toN^Z~w=_>K_yyfo+K6^kVMPT|V>ATYxoo>!Sdei%1w*2CO-fmPu zf4VLf^KGwmeDAknQmtXX%fdO~F1I?c6eZ_j4S}wq13h=nfhA=W5$t%EQKI`sZ@|+y z41W!Za}P&m-?2&ZYQAT4$?l~RI#P>}ZU@@dK_)};B~4i|-xdDOOXk|u`B=3X1X3&enJitCRq>8#r1GRdkQ!&GVl@usoJz@o9pmYx2 z*TA*T4k12UE1E7DiSumIJA@}>^wVHhLUGJc!e4I|!7HM>2k9tIuZ6*zlgYFkl07Hx zqHihn3zEy1@P$ewVfOz6ygOduF^n_^?_R9nBGTo zB;Ylt#_q-MQ+!EL-+@aF2?!w38Ii%DRok);A+Pzxi@oB+3Lp4sAMol|8&MUrs~1|( z>8XU}`omB|0|F)FR17M2Nb>^^U=tHRQd)>k3?h6xdit2|WV!3qa%#8C;KAzK*;Q4! z+fwOs0Yyzx-*0z%NoqNM^0?_S{ur&lxwF^u4z7g4riO=+y}j3Dd;ue39GH;a4aJMD zp`d!m8NQnM)vTxNW9mQj)3bny1YP?iG-j8Y!|s{)nzW#yJhFDk{vgoDt!A{zD%Yhn zA-iydoU}~>Q^EEL(Mwn1cb>K4sl4If{X}5JW##Qez{{LR38DCeo`N{n1j3Cn z%PP-0k1^;uo()G%1D6y>uHqt1_vwJ5z9q-<>=VrgFXRE2=&4i`98DNDy=Hv^y_dyJt!=~z=W)>rPuuGAY z!vYQ+|5z#}120LZb898oNrpmJ@Knyqg{j<|YnhO9~6P^Tb!*)??yJVE1n z+-X{M&@j_DEH7gY18T#MsME-JQik#c2H>gIqS!W^UUKnOO&ybqR8|7@m&MM&_4M|( zB!69TYOZz(V5J1~czbQD2j(bo_?Yg)*B|P}9!qV9!5^~z@>3TSzajA!qezs_LZZz9 zy4?{0qt@c1GrkYwC%m>R9dJdH^W$;_R?pY(>NYGrW^k@Zg#R3kk)e8Yp6V0}KMRfD zzxUflr%8O~rc3n!f;6FZ_Tl{ABz$Zn7JN4p2v<&(z^^xWh_LUGx+BZ}xOwe+rMV}T z+nHqbXPR+`b57r?f_5!=Z;uhAH^BqniaNP$P2-%#1p7)Ea;Peds9ly?XGK089EFUI zq=N5;113=gg^JTe5Zz7zNs?iwLidQxpC>U7y*^~P_!-)mr&#cC(X|A}MZ3Um01H|&f4@E9x|Ddyyr9xbfHRQICDWLHXI90>I|H`32 zo>EE*t-t%W3+5Q|q9$qYjAX(O`%-)Wq+hGnF!W+Oymj zH#zHk+v_S>cPe(pq4F%!mQk2*gGhQ>)AO}Nd z3#TU@Nbn_MKs1otB)BCU2r^F)zrRU<-iMQpwQV26C5um6zmYYL424(u_{thCIlE=}a4DT+Ymc9mh^mJWs%TmRN!7$?;eux-{?MJK&?12P zfcCL_3@^b4rqSjw(wD6_cKW;SzJ1CeRL;s4NU6VF;xr0O02~__?fJlk%))*8JltfN zy)@V&UKNZp_E+i)4sM_Ob%@)8kd5Y^R%_v9Q1nF@#r<(4;ES_Y}A_>gyPzn9NGtk;?eW~0FMGHQRflgZ^v zpsZ;On4Y8mGbHV0#%WY|pSo+WHD^OlnW**7%K5u6qT&H^L@QH5mNHNcmGQy#LpScm5ghc7c@+29lUS|9 zVJSDAdA>WsPTp7i5r=BP!5|(|^v05ozK5`Z&K>S|jo8jNes52>o=oU{qx}67KFDvv zOmF|Re`c#OY2d|Yz%DFYdM$KbYeQDjn~ZvU+$Iw7jAz+#v}n6>bi!MGzDD?sL@K~~ z>5vzS?78O>L-XacT@@q;7^92BQ|$uZ6>6Q`_1rwrIser|oAKq)M^-iT;+{rX@p zH!I5+j-$~QGAt9mHoxKUwtID|x2ZdGaQATq&znkKYycL|A#KH7@7-Lx)zn_3#d#b` zWaL>B4mR(Yz9229>wN*!Uvl)w+E5q3odG~TA4a#VAzD4pCkVV#3iV2;VOg*xY@(zk zZDV%h%BQVVS2+KOpG-M$V3zoat)g!*R%^U;4|Ica-mU&4I%DU7JNHm`y+O*j#&RZl zvCZ`q&MfYTxVjGbO*nWI1^}ir>l=J1lcMwmPq)%oJ(=U^wW@uthk9So&){Y+E0pus zyS5GIw6EGdrNm4_(7X0HfOSwnOpel6#i}}m@90u}w|qQKnqd2!!}6Hes#WU1PU6f5 zPcty9Gl}+`UsLXa7n48U%lB=DM$NJD@hlNyymMC+3-jN(W7KR8esI6<=9hE zxW9PMXYQeL-a@CQ=Ni7HToW_v6PX{aH_-#1G9o}M!HvD$&cyF1RAeDhh9W$L!QsL_ z>`!q=A82ac|4eA3LO=!uGKNtm^MVP%1m%44hSw_^l8O0|!#tAh(d<3t};k`IzyC87Eey~p9X zKQ{dQS?R;#rE8Fga!g`%rqmYN#x?x#RSbn~fJ~GP*4eiu8*X>L7!EQY1Kvxn@4_EL zm&|4j#*P$Y@AZbp!s=yF@;b0f*WY%j7JcF!CKXOu;TLh0_&`MaLmhYDVZsW&_+a?s zcIxaxrb?iiDu{`<5=os?q)s$Ij`VnMa2<Ng*sG$H&h|Cg<9Af-@4Xx^)TYT zB-92MXpFxdF;haoqq21S@VY*a2QWXrdD+YtYPzW`)A~$xDbLB^cUpyHSzQI5RF7m# z)EDri1S!!uK~kMK?oEeD5Xu_xbOzoNt50(0UMg4jc_A)%1}Zd>Ah~O(=`o|-FYyKy zYQqeCOvw`xl1T8VT-y_xAGHOw6boE-%i6BaqnHsQB#ME-OEGu@rv{W!kqy@`TMVdW z_;}^ORk*ua?=J~;;8iHG5X(6%nyi|6OPmLV?A2}QT}E=DvjcC$?>K%*CkJNh|M`sj z($d2pP4xGmBPZV^!CT~0s+)PA0~8dF8!LP#cX*3xv<kqThMNaA=^}UQ0W+t*T2)|>eP>QQ9nED~p0xrR}et`*Hg8Q&s^nd*W45U~S3lXdp_mJY^Y zD2JE2RxXe5utpf-L3Q_pv3Yi>s|e{E=wyW~dw$qKzN?SMr~7(Qy6&o9(`Vdem-aJY zBX@dR@Q&?Ud41jjmGt;~Hfg3P8EH2Obs(``6uzXY26)7L;PZr4z& zYJkWYEz0)dJ~badGuCX3&!rGgOa724lWjrqZmQCp_sd`^X7lHoXmwZ5+Sb8>5`KjM zoBat@j<;HmJTsOnSn@{bgZ1Q5R>@LBtm%@jMo8wutI1PoEJI?B0OmQAK;PC32!1x= z2sN|t%I0lOTAe9 zTtrwbSj`@bhs7K7#+^}X|DLzfn5}S1?brU=$R`}lL)^TqoOKTn^O1zb>kw=>16FSBKYY?mpB4EV=nsw;NPzf_MS_Fab zzsEk&@aqG)0giEv;x&6Hz9Zx@+bo}FHli7QW80RY-&4v{2yc zN6RA;N|3l5b{^T;Ym0uAKuTuWneLkVk=kJd$vI$M)&681J$wX50nJG0SG|BB zg%1mwLM}_tfR-CHHMa>f2B8DAfhSMcQX;_n$T97@md{BEWT@WOtvAWW`b^;wX-24O ziZlx>UwF*cyz(?P76^S3(9QXkU;6+v2a`JTT8z+$ymwPV5){8r9E-d6_fCmeGT8`m zSWqPiu7&*>9u#BCJ?W5GAGGt_ZC>UCpWJGs5dibj2qAF>02G!S9Qaw&ex(ZnUL)K`%Zp65dUm(Z61Zn!3kR}xAk-@^igKnqEWk7IC zj5&cC8`!tq2DnVU8p_@2%bm!lBi8zcmG5;ZgycUJg`7a)zF&jZ$u7gZArU^qcNgfv zE!H?y9p?0m3k-C17b`Yh>0r0RStm_`ZXJ3_A!b`#Fh|Jy--t;lmlERwSG?3*w<8Z} z5$W-1tVo0i^R97YV$yC{jxW6f;I1Waj3zzlOWCCvSfgU7n0&;DOV<2!=rg5`m_O=Q cOxq1cXS22O7yeI9D8P@hf`)vVti|*H1JUSFp8x;= literal 0 HcmV?d00001 diff --git a/service/src/main/resources/static/images/dwdocs_logo.png b/service/src/main/resources/static/images/dwdocs_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..cbb67048feee9d75f50df8de9eca70fa9b5a4457 GIT binary patch literal 28442 zcmZ^~19WA}wm%$ntd2Xjopj8OZCg9GZKq?~Nyi=A-myEjZ6{yOx%a($zB}H3kFocv zHRt@*L}Bf!H7i_RRty0a8x{lv1VKVvSP=vSTkA5!p7{*~#tu?)Yv8^G1-p$(XQyK(>$BpZ= zXbo^OAat{~vT@{c<0bxEg6p&V7ny;W@NW?(OI~7i8F@k>TL%CkEB$wRMq)l#LPA0w z2V)a1MPbo@!$0qMiOrmx?6?>hTwPu1U0LXD9ZVURI5{~P7?~NEndv?y=p5Z`oDAIP zY#d4cRmuOa1p~O5|36@VasHn$W267|%Ffxr>TjQnjTit{0BeAaljA3j>Hk#s8QcE_ z{{M`0GqC%YvcH4)FW-3nvYbm1;Am^*{8tE6Y|Nebn0fwo=)cMTiO9cDLbld+4gg2T zPZ%HTzajsm{kOdOf7I}?|BsG;68{?@=V1Qnsli`%@iF~d!9QvLEwAx^DflPxAAr9? z&n0i}2Cz~WHvbIh-(GOAa`G_zpPc_j3fWrOIw;v07y~F|FNdGPUw;qlE=wWC3 zr<#8-{sA;L;&OC0v;Y`6{iC@)8}0ATFKA%O@DDN%!~ZGG^Vt=+WNeMiP27bIoB({x zj7)5Fj2v`~-<6q}xEMLO7}KL;%z>?iVnCICLzPytF85D)~gZlI@GFn(wQ^N(!@ ze=>Bs!Ij}123W!)-14g;2!UYuH2UqVElHfR|1Y41AA+Q&vlmad6ONle-i*8O3?dUn;rvC4e;@$tiPN0}lzgB$K8 z$EJPvj^7Yy40x5jzBd4%+c|U1$tp)TcPF2%H{F-V_O^pO56ACJhZ++**E-R-axEXzL>olj0qfr2DZI6IeDnGu|nb%G9io|@McMtc`^jfGSne=*663+ZsBR`(L`H!I;!QdCFvK+*RY7F_EMlG`Y*u8rPlJGKNaX4|XIYN}$D3_rr zh}kvzuEg;Vr}nu`b^?hNfYIcq39Uw2<2iSwy;SBkHMin28Y7h)AMA)3g(n}qA5#Z# zX6g9QG_5&p4236r2qIF6^qhIuSc~yn7QBp=!($1us*Lx%w z-(CfnuYRnY%1dKV!}jkg16@Y<#uJmrx}&5q??Cz}Fnyb=MW8;||N^py(>)r$lPqeSE7UuAQGO&rJ;tc-s#zI6xrK zg+a6aSI?$_TxSd(cVlx^Rh1qk8uita7lO7kjat>JRM+t#!%~?qkRa3wy}#h(e|iu; zR4X_Xn@pM;mf;e^*X90(k;>no1i$nU(^!35OPpE;6k}x7c0d#>%am10LzwG2YuiZ= zY07SC-mko`!G;WLu;d}YYxR|37UpLVYJq8 z$*_!m8|qFR83)Y8bA86~>D-*4c&O`2H}TzWX>c;ZA&U|9 z3K07$V7uFD!d-VRR5>6kT#k1O`eB1-pqJS*>c3gMS(WGBJ%>1q@l{k24e?6lm;uj%ANyWH9#s?6@`5pn%QYDSQ zn2Dh=ql-iWlK9Akg|qeEe&^BMnusr2e9x(VJ`}TXku~D z<2DIOzMT&kWa1|nwg2oxE_^y8*aZ&JBN+$Bl0CD!io0+0$kmO`Xbyp{8D}2M#mwc$ ze|&N>q;xq!-XLgpJD9US1K>`aoMJN-mU zRb~H;3*SvPiSBttZsg`mxdY74TD)Gm5+KIwGi(a{Sh4?)`_N*66i}UdTZ9sX>bi33 zfCE~zswfw?mXOHN;gTGmXU}7K zl2xK~xC;FO`(5?bSm>u0hta#*`y~USiYRnO#i=3d&RP5YzB|BZ%71h%Rtj!f4Lu@Uw z%ani1vRrq6a@IkdDrrkvRaOb+&Td9vp0j)Tn?YquUyoeJ4v-~Tg=GIeH;R8xu!q_nzDuTK36TU{gmRuPNw#L@f#^(MIqrg@ole!ojY&#K3?tI#6<#lJR(nKz!gxQms6TmbKO>l9&U5B zll^(B&CSgI2)y%?Z@g;n)49i8a8#Jv&Oq{ieef-LBKSRVR_LcJxvKtNYczydL z+??P9g%WzLE7gL>E>K^=PB6^u>AvEaOVe~R%Fy+S#roA%d$jw3G=*gsQxk(6beKCK z!XY`9t3eDBSa+t_U0$R;47-G+(_mY}$n^yYzhMW&+SP%J#oNV+#b1o#kiN0t#~XfQ zue&cW$QHb702V5>SzfoT-039OAZG02JXUOrIDhjr159vdil6#?@Ese758dONOjUIL2 znyt>AxFr|t)hmm0P}aDX!T1rs+db!p46cV+N^AU<>Eawed+UGv@B7P zKL6rC1tzsn={0bEEG#?!QLdDUIX%y5*S$-wAiDc|^BRzfKbNdv6~I(S`R$kf4jch} zAd$-#_--Br%~s=FZN9FuFB!d>>OiezOb$9foSLrENe%`4l%y0E(0Hw2IIJuB!Fr08 z5LdEdh3r6io-&0Xn{m|^V{mvht|yE)LM`_~i3gO#4T?-k?KP35h|3>*+1d}Yzh>L= zEs3(WaDnW^S)wFaURE7b5S&wZ%Sm^vQfKO71f$`#&OoY?Oaa|m6PhG>`jz?OctY@1 zN4!Fv(|-r-kGBPS(yEo3!tzS^Xf__!;wGhHt^P7S4|)Tv)>hit;3MC;ylJlS8si$uLSy5?7>oK92T^)q~l6!)Y!^EmdfM)asq^^zpU7M;O~)N??LL9sI$Q zH~1^)T~EBgXwqJ{L#bV!*&EW?V;y{Iah$4luilJmxFFa@dL_UL!4ek#0br;XRzr|uxj$jBtK z1hz-wR4iOIYj@JGHEjk@>@JYZEjf&%oVmh+^=#r^kJn?VgHzDsV`dNwgU*f51j|pw z^6V4a=@7`w_{`2P*7JNc=S|??PfCp#aQyzmKxZ zls-175g4W zhoE{Z)b{#)9WLO8cfm;(fu)WJT8~JA6RG*Waq(h#*3EaEw-un#jCSnd++cMYR~2tK z0yC(>nSF$RuGb_ijrbaNC}y0wv$=h@jggIO?MoXWqo=*I%gqzwv>0nEnNa+*&Gj1Z zaMD#^wOR)Tx8PO@X#XRke;ok!tmj5QX4l%+G7M=ota%y{QF*ni-XR&;WSVEC2$$9;(EY6pUo7LPTad7th zJLPIZCcgW$S>TYuP*jb*&6^1e_{!3Q)AcdpnZ+Djo;n)3dxdA+L4Pd`CX$W&nYUQk zF21P%e}z6BReJ8dnJ8kOmQqI}l^UguthhsJz|OOp?9IzO_CyUAW#N&*Ixc)qi*wtF zcJEIkzK$ z?437AGMOCqSlnMDMQADspIAV-@Y58u!$y@z9e7Pm`C<*zWe>ePCj@QPWwWfDzZ|ba z`&%5p%kePl9tzV2hfNoVJy=>Am5W>XwdRLZq==_o2EP+Un@T$R+>`*(=k$E)olZ5 zLP45J))eJhwzYyH86bee;aJ|(9h>aUO;|8zC zrFWA}cN&U4PcbicP9JNuy0ouRGECcaPU*Y@(p0yPyu5LFi!ESiKxj1GLD2-^_6RT4 z!q;C}af?Vn3$wSTDs#I#YzX`wNwFpyrL3ovF>^6umn(flCO&5PdJP6iJMzXzYv5{; z*SKQGpbh^{tTT&oLACO3lOMSg0=x=+1b?4J*U2gj`u*dVWDe=AVS!8T&!`);O7&!p z7a4*RRtHq?BznyOHhPui5Od4uhnG2DOvqQLEw0q;{5@2X)T*f%lCcoCekW2_m|aX5 z(RQA1j8E3xdrd*=>YmasdOa-qaU1c~LBs8CFO#WVwMOxBJHL5tic*i?^S-A@bDD2F zrcfsZTW>y~Nt)xsoPJ%ya0MA-QQNOon-Gc@6qv ztqdgD{}$qU$fc~@T_LIpH;4EMUZkFmPh5j4VoA5on-IsqFHKVyB+Sr$*=v*U!)`J6 z>&Oa(7^A!#RX&~btap zg{9@>7c+|vRvBo&ci)g^KaDM8SP~>DK;%7QN^&ySmQk@tkJ+^MAUu$|^DG&3J=-UD z$l!=S@eXA)0IVJ-5_NX8r?*A1{%1Qf*quS0h0l(X7xNEA%YLoZrBDH^7~w~;oMWxU zuJ_CRaU}{Lb?%1k@y|m0sOovn^P4{&rRGhpJ*r}HRDO}q?rg&~QOulh5PPH-uY5G; z)ckI7+mwdzw8vZRRp1}aZT2Gfo&qiPg5Uv_Q;iV5ou5)ZUHJ|5lv^DyXq7bz)9G4h zm8DB}G@0TFUV_edyWut5EwyBOF_hYLsanrpz(W$NGjzs9{3a4u#zgZ`Mw>BzTjQvB_CYS;G5J4?@9 zum7HatEC!b5av;6?f&GPN}}tY`g+29J4IVv=(^aA={)q_4OIG4x^{r;m6B}>1&awRUuB}=Wa^=D5(VnLS93II~$TvMSx=~xm zH@laqJfR#BMhS|@%n~eS$8&1!ciG#}w=Tiv7i@&h+cvs={~u$6Hyt$=jfE;^JXPH)qBMY7=j-{b|LU-= z+*oy*5Bg9w9oz6@$iv-~?!684DtU~4ABTY$tVL?3{%zP1S4uWY&mUL$gs3g@#O80= z62dNdBD}z~HgG|(GA#>9QH>KnK+)eZvA8MkMD6!3* z{lV?oATEeCI3l=}q^EVq*-_+pt1Q^a|I2FLIx$Z!>KOdAcsu`=&)+&r>U1nC(&JNjkH~U z;V=Ov7ZQN$swUE1<0E*t$=$OoZAuZ3pD#3}F0Be~c!0iVWvQnGncA}--R4PjkL(wq zV)kmRS$pAKUG{pU-dEBkCA?$YTl(%k{GnxH`YHdCLAP?RnsuQ6!#f$c87C764kc(4 zQAV;nbA25UFYtpbeV$!}q4&CAbh$XO%)gdIwW#~YTjGqpQDS_ZtXe=Qf;^hSitucF zT$@nT(IfIW@zQU4j*;yGiEW#+Tc`2rUt)-gF8pj_3j8Z{p>$%k(t-n_OR~1G!DwtC zE!Z2qbOh2_Zu|=!mGUU)bTxkVElWJF=#P199EQ_9if05F;pN#n_~CZfi5_b`O#%xc zZ^X#ikMI-h>L0fo3Qg_8zmHhx!%Ff;@ogif5T}-;3?=I~R&q@c_i-LwkGW6rpI%51r8Rn{F&;o=p965nU z-)#}?xYlwzbD*&7HXAcC(7WOuR-FOcYUiS*#?qHy##~6_dj||fKk(Af#7P03=`n#> z%p^e7#@-f}Aa^y)mCxz$qRKncW2F@(z>(bOax3kcGV4}W2m0s(sYIxYrVvY;_kpf4 zoL~r41XC#+K%a7?A+sop`z$_=_xaFaHgy!IPjQ&lQLx@07LpPpmiB3(W%yH_?N}@r9>AtY{8-n$gyggxT%z(Q^1_pB{ zgM<@{DV|@A5}Ta^ao>57_Kstz*}wvb$hH~}Ls%5clTPjD>a;qeTe{$koP><%PA?#1 z!?IYo7DT}31M%tHNx;K`ST;qqvG6x@0q_i$`q9O3OD21Eopm}W8{jl&U0e+@Q|=UY z2bn4R4B55uJzM6SJ)_@qnsZyg)mjbGV&zV>U8)wc|IQEauSkGuK-Q z{5kT->vgGSS_HyL>M>W59!#lo5UjHIg_PUgx>Sn7v$#}9JU+(pb=vGO3X9Luf31ls z%c}i5=e_{_ZZY+s2Yu!J^K}4Emjl*;d{#$WkqQzcNTH?<-bqq=PDx@^mpoO58g(ye>Y#h-ABolEXw{Q;R90p-(Ue+>1nWu7gp;^l zrzI1b_T9ePQv|>MxQ!QuD!S8c%YF;B{_x9phF15j8B*3vtRU`0z;XP}~L z33&;15$<_X#kOps|6Kt0>I215mA}bL8j;*Qs|C56O4G8v4zAQQ9IqJd509%gP02vK z;_afL^n=!o;$V!qAL^aFoLS)m92hg~kml+KFtX19O&A-{~O z*A5=mNb4{dP3CulVpS^O?b7Z@0zzGDlC#tvF0I0{Ho}}5xiGEkY{AdT9o>avi$RGn z=f%zej#sLAjl@DpdxhI-nr&i-TY*klZD_iobzJoM1}jk>#X6tiv2?8_s|`=aocp>K zpNY(i8{^-7qzP=v`R&1IjNnw)>FstWetC5kRo~tZ+tJA%b)DGv1RD2>cc^JddX0bO z8H1V1=>*gk)gQM?4?ZdxXc(I(^5-V%dd1CDJJfE|3`eTB{23rO99Qhut4`ze$6#Z# zifhZjq>rfGAyQ=-k-M@#(4hWZ-6FBnVp~1)XBowwB9T=)95%(w@Ko}fy;FbIrB_kQ z8Blz!TyaYP>U`uRrO~pYhU-VnAzH%M zp5kwE6vBJEObiScM9bX{GjUwLh;orehH#1LrR;2gXy4fuK7sIOtbmBKgro!l-<}7CX_6BvZ1%TSH0&>MAm4%3GI%#caEG2*={3zjGlkj12_pg-= zI5fQ~?jmdgRaJ~e$PCCM175w%dbIJ4_O^`$>+qExxGGE!a;RD|_VB%sJBh6#^#-u< zV=^laHzzz*I+aCm57-d8K@?Gd3RU^H>f0REaAz1eC1{8+H&zBc81Lt_ndWEoq9T{b ze2h9nl2&xD!Add!DS^EV?PKqiDfxj3P#EmO_Ac-nob{1w+*y#0Vk_%wK)N{^;Rb8x zPHmM}06M)F7TBvbIEDV}A>xBUU&ve9ope4?joJc92+#l*t&CVM>QMY_40 zXk)Z=5D+9x;I%!G1=)mWR-XNOMX3=!WcPA?Z6(QvtSTt~cPivD2ll3eQV<%b7i|d9 zDPn{8f&uc#Gcw|bt~e|;e&9Yy(BRXZr3HR;GN z5{s#i@JRffMt)EWihQAQRBC!k$?ZUys>8CGID5Ts+}ER@+OK7D=3us#Yc_kAoo`8o zXx*f+DW}*y$%L(4!xt}tmI`*TfJ7N9T&TgZ(PqRZDUbo371+Xnfx|;ou)J{2BykrB4v_8c?h-{a_U&9 zc|R9ZYE?lTYEX-(u5%|={u^XPEa$Mb`RFVn`(?-VMCc7ud4*Re&^-giuI74b)!zOC745#Lhv~z#fJ_P6- zb^?><;BSs-Rx`(j^}@)rOoyVJw#LqZhNpeBGZ%gRTto2qXStOomu9cyY<5Ve%zsDE zJJ!{>8o_d+yK-`AXgVa;&mB`*2U@{+U9jhw(tnHvX3{>Uuts@(JJuRs-a#_gv{lOR z`xA70MU~ruD5jTbKMO_pK=4-{DNI2iEC=#G7EiG8{mgz#zTaby=6sZB zFbY_ORhQ^J+4aqyDRB)9VX-$1C$AZ$)j&-kO4cY5%nZ*~GbqpP}<{8!>{E zITX+hVyGoX)U9q!ff?D(M9bxFo@UzUH!v{3%(OBsBYlm9i(|WjY#7%zWxEo?8>Qe! z&PyE$=p2b)!`rtv0osGf;^!YA(uyOVg>?iV0vcuwpz;60)x5T!Ka$sZVo%; zaeC<_v+ALfSm@Klk3i^(l<_{8-tD1H#!h}lvbt;lRM>g$35i5wG*+ycE7;> zP9g$Rd&g1bD0a4#_s^QpEXW8RZW~tG)lCY*GnU{`GlM*jhW5R`!$H3ZdOv?3peOF(2(F!!4T`h5rHF_^L0JlWXhWd;q)4zN=cVlVqN8x$ zH72!8b+w7m&xVM`YEbJ52C&wb2-W&RMUy%QoZo+{L-Hutni{BA)caU}SJ)aOq=;fc z#C)f=C>5 zTt?GEEC#XFk9k333$!T{p-J+jM{!3$&O!|hcjbLZe-HQqil>ArsF@qg_o{6ub%{6% zb2~{fIK=ai!(tkpb)pB~x;N=amlttugTgMBL0anXskjlJK3m*ax0{TJ7;TF!4Q!f{1OhieaQLkdHTD z2TMuOc#Q*GEZvYuwH_(iwDA`$R*ePPDkUIliM z3`}N>8B9I%3pZ_Pp%=L>@E_6am?pmn%^Dk)f(niv`DR z3z!T;vqJ)t)^Og*ZRPJ3>OWQ?!QbDGj3*J8r~q5>f+b3JZo|b`7X3+fp59rtDSEJaf+ppBN@zg+OwIcFUk*f^Bfv@}ea zi?Ysct5*O5ISg6+zV9jBp7V%{RG9-m4+ParNTh`bzoy#2N`vV8yQLyJDhWERk|YOa zgg3L&&v@D;f%8`Ch%61wTXIU*qPWC#Uo!?_l`K1@ak`xT*~a5`$uvmLMiQaG6a-J{ z>`#0*d_zyFL%pkyYoD>Y6n@66t7S3ox>&gs z*M_-h>KXZBbs|Vr5y=H7U3}<)Jz*R;)~Jkw&zwoa(cFSv^A`f!X{KEVqT!Y9ldm<1 z4G9TR-wkti>X>!$NM4L5!x<0rX-A$kM*@G4aL$FHza(!|Q@`}rqL1V!IyPUsH>d2-RHk}&)s>oJ@4^QvH_9lxO# z-X3?JbY5VcckUFg0&p)+DL5xcc&ddwwNXC9Hxh!H#gQPcT@jzV*PfU;u6 zBhqp8)YNjjw9J1nJ3$&KMX?Ne2MT@|8^J(g`|>~xL0&2wp=28JagDgBhu)? zg!dh?REn2;;UTsd0hCu~AL7{o|Dd>$L(|*)HG+;&eB+Sltej~dFiVyi+85HXBPjS} z(F^sNTf&jh4h)J|iTzr;+0FoNYz?H+$ode^5Zo0WBhNqBGiCO7vH2lkbfu;ssfE8= zP!2yMiKGvpd2|^5JkW$g*XtSgu=Fe<*ck?xNM>}XRck|2#Frs31q`-p8)LhYB_%cY zO}s!^Xfkzc<-!63(3n|R#K**K7UIw*(S=o+bR-{*8MUTiA9+}8#6L5Y-clP~a@Qx6 z3V_o7`SXVZ=JrDzn;jg=C?+(!2#(16R6!eB$Y!6W-lCd;VM_Da3t`|WRNOkWNa91j zlT@vgZ$pJr2Zx;o$ghbr?~BV~n|e!9mGEyBgeDm^BUwe~%zJt*1JTV@y zo&W3b^!|W>Mnw`hsQ??ppX}I#J!QjOk8tP^q82_=+oHPEX4QhS-XW&&2bqble$>SU z9i>?x$eVB1V7RqBgf-5WquAdwbScaL_Pbh!A2Bhdg$p7%6!)iE!qFdd(#@a1hQW(< zhnRb%t1vq!-KyU>t2u-ezTj!wAF0rRP{L?01ep$m(clV)IB$~p!>Ktd-F^(b0?Q4D$$v)Dd1Jsbf_18q(+&|IS`+GJGUnsLUDS zaebD+38UHa7=F9PWV-Nj^|5ujc zRb4@SfG;rb7sez0@Q#2c?qx(eO_fl@~GTy@sL-r8Z@l!~4R$ zhYJ<$SxF)~1`K64><6mhZ2x;mOvw-*#mo4EedKHuS$xsOggpcVglis8RH$5Z%rDTU zM6QjAW?$~wN$hM>Vm$_dD~}g}8{)pw0B|j@RI^cm)U$MrILP*q;=xG!sAZGDSs2_q z)&?iuLVQ!#k}NeYm0CEl+c_6>kxoUk5+`&b#Ci z0aZ2qTcdX4u^47S-s;gfl>%V=KBM~6q89K@yVEq_4Ig2f2EhZ6p9{W3$p~t&ntaJe zh&Hl2+a2*{mn(NmbjFg49hMQHX@n0or1(tQecxZ)=yKS0uK~Ge8UkhKh;3Qj#qpO3 zO$eci(x!rdg83rSPSo$P5YhrILeuUVlh1lSv5A81JhfbTSjoX0)cXzi)fY7~;n7No zU1$sCyTWRE{z@-L7FfVoLDNT;kIWMT%N*1impd#_{Ghhwz+%u@B@Kn z*AC7r#zK``Gd{q2kv+CRF23$Zl$V3^#R3pxY!xs>i48;t!1Y|CU2~04^IxaiKr{u! z-4oF?cdNe`l$ZAv2mCOdU|Z23^JZPXRlhNCQd@x<|D{TZvKKJie#V-Okf2he}Uf& zF-IRdf!Jz*Hr<}cy7L=z)w@WTvl`7q%<$w z7qt)mwt*@#vGr^=N#9UaG_Cg)I$ajXNRGej=iGp3=Rq6M4Awi#;5ag|miFc9yfo;* z_eZpD+vf%dS7LRMSHZ=8y4!x5X}ZUUalEx&HdDPZ$AO{11g5c{%m{l#V= zQnpc`5v)n$5%R0+1!T`LURN^K_DW55LmJ98NU1}vSQAa)C<=GT)PwL+x@Bkz^ubyi zBNP4plkS~UHxx5jehHS$DM2% zz@Ggn2m0x0>5>#7VV9A5rTjn2S|2|?p8FR(o&HLG&*|DEmfrDD z`n5T-Q(peiBDPlGM93LivszdlYLR{}m0#AZL`SQ+El7D%f1Kl^Y3{LuP@zEQ0IwFW zPfkx32UpbX?k;I|U@B?0h(Dm#?NP}XCm|R=C-M7U?2G}TYpNw(ODpMtf8~fketA-P zDIX8ffv_v$!?+reQu{_BY$8NYuA4#0r+Aq3yG;?U$6h?5t8Xw5+tn*dOk^6RkLCnC zYN}SA7L698Ni`lx_sE~x-~go{M*Acc-5F#II?aCSiCZwi4XE$;H;Idq0=OXU0l589 z18`Ulx$N~cHp)1uJx|c82+T&9-qduX7~Vec=HtU$_C!AmgQF`}p0lxtwg&=|<7>%m zz;M`Y>MJmN9}OdUSitT=IsuGfjh=}cT?dlXh$2^m+0F|Z-n-qBBs#6G_C&@zi7={1 zM@n7=EDy8Q9mm_K&f`t7?Ylcx=&HF`Z+Zo)95@4o^RNo~R;YT4hqmby&P@*IsVRG%3-9x~BQ9s0Z zIO5ET6FIW4Q{DGb;^qUZu{9t~VN%H#ZL>f^Q?DO%miuE6ORG`7-WW=-jTH*Z_e<%d zkju|}T=tbMH#<|@L%P3a4~xFIIO6W*&3^wYPd@)n4n}(BegB@3E0>kT3(k0EXLD@f zB}po)_4X&J#K~JBivCw0--QLsyW~zXFElmzHa<};wBHHk(r5r&JEC@IVi)?LD)k1Z z@ei@P$!@m5c+%C(xCSUkG6r1M`2u zL|FwbmnOvIPsj;+3c$Y!34)$6Fioyc+KhodTZ%fM9*b+Jt9j+i4<~Wb#K#g!uD;e6 zn~-UDsZHOXMYi?wj3#}v<-xfW&V;TkE{3*#^ivc@*cNGzCkZy_c$3*lF*h*Ex>?-rYER%%>PH zWJ3G$H8zd4ge21&i&)DVS|_mRX#Sx@hNBOX0dTnw3>Q0*bmN3Ul9H~lh^y;GDd;$` z83x7NEe1m)LKa<{s*iV!o$Ug^x%rwCkl79QoE;dAU};R#bnDQ6E|6@xckG@MM9pf7 z+Hoz$dbYQ(Qh0m~XH}ZvBODI6jbt}^JI4^44OmS*erS!Al8US+&2fVo=*^&|()%p!7gea)Z?7p5x-wV>#$1 zBSh0Z24m{}kpZU%xl)8PCfI8MeVnATqOZDizNG@e#wcw2)IW^d7Slp0E8>@`d8pSCd{Bku^fT9fBR_0Id5k{hj+_uy1ECi5f^xh~z^G;p~MfQe;YBvEP(l;jNFU-Y3@w#2CkT7l<-g@YNb zax04L>@LaMq1XwxP*W;uHhKLz)oPK1S)8ZfG?kiZjC2sC7i5~(Pwb|~ou+y}Z<+0)Cuvob=QDM^(aUpXCjJ9vMbgo0U zJyJ5&>M=5?2nFsut^R5LvoYntb-y%X`vpWQg^!QRD02!K_xgZUk?sV1m6^Z9oviOf z#(VS^KNM7CTV!)i8su`3-N8f_sx&e#NM`d_E-WEDWAqz&?J`p^w&QsN$YTVAf)$CsGuerisC?+rTyq65N)5U!pd*rIj zK$Tg~5GihXGOH#**ZY3%k}H2(4|Q=CoWp7gj8H79TbCH0REv=1Q=sBB4N&B|7c9E% zt$|Ui1b9;;;s}k-Zmz*HqVwv|hy0qG~|evS_ooYeV;&`!vj)MvK+~ zX-05SqX7lrhR6QPS(T%-JC#TmM5c|IKt)MuNBvQn>|@<;1!pCRuVJpt+CPUrLsEU~ z+wbLNO)Pz57YcI$dz8<0Q(k`lHY@P5RD)RmAg*LSw48*BMQ13`@&G9(X`IQ!XoLBi z9L_kSPsW;g%1><=2|aUBt!*kH7H$b%o8SaktHXku?3>`E9d)q}RGoR-+1}%DA=8PB z6xb*yJB8-+>9wUy@BsxAkF5-{Yq?~xv6a! z7As%2S&N6kcCrYF}r5p+DLU2+99mKWmIp)F*GE=-;oHmP;0uTz~dD zEhK>)78KJqrq6Wc;tA28dw-s@tuvTjHLvYzWR4w3yB5>Q7jj}#HE%bh+9Hz>m@noX zT=ebY`i;O^v_Pv3S+VpdBmztEx2rLEy)oPAcY_vifTC95CGolsgeS-G&n=^|byX26KJ+7sf1|P2=3c4Su-k(h+E- z&OrM~ZkRvI#+QFMxSRBVkI7m#CN)pMi}gQE@c3wGXtZWFHdQH|#{FdL{!s^k%A_Z^ zCe!j7uTfdIeHGjPg@9j`@={nSqyWE-vXnyy8r|3Ljr~AEd$9Uo>hgIUasSQd zqu`+&%C7tN?PqC5X>Bmmp;A{FA|g}lHD9D=q5-3VrLYzZ(a>>b6Im8(rUhHLNWo%g z%Q7h+GDZ>JQr1)<4`N-)oZ|8Q8~Yg2Zm@UTRt}zu+*0jMER43Xxd5@!urtm^7W1Pw zSJz}LcS?nloMj!{j5F*MH}ujW7TfeAGxHQ`IP{w4QociDAS6d|peQj*JWr&c5w@E~ z@7VUr#C|#oBj{%G<5#ePQhi(*w(to@M!`{8RA$}!&)ssJHVh05deb&5$7qlr3LF_3 zm9+!hfHJE%zB3q*bqY>fn$lV_2q?CY70oAW)QJXv(}6Pd@w&x3y1+$-j$1`k*c+?t zYL@Nz)yOBp5z=I7$|AJC4v-~u&p%<|G zfNy37^SG;WSJfmrzpPu58=4R=u_ZQWX{6jw3i~~*c5rrcq{pqIE!@eek_h=eARo)F_XEGhdz zK!k}a`GBQa7zQ;aYp1ycPkeHp$%ECYE4tU&mX-)BvCYwL262 zAFmX+4U334Dx*Pf(82y#+70pult%3Ejz~MkBLS@)k31NZhh-Zv^e7_N#gu7`gucWO z2_uvLM+*t@2-4#sK@pvOpC6+1=aEi)#n&FvKEvmFb7~+v&UlR&N^p0oZlt^CVhHdH{vY?RDtw0O% z6`g!1Z-6nW5pA$)^}f5*fYTi_#K%7ZAM*d4xpR(J0K9qT znAiBY1C#A~4G#CtTG7}(H{)*)4yBtpSW~3B?kHphZUX zPX2e~nfP0Z5?h{FxWFVt;y@CGOMLu7$*Leoa%CEHjDKoKQ}m5yl=v1oaU&dJmfnkZ zp99Ds+o3An5f956_Kh1ij^%rG1J9GVFoik`qzRz3J=e1L=V2@T<4C>k`=Xbu(7MpTn8i z8OngO*#bvqe9XctSkRoz)|sW3c(FPqRkkQ!;U}MbA_}3@BWXa2WeoH$SvzfW9b!UMvKd?b5(7&hxkgk5D) z?X-$xn3%s6A@e$(OEai=8);|ZrS+>JkJP-A0Ua-CH{by8W=3+8rib4EvTSq!04jt@ zL_t(;(fnaeCf%~Q1w7?cbPhEZrKrP&FjM~GN~i?HvemA{nMrtN@lLXjq(xWMbS&L;ss_X>e=36HgExn~B za$s@sd1osH!-uhxU?U2bnL~v`CgY;+D=t!6KKMv!Sh6Mt)#Ps2z9|){(wUV(oE&>!DpR()t?w=ja9YmZ z-mdb(^Wv{#kpH3?&lns=1A8DDoDErqp$H9029r1xrO3ilGq{s@+|ZVDLVFu!44UAJ zFnOB^Orp=VfGHgCoVJ^z72V$6w_S}RQnJyO$P(P@gbZnbHd^G$RjLdf=vSU6@Q>_S*mfFX{()^iB_9h8MRIYL15#qm^098K@VMiZlF?@= zS_8U<&yhjFoIMzte8v3~|7))+&Fj`Ht!)@|Bm2K6Cdo*7jBL*+6hgYbEs)n_YAMdk z%Ui{{fr*)SdggohB_fx10(7wmyut2a9G1yitMGH=kd)W!S`o}8jIzJQ_vk__R{tj7 ztL^fJAEIMj#r3Cdt8B4b-$=WP^09d{RUdxY0DyHVm)X(oLhxeNicF(C_jaK&b-AK= z?b_ZXy!>JnGF7&KiL#Y6pio#@wdzrnRwhlDLZtXNBdCj|_S|I2DF4XOqeu69;e{7= zC&F2;H3T=p&n3ni&x{B4B_$=RjAys*Uf+u6q$oYOH_ssU_ISwtC)%+Tm(a1q)a;K< z{>zCAA&UOP$y4{PSXHLwtLMskU%9;CaUhNZZpcm2$2RHOOL>9diQzUqkL@fZkI0|6 z%+h@jN-}LC7PGuOeiSKcQ45cmShi5$P_`7M5SsbzRx-&WX-K~+jB6Z^r>a4p+Rzr# z*Nj{AMnlpph~$WhPZ%fP=jTyerU8=T!L{5Jp^E@0r+9=}?&ilh{Ye)oZra5_MnR9( zuVwPN9+~J?2ZXKz=TDjRUI%$(4}}l<`F7M{$SZa@iEw5bt|DDQVtiMAro7$B5_dpY z>9uP++j(!6S~UQD=@>auzy!xs*&^K%8H7yIr4&sWtz7+ljg5`xB_kMhflLN6;+DHr zMuVW-y}lL8jrQMI5NCFdjgO(vh%2FO;Dcs~M}Jv|3*j5KgggX~1HGIBuE|Y4-!rXR zxQPH7Z*@iSWV?4=-wRD#5@p4i zsWGnoh=g>z3M!(giF({_RqoHNobORmz1dn$Piwac%lsOU{QobCXpP4oc+ z27HgQzVD#CC!ti2=$4i(aVx>JdWBsj(V|n{1GqaXE|hr`CfqN20*85z@Y=PVH=L^e z9PbWCrC2WV6zf*pdK%f8XTydKNtGO-%;2-AJ%`NyB3uitUz$;(>|=mEkpzBDlDOC? zDyNOm8F6yS+-DEMk45me{8o4 z@Z)9*{q1g7@EjD_YuEiu#q|RJTRv$<1>g8_ zz^ry5)aiSVn5FmPJq~yrXy<@iauceEPyNq-nm0_Tc&<3$KL*9(uvRvoK%o(xCXJAw z+(gdV%8MvObx$*7Kgp*>tWarE?6z+~!Sn!!koi@m zp*gItov`B7t?jv)Yj2z_xe^5R4lJCU2$b*|d}803yYJbib6 zOy^&G!(|!7md=m^6N7;tem?$ik z0lL5Edd(8aWgwkUUNlQr8p{4(o&q5(-pQn5yP#6ct&Tz?r!mM2!ZjtLp42*AEz|#VGeaoC} zoVhlO^{;n>VI?L>8=ERyz>@X>BUk#~^YpzBjA!realqq%DFZN$vn`G6ekpEbX@(eOD`BLNoeA4R0wA57A|#^fk0@^)9+0uL+sh(_Yk+|57p)|@vR_-Z@=`mpz(P5?~ z#wR@QV;1#ieB5#R&0U~uuQU5$DRfYp{>%c)#vZmN&D@pzz~|((Yv(qg4=mcGFtWdG2q^B;3L#`QV|QfZ&|Z* z<;s=sqcpDoYqSOCMEDq_uXPegqAr9o(PSEESI>pelsg{4<3KwH+?1Q55f$7^_9W;xg{-H4s(Cz>8IbjJS5J_`e38ffGXm zWlP7R6v;yrugrsvOQnxPjFo+1ZSx6gzlgHcZ|E9Lu~^I^GAhFQ0=};zyO{>YxpA@) zlMClqn&y!%fU?W^rQ&U7Ndma#=2XF8a2p$3S<(R5rIf8`VK;C;*QVgzPuW7z`ZbLf zw5gbELw&CuPAnY3?h5GIxh7V}3ZSv+v-z7m*YQHWPWkLiwPKm`l&zS?@Y*$o8PT7m zrKKxyDKXt%J^Fr~2KrJ-g3wz6{+ZF_oo>8qt}9!zMZ3+w8>ehhD9iO)e5@9CDtLww zUf;53AbgiIx2%+{{=mwdVJw_Y- znf3|9NAG_ECKk+2$`)}}mQAr%F0@*3y+4QS|Jpd^k`s`%E~YMH$?aq&xcjxcDe;8B z7!l?Cy3$?o!0q)Qa7twl^+QQWBI$w(GUA_|I- zG=3YIC*pU&0f;q=9SKzwx`N}5A-Uw&fnZQ@dY7}c(6m&gDHLAkQ~a}>CKNo};TK~W z+Qir~%qku5+mP=bK)$nex3i*?1=v^YXfjsMw=-0*6@EQ^wUK%jnb8B*6*mhx#H_qV zJmvnt^aE%hb(OnpEjf4Y+&@qr+c$!SP06Z72R;TjJrORgLM;m1bI`WJ$r_A&Wrp~_ z$sqFGHpxguNWKZ!Z6@$u745aFd2`dBmeQY0l`T@nS|ctrqXdnWGDw=iGl_?`Sj>_t zTf`sA2kML5UuS?m1l;2+@>;)o^=c=xW)>vkL0;dA=ZD!7f`lBTE$wLK`aKEj>DQP9 z2pM$bz~k-Pw^z`vMpUOeu!lQE<}hLdW%QT%uT>S_SJ^Cjq*YlHigh?&k!gI%rs+yp zl&@vX>HobkTsZr9+=>!#y+@A&X>%Z3a+6wBrHSR{lzZ=Od-uwU`KVZD_RGsVc4u>I z2mwWQCv+4#UT@?ot+gehY~_7!a8w<`P86Xc)Z~{zHSxC~88G>6z>T}aNid$}E-7Rq z4E_aqChv%HC-NP;A;!~-pj-V3l?41jw)owKt~u|L<*T;~->Sel~99RjPg@_e!dRM5wl!3Y(X;*)| zf3{c&?p>uWk#eAD{fY|NDMV87`a=#m#A(AuXrpM|U{_g~^`N{Ic6ix>BeZNK|7RW0 zzYf8r+bGcv>XSB#*RJNQqd!|(qj=X&m+ju86ik&Z`eZVTgn7oPL7uTeA0u$eAc|r8 z7c_YSJ37fzS;TsM%Yr!xAlk>$wVA$lFm9^lvB^-h#t~l{8yEW|?iEEV!@SE8`(_sUn?GO;%>P@@cK8|6^OAcUh;SgAa?@MY)e(8< z^Ljx|TUeXhf~6=Q>lTrYlni5CR^+f@QE*s8B?LJUSn&n@T%xwMUlOMhr*O${iI?a0 zc;mM`8*zfmCn17s#0g%^l^4YkwM%vcWXYKxV!j7!msq?QW9)ko>(oG#bBvXct}X5p z4$X`ts;aV28-r0;Som)qZ6{h76R%;SPh9-FL_pEHgE9rJ3L|4ROBW(**sSmb2Cfsg z@ayd8j4R)hx0di5>y>+SR%l^YZdmG60CPx>@M6Y15_?nQ=QDgyYSScrV0ik(B`LWt6eQ z+L0uah_Xfexk+HLQHx_AGX3rKt<*S(ATf`2v|OjexE1Fz$n%v}xd49xOPE#vwMw2g zY}1z~eWCW9H7|*h#n)y~+JurP9DsQD2b8Y$o2E>(T*Hw(xfknkU_S@4DK|+jRoH;U zx}~x%XXEsuD6CTDD_o*`FU)`Ua0u;@>oX| zh$>B@aLE<_EDBX@T)ToL3RV0iIPI6j8#Ii23~lo4$UE3i(L$OMYG`Ozz@lYMs71#k z6%CgK1HPtkcpduM?6Y53x0^U$6};&{a`5BD7hl`~UuHW%Rhj7f8{Z&PS33tZTtSI? zmR#$A*led`HFC+LcJ%087m)V^JG@-Lk&gz`ZsK8M8M=jKi?t=<2mde2n(PN|X14RE z`yU4lN zJ1kRTUi#k0fgH$95vEpFS8M!GkF8#?7Tk8=fV z=(obdU)tbwfMo|;9!$Ua1|s?1hYRCD#`6KZe~q$WBDD-Vii?Z8_L?oU#&1&vIWWww zGE!5Gnegk-pxMU8d0>{niDL4AbBfu+6pI~$sLXy`ZIjEzVv!KnLmRvv3-;R8zG)c* zm-23%XNC!UG;k~H!D4*VDo#NBhen|}zgQ{!T#v-wJPyUmYYO^Q2@bYI>X;`%k(u`+s zA>WC@^(!OJ?9Ny^9`-)K{mYE4NeXC|GV^%yl&#*Ub06-2+CA)bu+aft2MSi8S=7za zRg%01;==e3?v=T&zfn+8I4P&C{y_QeMK1{jjD%h3e&(5H9@^O0_r&Poi`+t}T ze<3!)?)Pb@oi>Q=r7t2MQ&bq$$o9gvrcF}0+v zx6Uy%1olwSNzLq~?FqCgvr~2?uf|Hzihvej+QbRCUgrJ?~7N_&_)uI;qg_ zuSC@f$(MsGeteei37uMHqjYUgozn2sV&#*j2BoE~6>O+k2B9c8#@!M*1)!A4$1aGi zia#%3@fQ|E%E(xgtVFyLs-DMIqAi;Cyr!x5PI%#!&F!^{6fA{&<5BU<`Yp>qE+^lU z;~|}&z?p_(Eo168J43S*BO8C9X#J7AEjo!chFg9-7IFm9QY(*OxsSWtw1SkF_uu7E!n79PVtV~l=(|crRsZhig zhYGD5F=9lj?t==N@pUju6cf_`e0J0g#uO))08Si#mipRm{Rz&El-+LR%4N}WmY3JA z&R9KU$dJ`wn!2UZe-4t%j2c_MXIGN1Mx>&!1$)Cu5OOu^(`xIo5P|9 z%s*z1Ayee~r5xN@NSjTx!eiz$XD);&T0f#JmWmdF*BafYUovE7&F6>As`)uqsMX07 zEvXZ`4emQ?R?WhpvueJ-W&7UGF+^WZwEZNZs(i~|7Fbm~t>PG=lg)r7?T>so{y-jI zlnVv&kDI1V_(?YN^k}Egfn3W?W+j!4V~&|QZ)N3MIKasn z3iXuP(nstvq<+KT>v?u~Jq_-)tMe9>cZNaDUeHTIl5}sgj`71JAXlU0<3pP+Nt5Uv zEGnW}0@eXb&Ys}#8$E#mArKu}>QAECW)XaIK zhRvA&B!XE=g4TKT?P0U#Ju!4<&DqGGmy*v0)4bXNUqU{8Wz*D&PQF__W#W$_Nm z^4&zJs{3wDjPsrkEC&vR++HBvnRm!84ivQ^2=h2VbZfXsMxBGoQ*{M&q8SMOS zh|(qIFLnUzQ&6BB-uECSKfr!v5|5RMcRlA<#3@p|AI19CImqm_hR}>nPT7OW_v^)K z>FR+CPYT^tRYmPn+Z3ZS`Sut&BbK^5_DW*B9FKwGO_Z){&1hyCj|FM?jvYH*Ag_r| zXi@%G@SORp6WnAtC@BAehNJI3WvW2oT1)-MB%|eK9Pix0v$LqD5_dEJO}dXFk1Lv8Rsnml>( zvo<(62V+r~?j%z>#LZ6ddJ>M;uBo-A(*Z4!D(SLr)iUzTAg^W7$q9p0Mt84oWyC&( zbym+q;Ug<54nAU*3Y!(}^k8~nvUCVr>v11C8B5kb_?=WTOP0$4Wi)|A$;R)@9~QMgc@&|*3V#&;#eR({3#`6yidm4ZM3q|OGh&M9*m zBk#DzyY3JAeD&1p)_pr$o?fmfp-QD%t8z)vGHL`*e#P~)@$BHiL??+T_Xl_%Ok`P)Q|nw(ve38%T1`oT36YfVSgvO%>*J@Ka*Cz5 zOF~jdkFseKe7yk!%^M_;4YHhsAEBFcY#^pmxL;7nJ8V+E4?L6A$i<}ZD;LbObWH-s zX?Qaj3|^e>>^k+91>4iCGNY__!>rq^a1Im%1C^&+h=XP1)xdDTp z4e*myuI7Q6j$?TN-!aF?9OAWWYCNPLjUvU%sil(1NA+UfjfJVEO9x+O)fnXLX8k*n zX0ZMyvxN6_0;Cz?^{q4+7n6*({2guS$VDV3S;tSt{9(D=85X7H9(;)stvWA+{h>=n zJiM^JqhK!n*m&i{vwXhDfz5+$$hHFivmS2x<*@3-R<5>|WJGCG{n`Tv!)KZz>dIen zef)+gyq5tO3wov8sf zK6 z&u*d&_dy6YrxaeMEDv$)$Sm1bKc$pWWZ?rl9)1bS>qLIbH^8wmh`eVaAI0exGKJtI z90ihG83ziTF%2rRga*BgPL$RMMz)M(V~oq}3W9R*fE~_(f>ZzL2rd6@l|Q^Lz%57C zzWvz>ucyGgc1^t{S?jnK|6|!=Jr&p87W&`v)N@NOh3sa8Qn;Dac9!V44EWGI6Bm)e zgV(n*U|vj8#%^(ePD|M$?Mt~-w4~Stjmi&Nij+e9hh!WLV|jX{Ft6oX1E()CUF#_6 zG6};UowuTP^5hfs!tlM=%qlHtMy(nrSJhful|aK%<*KFY%8yYL;nw#DO_UI zeA9eX&0J#zhL{NBTT4|;&?Vm<&K3?ObG zX$D5`LzoymmuJfjFgoQ*1nJjtOw-MYuudI55uuAciy8W#!-o(5R;p{XV(UQ%yozV@ zk4ro9s#jil<=xnHUFmmze*VMw>E2D5>^k60;PeV*%WK!vd#JeCF_=9^$_l5c8BA6% zpf)@k00H zfGcuSDpfZWu91~Peaf*T*SxoGO=Xi3{v`_6gaW@WzrM9i1ar2bP<*l43UNx9z0`}? z5%AE`GDWM?m3=5&K`dU`4snVW)-0H&v8!SrkB#j)w0Q*+EkD?@SAC`5nxR@bu7i{( zw&+$(xuMO~v)mQI;rN z|Kgh=^rusoRCN|<)}u)$SLEms6st(SvQ|)naR(lGHtvy!6#u}7l_<5BB|C1$PEb!V ztE{5WW~TZ}uf6u#S2Izxq9yF4btaY^rah1t*Oi`e4g8ogF10HY9p{Vqu%2p%*F)f5 zyQa~Q^rI!IBm>{|0o5*l@?2zxlK_7<;W^VsuWx0_!TsqU+5gkoFWO%=N_o^7w)GZ$_c~r_At8q%M-{HVN(q~1v_%EvlnRg@vJ|FGMZxNy9}F;w(b%Mg z|H6dBP2-op`qH*FYr?WdD^j=g=%<`YCPZ29dOhs=J&EHrrj)6aBJTj#VAMVExbS?_=wq&%rys||Fp5hvXl64uDrZ@ zq?}P(>j?m_56?b)QA3xJ>h$M zT3VMC`2Fg^1^EG7^F+B4GeevrAv#!zSh)I>wJTg&D#L$@C6*#!aWKyedFD#s25Q#6)mF{a;_lb-}g~!PN8jIG~y5Row7W_@hqco?_Qtn zvY^Dw!K3R~^0YicB-`akLSJ@uzLoa5*~$>qRh6yX!cc@d*=cY*SI3SXNh+FB6m-R{ z&MwSL=X?M=qHe=c2%n%nVqUciQ{{{VJEZOm_u4i6)?*M`2mf|Dr0@yuc`5Jy>5#T5 zTJ(KfZHf>N&{-Jknq|xDTk*`z^hw-1;i=QMjAV!4m2(%uOzY`p`XT>W*37}ecWR37 zXoSL)ErThPr5RrT*N|EB#z$QlQ%LHe9$*e+O>WAhUR6~nQB_r9F;c&{V8QNx&zSoV zhXMbvIi&p$>$l!54hGcHykLN}S>jSCN{=XAf-g4|t^#F%C|umxoPwBkNCceOAG*3O zd9lfb>MLShRxzarY5Iqa;qX6OHSM-tdh@-PKfh*MRaDX99;XmymVN4}F*v-6tKMi* zUeC27bqemJVEq1&>t2!SuyRUj33RN4`h5E7 zr&Ao@bxRblT{CMo-Z}YtEpteP^&k;^hv=@NIoj}o^1YGn4n&Ei_4-yKN8EX)jb@}k zso~H=4|Oumr9iTKq7I)q_rqZ`Yp&uAJOi2YabDcWY`hD#kul&=+y;*sHf#PL4#e!e z3w7vKBzT8%A~(6gROwH0(aC&OC@gaD_sa0b+v{3eZ*OP|JwRW6T~YPMd^V$CYxT%R z6;XVYFI_Jdg^NuyO7aUL#Ysouk`7ML$wwmtaRpQ`t0|Hn(XIS_8zrk6i`Jc?*47_h zvTW^BS1wz>ud2PwI`;#rXi>;Wknb;^UJoFrexBcd=5jLOZY0m>PF`<8;ChsuVaVwB zXGAQ6#IiPQ;J|?=P@Y@4QkuBSAX_7;PM$OI&^V5ExDh^VF#uNgfECaU7=>zL_CR+3 z_zZeGV}5_e@14ZIi5YSQ+HtdyAB8u5x_akOwksf5YD4ssK7IP+d`*f?nXJI#N@%M< zf+Mbk|FR;~U146kX5Oy6i%V&Lc|R-z?`zIo2#o?Uz%OEP>rahmi~C{=SV|zTZ)M)M zk};hAChMG6hMxt)%N}v1!UG1Z&+w1oz%^{LXJbS-M`67R!1r?=A(nTL5H#9w+ z2@2S%=sx!3-~kjh9FSeJQYl{QvgOShZmo2*XOOKnfW` zmwE}^Oot8yzeS--hdx4{qtDStXdWO#$Cbu{exzqzS{G9U z6ahs*5l{pa0YyL&Py~)6z^15n@*M{;g2)7>H6tY!k_6p#{B>aLO;wb)lmJFu+Zn99 z57TTF@)w~;F|w_L*{V*iJf|D}Ao87Zn&5)@uT9nZK)v?Uc7JF0kOX&kcY-?`C%C%>CqVEZ3GVLh5L|-06N0EF~%W0R)252A*Z%VS)dtaHY*4 z5JH-ViiXn%uq&yZgRQBBl?kbnyPXNCiJOHf2;{aBN2DTL-l8$#J% zbU->Z1}C@J=Ch?nm0+XrMTB3_}jX!9A`YI$tPOc=YpJipENh8=ul`S}}l6?V$a zVSCL{_ZcbYcE81|O<>b;mDX2t^tZ$!PtWyd-$yxqFSr=4elN6Q-|a_iErOqJQ5$n? zf8Ce5Tzj-%dzIQGO6TsKYpUEKof6xxXT{&GsJV0aL~LHQ!5xocaEQ77)M>e<5MiEf z`_N-Dm~j5IQk85t@P@r6hykx$AFh0!oX&jReMxp!GRh>el=rmXZzlx!$x=jrRw#o+YfMjB&|bZM^a{D` zd+(=;R#QLx9cHwqEm(tEpdpuGbkI3*Fs^NfenbFDkvz>>7FPVVlVq?#3Z+uNNk`sP&ypxj*&yqB2 zO_zg5ifUDgYwPfd*;4B!N~UMDY_mtoTAlW5>sRh~9Jewxk|MbmSnhC*)wR7N**GrdO%}_V58L|E(sYS#Juk1^uGi1|{iX8>c`j=v$2iZsYK!Z2 znlI}Xj~mW;K9Z>Tip%*39UUvhp?>t&a4)wMP~3E~Tn&z{z(gDtqiUTjWLzt9H zpSvYU&nQDdfiRRojkDGOZAhvSW!!v5)J)RLuf64X#+GAS#<49+qrGVU?Egk7^!LNosB8ccBm>10o?`nwJHg|RFh8xg-E))(%XCJPAaV`U{AHfw80T$$PKIw?@dtb6>Ryy(5AV#;&qA&0DBA=ss^TRd0Mfi>jS<$ugMm=qPP0gP(F6v4lfsWR2 zCh5zGp3R=SBgLIlLZ5YqGd9DEuY^yE8p_7+S=?=!_+ zLQd^npmLZxfgs!)Z{GMwOv6?YsfS%PJj3K??GE`^7Jlr-6!%R09mP|xK%Dsv1JL$zhP5lHeT?b#R9v>osp2y`0x=Gl)j z&lQ{mW_Fjn;Gxc^>Q3Gh=6196bp0la*b~F5VBzxqA$VQEzwSE5$pFV>eeeLQaLk6q#Mn~4-HN^3zct;b=BK~ zLgL{N9d)(e>$HzJRF%J(I>}*k;Z7o=@A#y=_yfIgMJuTQy@fl6*AtaX>A)-XTTqrp zgmQwSh8H%&Yv#@b@?TUCqu~>Le)B!rk{c4?`!T0egR6ba85|B!ei5IR;mKRiRvd^- z==yqXnp%e=uT{m<-&rZzjgshncXHwDE3f#JVc8;eF%guZ#fX313DwL2YMo5`P|ATcu`V(por^ zzO4*`b>oc?w^z`}dx)kf!M{c_n(41#JLGu8A$0B73hb+Xe$gtvy4dLKb_Bos6)*mU zdk2bp+Ouk-l9;^EX@@(S)&iUIf!;*ajo15>43$bOD0sO>P~rVF+{L zk%s_r%A-|c%X_;-*a8ll7us&r<>a(_E&!QoOB2QT+wVo7A4GX^tA2l@lpl^inVCcH zhQGt0Xef_a3YN_oNR>He^v&f-gx7L9*~H^=4B7k=A}-lQie%7$l}CSt-X)N--t^?; zrHTBRFzE-4w0g)u#UrD2$H$A+sr-Rb)!j(M`yFp6D%r*4C!v>iqpOitFP{Mtcd!0QeYhsGi!tz8o6BRN+9z#hh6(>Mq zX2^qCnpKpqb{8~dS)+Bw7H&J)FTq#Uoe@N&5@+M{YUuS6?E^uv#g5!70gDOh3?z@> zG3~r=k>1|hobMrOBWOR%W|R?zxVygX%{#?0?NZFC*kaMunW*hT2ZjGe{w*8}-(s*n z8qtG*8`Hnj?%1*%0gHnU{n~=El(BgEYn8u*C-zySN2Xb(D)r5g(-$YE=kd*LA?@CQ zT}06&gzN#^A#Uwr5&7N<*znv>u4@v>`O174)Nwt`Z+J2A^r2|0^n~8vdWZFYoX5W+ z$YWid(6775W>0+WqD&qL^Jv$U)bCh>s~W5ZHPkt6|6BGo#RVC2dlYeK;Ag=q^UQH! ztk+uEBwIGg7e)C;;Uur{DMvS=hTTyR?4}4i88lNtu?0&9{YuFiWXrAJox}}0=#AKE zoCSiG;%7}Qv2LT_k>9*>!tn3@BS3Pp_jcW znneEm$@UGhf`&{i>2LINqg^MBZRrb2zDZZER`oP{0=&p2BZXI|ZffH4oxH^Q3o_!5 z8G>qh=Lc+=YrVH~dxe#zdRuT^E^pp{hcmNhcB;{?x;m7hQ3E5^`9a-BuuhA}j*l96 zk>{d&X8oR&qTpR`9SLPiKCWrd6hR`*{PdGi5v2`h_dD0NQ(Ru9Y0T~-w96KwU5 z9xpYkb=j~heJ3 z8f~j2H8?`Q(?c%-fpRLCBz%-I7FHl)QwlvD9LJ^o7!8~1KN|jl#`le5-$49TZ&6}5 z6K+^!Hu@)33M#$>?GLqlQt$F{ z&vmB-ZC6){A5h!0yoqQP{~DYGO;KXBs@6W|K3^I6do`p(v8hM+2Q~TGh1a&{sn z<`$A34kpST@+w9imPTC0N*4xVpO1 zyRy*RI+!sqadB}mFfubRGt&VWbdK&ePGC1W8%K&45`WPUHE}d@u&{Hou(ct5p$Rs$ zb#~$-CkNo9{~!)Pjl|r5#(xa(^8Sasqmwa%6!5|dtOr0~U}j|Gq+?{JW9DM`dw&2b zC-=|ZHje+`BEU}uH?SQ86Fnn?we`R2;pimh@~`p!(;kj0K)}H8!Nk$l*}=#}%*DjU ziQ?~r+F3a}{(VemN0XPPKl8RSHe~=N^=IV2_mPm2Q~YP27dDz%Slj*S@q+z#Ok<;e z;@LSnSpC5oLhI9ZtfAJAVQ{~J0myZ@Z{ zzp?kv)cnJ_6hNHb0-(dn$;k<3VKSs+W#QtaV>RO7qyq!g z=#034RyIR66DBbDZzL2PEC9s;Tm5}jFQkkCQjDe?EG8^mjC3!PqGRP^G^GQZ7&Fp= z!E9_?9PEZhjE07PNWE+uw~(R~A2~BUO%`ivt9~fM{A+gUw7B>}<^b z0A4ncTg2AN)E#wxS3dKhyr8U{6&zHjq5)^HU9?XVE+Sr*=K;FwVi{B z<6qs_{?!g}4cI~Oi_ZSrP=D}Ww86^xrCHU+0+=N8zq)Ar3&uyz@i&0xe*yq@fLp}D z1ngw%pkixl#Yg_aVbT{y{2}-Pma{dsFm)FNI{}MjW@KWgd(nJW6=o)GMow-o4F47K|3*pK+}73R{~Miuq5hMKkb{$} zt%Ie4gM#5l6C{k7+gU>6hPKYIc2@h^~(IoQU`1UP#BVR!zm-Qqt{QZ9C5 z7Isc%LqIi6Sm{{7rY3ZThAe=xGqRd+aDq9QO&R~vgnv?Zv^8~d1v{7snE`?a$OB+F z{>TF<%^%%*`|s!CYHk7uCLJRiHzOnYU)xK{%kXl#{M+()U+nLHU;yunedd;WQ7mO= zJ3A{26Ni7#n12(L{{!CN`u{_C|8L#@j`mk?5nDTVz$BSFDY)AFm&5-jh<{O#wJ-t{ z-S)ql`rlFhV$0tSVPMXGwE>4a;43iv(^>e3SiES?|HGeuh}-|e5dhNvGsu67zyHIo z|6$jEiv#~HX zdVcxMXw8oUT3*>nYC3{Ih?p-Q2vEv5T%ZxoNlH!(ZVQ40jRWa4VuBn5A_Yl_3aPj) z9C*8Xsr+huzUDHceoqN;tqVC-TF>_e6B1lmDTYPqHxrYgrd_MI=-IrqVsqspb^X+I zUDsOc!Vqpy0trzV%J~kSOxCny11GitGD{eeGWoD&?Dz{yCmQqb7MA4X;lT4tN6O?D z$CXF#^y7Pe#7=)GI21xoyJ+@=rbpEdh1xZ|L8J{9G;#a>d)T~?QI}>tp5--ic zKc|_x9bE;$dOx}85{>fYA_)Uo85lnUK?n$hj)!%$0eZJ1T>PcD9&GWhhD*%@?C^Pz#u&Ted5-D^yNCaLy z0`C(Z;Tw6ZH%2yM{zYgi6qe*r)PAF@uH_QaIROQq~Vbwr~!yX_16Mo#JOGllP>m zw)SUpRs>a5B|W{UNCJ*DG$Wgp7WbXqT_IYWfGiqCESSZ5TR5fd!f27YKm7C$p!`|s zG|mR{{R;m&|Frhd;AtgCbw80OXCHSM+K$h!5kxQ9eSZac0?|Oy(G4Qc)@gSIeKU^B zBkpKYqW^6LQkc(S0yP{+;@}f}iA~mB;ciI4C{Sc->wqD{&0lT4!v_8SHE=^%NccUh zNqjQ|g$x7-YJpiBx$*Bt1ZlC#dH!k$NApkq@}#Y#1hJYyAc2SlI_X0dI_SNl0$rIE zv9#zhaC39ZGSl}kiH6_an6Tj@jfRqR(Xf~6L97MG!%3;CPGg^n(X!;f9)1*cLhBGB z>k$%DLZJzF_<86;Qc_UhX4V~yyk1$yGBrKj=;7hfCL>B8c6BR5isr^o@T;h(=#JN~ zx_86o;QGE@R@Cd$)xIx?b2X1Q70SOO*y`$K0B2C%d@iOiFBTGP`l#u=lWu!_=$>^f z-{j1U%UD>G=}yy=Gx@uEshQ=OnX5+~-PZs{Kq&QMjhO@~T7%5Cw=S~Gs2Pr{|4n$r zVX|PV@J(0FeM&l~|0^{vzh`K>)a3bgXR`H=ysejn-y9)lq`-ZCcHUxl-w%n$OeKvzHrV*2&5 z%CTU?Fo-D$ik{p*CQd>MekPiSJt6h`l9y^wTVGE?rPdu&w2p=Z`xyic(lL*7(*H(D zELj^tZ<;nTtUql01*NP34N@4s$B-#dm`r5$(hOupl~rm}?`bI`GemIPCPT>WWX{jx zt6J4P1hJGB>F|oo1~TKzkMFmOt4~+z(d(0II*00}ZGOoDMvVk$x)}Nz)v#ne90tA< zFy!1Yf@%XeEWWYHwl&h(G=pCiPS}jX(Kbw9bvgoKtUX$2I+Ds!esNg8`jm{8)kD2I zYrHbsSn|Yex`|2}Z=&H)Y#3Z_G~bmV-M+6Jr&fN%?=Pu_=f}_&rsj@wmI_)^2JydI z(*v#g7dn`~pX^6)+7+h7A&5@R*uPT(&mFpC=jKx7^CZAk#-fzJ-a_a`7OpH3B%53h zGRRQCwTou^hAlAMfZS2sX>c+N^2xW*T2-9ag`eaK%d;RQuc@s?&i9ywU4sL)S;QT) zG1?ZXN7O*Fv9X1UXPevFDuf#&cezr9;((UQ_V4`toAMz|RNjzQ4-TsiKj4w|4BBx% zG?nd_?=`h`ZJyFvSXrr@*+a9Omr7CECUMy66|x+E_mR`&r0~$mI;frhra=7VHxgbJ zHp8Ms)$&(Sr^3UH0-99$t;$@QL4<)S{nXFrm)~7i8pD$hLvb);toeApPWz%psM<4= zey;CuOOzwq5=&wiJu`|g^#2@T#+^7l#xImO_oYk4{w|#sy|mK~q5-eN5!42GKY2U8 z6cWWR3?JAH-f2S7on1e5`4&Fo^uucNA3uJiW;$>3WSt zw@`8F?4H1jzDi@2yUx^Fb@Ht^>=VLTbPvxMRd9JRc)fmZU$uJLMRy2id1+qp*{!eY zZsfbn>y`NRgF$c+4j*W8{)c%MZpRnU0W>3<{#8E2nF%f|u(E)nqIbio9JUujFl*Y) zEgfaL=K9wX*49>vCe5f~)F;(XP_3`l&PDc#BgK_eRfC$EnjWaMMc%QA!Jno+-e0!^ zqKN9OE5I@mD1#t77%fZ|!y`R~^1iBKtLO+9oL)qAfT4&by4q;fG z2k#`iUn75=7r%Z)=+W1m=Ax986sbm5{Y^M0M9RBcz2f%vDyMZbaCucDL5CdN;MF(2 z+cxL)bf?t?UwNB_DHfeB#FUvDeCK0I3=q z3lbmd4k*g)x^AdRpgI(11yK6^%S2j z>z8Fo_c=&U?JiST%s;D34&mxD#DGzR?RN%+!4|O%?1>~JE}<>Q3~Xp0=>uY(M;Z|h zM8n%;`%(+7rYYBUL@qO)dW)2q@8(g?uC7K;54X1pb+BMa7GG?^=Cb|ztrKzykd55B zXKhoN{=u+qzP>UHx^*Ir0{g(s66NAY8=N7RLeUh7@0Bq91#=LBv^Ya=Rl9)MF;pm% zQDzk^9@Zfhb&ICg-#FrSJQLC^I@)ey&WzZ7?zpH6CZ%nyg;S&c+ z2Oi^L90ACSq{Ag?lpt0J#_y8Ry6MdM7}Bo6L< z%xawm;^d9oBt6gNl=AUhgT0Y0eL{Ir(cTBsv=qZ-n>a!c$~Parrzue-Vh*Iuc1M3J zRdw}t0i4g@W431ZTJ*{nrl#VN5qD9ga^|-p=sJ$C>vx(~Ua{oGsA{$_@-_W}4&C7n z!~E&nYa3J=-S4u44dXO&`+gQyTxX_N9AZtOxn)K5$H$#}5Wi)N7dZ>@o_WD@u_3Oj z{6`Nr{icQCmT~-G^JJQ#4niGN7WPD1lrELWMuI$K9de|exdpgMA7gC4^6rZpSC-*a zY0cMX2!5-*zE8URnDla!c6){Sro4F*pz$-pk{(Ml@eFtrGG*q#rtFN24cdCtSwp@1ev<&DK1N31?*3fLzA!?J7N}EO}5#Ff0*}R9p(a2y2e9 z_w#;GVH#d#wGcElTt9E1Xs34e_x370uaD4|&5@VVU?CwiFFR*J5A_ER_@p{&dcx-u z5HNlO*T<{J0CVpIqZ0OK(2;__xVNrAhAeDRI<)EQ($6Z3VpU7ITt(oqjE*^J2BLj^ zb5CSSf4sAkl(<_pJu{Q`V_4bhu4H*I>bEGqw!P}$J>^}w*6igsyXk{dH|c0t?5>8j zhV2{Fw=_#-ySt=X0rpbLlQDw{Bk4vzg+aF3i9|P?@Arm?WcJ$^_`~j>Id0ZdBo{Y+0+bme)?kSV zfx%>-I!Ph@LBO8@_elkhIT%#`F~voay&R&;>q{du8rZ)ixtXSOBZgzTF_2S|}$Fj2Wx{}LK8q;ZJfQX1l z#sF4pcz8e-yiQIgDjry^L0k7Z`E$ucL6JvL77d;~u(xR_>Cuk}AUZm_5LUVKK7|Sl zTcEDOD#@OY8P>Fo_s8)_P~lM=;9X^MVGOWyS@trC`k2t-R6C*xF}4modezJtb~Bbi z8vG7Au>*tGweG)O`VxN}7xlwQPG#h17Td!7P3yZQm^Uck{%hf~e93JDPK@=;6>X%} z{PYz1L6OeYyDl3%zFahPd4GQFu9zs8@fdt-B0YR*LA~^9Kec!^0&-S;Ql+&T#2XJi z?tzo{&VYM!@D<1x@&q-h-k3Q`1%7*EMg~;ZZ@T5u%wi96Q0b7d|4!ru72~5b5xU&$ zB8-iI1rz!z3T|%DHdhFiGzpy@8bK9?9$`J9tK0k%GV*IrD}59*Y~K&98uM|xw72?@ zCwL&=U_5=G+`{PjwM*5HwbNV+d|=ZIR>J8Z7`wME1wsm&p6}LBi;IiFxZ)qqg_Yt( zPdh$BJbdjbK1$*9yvl29Ys(SlMdCyXi))(d$3Z9hXl;Es+_M&nNJs+SejmCcZOZ>9 zOe{Mz`8jX@&eC7dV9^}q?C9udAMC*_tuk{T)?eXpN~n%Fn6wr@fNR=RHWdARmYFR| ze#?i5Re^)$+$oidg+QZ2Begah~yzz*0kK%H}co$-YU zJdVB8^M>_rr!pKbcV?Zkg#=R|Ru(pUt;&cT_Aj)aD=6viAu-a4BhC6VuH@0&huQsz z|Al&}AW|E?A-c~xa8V5MBHSKP6VD^{+c)qaxaT^UE?#2fqgI1*6ETd)tgU6U4vIu9 zhjN%4$9#r>;K8dhd4n4T-K>8mq6+_mUfKC1NwU|%Owdafsl2ntoJLzxG{gL4EZpIu zRV@%v(SXtKC+CjDb~gbkm}|kyl5xC(Y!U)QoFr7l7N$N69ni@wH|bSnQdm~jJNlgm zIcYCYsJIm+ujz^ef)E);!28Y#a33wQemOLjY-^+YwRh;HmLqn2t+!$tQ_ho14K`;@ znOZvYXf^-Hy@KU7H4X30ukT8Vb|b-v+u}VK|27Uw?AEovk$_l`%nvSKA>HT7d?p~& zrZyRR2wSXiHZPk$7MYC!&ZthGLnbFVZUEBB*?C0N!38&9v5>tCGLeb#JeU2^IzA!6 z+NF2K-b9ydypBcOljzEgmrB%lUwpMpH^gvaZ+dy2Tq^)tG_9rRRstb@-kz)j#RhU4 zbMrOA+c9}wC)0vz@c6zi^5vK_JFQn?b>@?`s`DY-q!y@t0h7){1gyld+qV6BQYDuJ z6s?NghG@Drsfd$u_toBCaU1@T6*L6+nFV`2w_g0vkPy%TLbA!sFyXN&G#uV{7d2ny zOfpJ3BJGAj0kLniidwSbc2IL4a76?VgV6W6pvPy^qpgt9$|}GHEsHR)oorEpk$G)P z=F7`(!#@=&=TkS0O9fIe@<9$R0L8))zxy?4+1EHXOa$LK_3|Q}J>ZA+TX18bhlhtB zps*WH!wI0wFL4hC` z9YWE7Nu9o^wDfy}^$N;O z$6cnY)_K8bQoHzplkkZ~7F9VL5)WILd_i4P^HwAF(?9`uVvvpQkYjjOMdXn`;E43r zTN~oW3U*X}{UewMud;YdAss3TUxVifH$oM}|J`KXnQM6RQeP&gG{EBgGK4TDby)a^ zZS%&ts;N8$Mp#B&9nNB1t^$*J%VLmEw;zOG?4ht~ko``D)7h)pe1;y9w+In!dW7@q zaP5oM81#q^;{>0grUch4tZ1*pfa5xoF_R zLnE-t#noNo8~0tjPg*BBK=B0J0CZ6K;L1$m*RS2b&o!xHR7oD#{N4oLi)d>TIO%>( zQ4fQ+koTatxb>0%d1dG2nYy#$FH>e~^$({h(p==Labsls82wz5)-Z7 zESo7kDPXCK0kWl_b66fs!@>PINI9@ zrxvin2OmVUu5E~P(PwD)76Gf%trU!2P|1|76(`4q$LsH(z- zEo=jMTn^muIta=4qOmR#cbn3t-53B_{{gb8ShCP5`*mFUW{{8FnKQq&^${UtFM|H8 zRzGaVesM%Glnd`;b=dGhCCws5QPDz7vTW~SmGkY9LnAwv(R(suLz@x8(zWxdcY(s8 zo-z5H$+kHxmzceb9)dXKmZ%`7!1YrfU<|GC4dL>95z7tTq@R=cr=J z2px))vdK7zuzRU~<<<1Ab7MtPE7XK`#K6bf=hCEJqu-V{>YPdtZA)F;X7rVQTB46$ zL@q#x->|1iRsH-k(Ba}o78clN_v~X zQ?1$7upk?ximj!s{e^x7aE_iKPri*<+B)ux?vU%>CnQ@RMuyT*oYo0v{tPLd!6NJF zotc@@6|L#ZdLT*Sa@;OVrb$QZ8s!o{&XI~sbLmwReh?vV@e>xEjp+S`EleA&$m7d} zi&RHGX8%;QjSTROfd&5f&7_Q;g7iKN8q_9<@XrIlJ$1}4(R$WN`Md{v5=j|4YR;pG z0Cl6G1FG(}268pubO=eHq8LJ}Y_+zf$rRX;6u}UKzxAWe#1OZBd$F=ty}<|w6({Qg z+p)s-pmZO)Wa_J)*yuBOh-6;32oMXnW?qykE)TP z%>mi11YF|=Q>6hQ)mT)rjP#1kDsm|V&+B>tX*6guUqJvFMI8Bdp*%m`#{&`la&Z`4 zL04B-3AQTTQt8_DE3GDT#Rtu08zo=g=a5O=3LvaU0Fl(}rso53IQi(DbOTB|atAy4 zI~IK&d`EU-jd`2Ru|@6Db;irKZQVFEcHg^c?+^vkE<$QAxQu)dWrHCb{1VNSo$%(Po$0LD~&49sfy9evePgs-D zVK7-8E~Dh9xa6n{+lYfs26dIU2wt(9UpJ@y0cfsdF~-qyvn!)StC(;i1I)=+1IDE2 z5o~PDaK@^=0ERA=%y@AjA%D67V^cTYzIV3`(QkmPW;u{!@>YgFbqpJQ#FwRLju4)# zDXdVGBPSP8oxTr7&r`z5p91aU&RQU!XGl}P z7$ZMb@{vq*v)mv~8^q*O^BnbZ9}v%cWs}<09MI<1Q0l+ztqkix!m` z(vz5~Dma$F ze0zmTOR88?)r1pK6|!NdCGzB>uUuMB`gx6M*;E`-i#B-TINzM!QIBVJ3CHW+#&C}B zv4kap1>e5?>XV?Q#NU;VB&{@VPfqoN1WiZ@E6_;kV3*&oO%j)0j88sS5h7E-l=L0$ zyXF&*2eo~la*k3t4$cZ^Skg_$_7l!4Rr1&P)y4Y_ivIPTD>F6F9r^*C8MNwv_p=n# z04rFO!?6oaw=D+z{<&9;n72c%dbUD&gX5Ev)qcri6_O63s?*c2Bow6vpg;!)B7_on zn!Or3vsG19-!Tx)tc{F}7Pi|H5XjK^RwpLTdZWLP)S^dBx-!B~H013}Zx7^g!3Y&2Q7;Dn!VahYc9rr0HEn znLnDXuwn*oULkZEbo<>$Oe(tEnrxeiBcVY`Tfm4GC_I@6N34M>*1%A>=bi)v9Y~O5 zgrC<(j-YlT1J^%H8?{S7p|HUJvUyEz#XVK>Et=n#!KR*PvO7n|p^Z2oKRxmMEgMpd zFHy&9$~JDqj^81%-vP6}KH)=hwHLsY6q5GVlMl!n$ldX{OVn*tEFUB{78Dm-YQYZK z6_0E`p|-E4+fC0^nG6YmTwK>?dy(&nj8WZJ%lXq8#6=b6hrgU%?$7#?65YDD>Lnx# zSjEitf)~c+t!b={i<_I96O(n>ADaNZa!BHJKcis0yf5wjDlMvRX4Hj1w%x54bTa;O z7`0com>_=zajKIqdq_6u5R3KiSfMk&zNRfU*j~3GopAr)$g07J{aOT19bwW5Hls@z zHw1NKxqUe`GQ6uL)sWEAM7$5P%&qd3ASKR^v^1z4P`mB~eYSFw6 zk3GS|G%n7{65bAcU8zA!@s913li5XlW9V>}$g&p4lpYB7+lW!BJ~7Rd*U_&bcT0YL z2Ub`A3Mbel86TcS_`4*prN!%W!mOxinau160s4FyluiA;l9`p1bb4N2FB~QNtW_Zf z_q)KxklWZ@;=xl7UYg{iLEh-poQWTX^I@;WM30&9O{29Up3Krj^ewer&Lj-8*Vmue zJKhM7!(Uwy_QcrrM>MmBCgFjgS;AGZ4`+wrevNt2RW5YHf#*c(&dwYfmTiXeGe|SD z<_|$A3tQUI%HL1S4KB!lyGycs{u?B}~;jS zt;u$(wfx}lcU$g%s|8rqX>qG8U6yrszx5#Bt1Rp;9c}wg6p`zbP`N0rZf8UmnDDxb z5uJ}~m=J}jdirp&XS!J3b7G3nWt2$|UwmX7u(a;|1sHm2%F2!(sE3q*o4Da&MC-u? zbv3oR1>+i)irP zF^pH2PT3rU+|r|*@--mg%psWCj+N&n^TtII*lJJ##%`JnYaQTE(xv+wEOr~9`i)6% z@j>Oi?)K-yo}QkDYWG3yxVLH}3EK6xQ-RfJD?H=GNjndFq zr}-dU!9`}e{q9SjG7MZYwU$wl{{H@cK!pHZNW0({4*cO7^SC;2nm*f_=d^`j=Grth7Z&^{&B6v;gKJDOcI{UWPhgg zo^+Tr$fC2xVv@gSSece!`j-~{J-?XJ5gStX8KoAyJ%OJB&%Nt5Mwaq8V#&{KTp_T^xWI{QqtE?|n^2ARFWRyDAP5KKU#r&fdj1^98mY_idq zn~|k`Yx(#{FD%YMMHo?mBB!*2idq`*Fwg;b=pvw=IeuS$Q>!_H`57)h;W@Vf zqCjgr+CC!CcE^HC30lW0;3^@>O-2Zkk{0vN0avft^Zo9g)y{Xj`R$UTb%Q^ILuqzw*Akyj1AWlUu(7RLGFCAGg=A9V6zjmn2eLBIl{f_xTAGf~I0l z7K6Jhv{|bj!Hb6l`OxV18K0Vl#!UlD-G@0|EHUwf6N+lHvC;mp_8f$Dp3S~5V;X+< zefxVlL^7{+CmU;gtSj-kNwHRtw$mXmq(CHOKV?DFpga~kWOROoh}CyFyo=L{?d8Rp zPb*RTWhq1bZ_ZJ)`?*)#`4#rgg;VzVQXJx@4ijb3wsics?IEYauusjp7~4_r+`FI) z-pB=V`w(WllW0XEmbfvN)Z@+7R<)H<9MOb$YJ+hJTZ;M<5W`SObAnRvD^c8Q`* zr69c#1FjXw6B3NwA_ya?YOVmHdnOWumQw->@M$@ zI8eH#5fj}q<%-6S05iU{puOFf{a_kdnOYm9rLCof0z}5{2TP4Cz{UILVXsUvt|f18 z@6QkqY|FPH-C%8oKvAdmvEMTH)Qgv=P2}jrzHm%!NE%J&^<9NXItt+q-s&QcNAhde zG5mLrL6)<`;y#-?15ONEQr>!!j;#{PHlND&xpc)pY`q$k;#ekqHe}C;j}U37Mhx** z=-`|~#(Iq^l6~;@H8Pb>44NG@kXVpLK!7lfZ%`!+qt?3yJb0Il6ip-MEh2GaktZFP zwd{&njT=)Sw*cp`7Z53FV7VPlH{aO(P(n0sxLxzT=<4!PcWhV;hoCu=X3%?}OeeDa-sYs!Js#L`B)K(pdd=Yc2MUKby}c%@Q&Og9;kPaWadz<`gQ8I zBpjLyI4N94mZBQqdQC*KvJq%tg#CjFc(REy;f7}L`ZQsdKDwPezCI63hXR81jSO)8 z((c|yb4S6(7TH9ZNIseSS2UeB0Z~_D~?#)Ko6SyXIVXNaYz){mu+oelln;k#GAUtV4rFq8)Fa@WowdQjz$P zCkv+waV!u(OH>rF5I6Sp!_6(K6cLDo43f0?n3D-FvHZ|1b^B^N$NY6&ZSCVwlsn7$ zx4peR#5K}G+~P-RQCZP1i(9_nBI3=BqhW^1e&6gKOZNFGGLtuW!w2AP44$gr5v5b> zF3l#8Fk_c)yuWsbIm(3H66C7N3>a^mn$8Y@6nd7U=Hny6j!@25bL8Ga*@k^=wu|BS zvvf^X5pkv7?jJ}X)`-HK%zlZO1b>Ez=Ogd1kcxeedvK+z3jb(df8rg3h9wq+#^tHU zOq547DHgli4xJ$&W8w*y_c$Yo4ry1NX93(b{bWT3EMWoQce||}F-g=V-TDP~B;!=& z?t8FV@zPQiqp)r@U(e-+{VUbU`6k;%b39ZBrle2XruV_WUV*I>WdyP|Ubl4t;KBC%Ms zUxnr~Ir42~FYs`7-UD@Lb%4=>rB(4v;-UhwtB!Q?QXJH$X%fk|h&WQ)kmk~PR(j>? z%dMW*Rs>+LIF`DJDwI{!HM0u-mQ)VQnUYiOdK;q3^AO-{+IV{M;b8~_tf@7HOUF;= zN65&aDu}DfmO6}^S?{cyOc@~6<8r+G>oV%S+Wl9i9Tdm|!z|VJ@8Gxz_5^ertGG5@ zFipQt`!d_%!*tYNK>bUAn;%grJGv`ozGg}n<2k+=C#H+oyMol|G!sEDrn<}Z! zmnxA-y=V{|wM&qj$cy$S*fQ=xPr&bsWeW8NBG++qtdu+=s-Ezyj$;zdP^RCeGJ6?f zUh&C?Rw#-2nf%w1NTG{sq;#B0cdEh(KjTVLPXH5e&2qkrBf1?!Ie48D$rx9&tY&8R z8!owV_+4aC(HCwb%Pn|lNKZ;DsLvjrp8J#Jm%-iYU3x)`boE-aQ~Gmgr7q+t`aspq zRZhab+5j3Is3E6tSMt}Ovu{#M)jU+F47fa;3#DU~hhXnHtxGPBcl@J<13%&=7?h8E zcN3o^^FSRo8zEMOjsM>G{k2V9?}w=XAxk(vJJv*rM?0h^=rv~0y+kA72MvQ%+KRVN4_u# zBtwSqv@HT6e7v<(UQxB6A%Nabs+503wAh;!S2$RuUar4Nrx?g^_;r_0U22uq*?|nY znxlNysKiHTsj>oFH}O>dePgn1^6OG0&p8qpL`5Wv1xy`@=rUG#*CU zaUKNjzB;KRlCx=d&)#+7S@#$yg7z&*N$DiU6@l!1D6)GhD_`LmeG1+m)7y zm;!m}A~b$t*Opk=&M-a03bXz`B~ZOunRL1|2}DG~$a}qp-8~5eig5ub<6nx#;RXu} z^NPNm$higZ2L5%n&uI7c z{K)8^{Z+6fV{r#+h|zx`tna~9;BO;Oh;O9{^b;qY))>pavKeQk%l<{OWCq2w59#ms zSTSnmpPbw$<)Sjkh_s36Vxgicx0)y8R4&>oFg`bzH0#Z=z!`Jr+!sZp3*xpuOQX2w zFQ3&0onO2kZzh(iiwMu+{V0;QO^+*Pa7w_wXSa#!DxLbdeE#fBJMk#2fOq zI&89;Ik8VY6RojHmN<-2_RK2SRZ%Igw2*2y+F*)Nw#z2y@-uWgx@3I;C~2?Jt+i^K|y>H{I+o5FAmQWSyFtiqHit zph+^NwmpL$$71|CUi<2^Rnwk|o?e^Y2K&*&_V#w8-P%R*gO2;s_AyY@TON-utFAsP zTYRG$o@FIl=LNXxLgXHcSzce6s;Bn20JAKUG(W2QKy7A~>m4y)gu#3ss`Bcv1r0-~ z|Ip=`Lf086wmyst_y3%1{|3`n4mU@&OsAQB(ozL56YVCdh2Y@t%HbHW%Z0eWsh)d@ z>cmc~b+M5pwrLG5xM!}RvRqm7M34U21$2`n*XLs~@wgwyB^X1d{iB~@B;^#t0_C@e zn8TP_sB#_+`4rF2E=&4JM!HDr_VK!2Xqz6Wb_ylH{HuD{%yP2;Wbf-6BCyX*rK9$mZVx57Xb z0ccGvwQ4z-lDCbEv+|y;jNP^P$LbJZi1)hXWG``PHDS zgR^rj3JY2`)q%=Q=+3QEVKSG|J|7k~b}4F*a7YPZ9OJnc5D;%%-^X3q0XZ@CspVzs z-^NC|K>2Q_)6D)u-Zni*VDO5Y?RJH2?5MCj9p~LIeISAvVnS{ZyrJnJasEm=Gq1P` z*^3C$gD40ITZT@NT>H%9zYEPTa|U|$eRr&VUrqV>sVrg`m4$wqcnePctK@lIb#q>T z6%_=SqbKU|J2Irit5>$=NlX(%M8WQ#g`hh&yck2K;NcjFfO56E+AqKE3(ju6`W}u> z-JSZ6Fj1YeKP`rYSLp7H1*U8uhv?+W4?)#xsO!V0O&)&#@Yy^_~ zcf_$l28#AfS+)oMoLjLKvz4 z=w<3Ne0+T@=bDQ{uXD4tH(g9;L;*pACcFe4)?p14 znwgP5I6l7CK+i)*3zkz0UbL^eG^ecHupXOO8IP2lClv;GFd zr4Ufg%~OJe4VJ0;L93v+a(jNZb!G~yf-P;29>4LwIymbW&^AX+yacI}+@TH5qKp!J`$ilH1^^ikL1}4l^bHxg1=k@V%hg5MJV}hhFLd=G_ELS6S&}%t$1=S-#q~%hjG(jo8eWB(Fz_M z2ha$XWxqW351N~vBO`+u;Lr2{msdD*wKOq^6psCN@^Z3fgP#Mb`(lG5x*Cx;m|5;2 z#edE=iPr3XpP^6@GaEqRx(->`2?hx+a0zTeR1N|}W97Sf6gV@FW7Ba&z|n|0=s6g2 zzX6Y-rvXBS8nJMAWp;DC!g}>v%Apo`LrM?g?`q%aGNm}EVxItLTb2G1MRv9`PYx@?!VO0G~%BQO!4rccPgUY`I0<|Vl)ga*w;E%Zm+n(txF z$NCS>(NKCXt~GCmmG4*e{H=UVRzSMSsy(dEy5rZ*J33wO(|N^U^K&|>jjr)#0~NNV zMbM3F;yTxOkMj{eO}mUM`mha+M%FC@cfI*PLh}LoahIEHTRv^{^TqZbpIT1rsdn@) z_cNgaGfa*=_%!mzIH5?H*k$eWj0TXes6U^XbwPF$3l4uV>On9z-6$NSUgn$n zYDNxrzLV2GY7gZe6k8{=AVB#c7c{BgI5R*0^*^Idy|k|`Q+C*Z!st2nT2wV>LnzD0 zf=UEo7sU*+Z%VKc5W|>t+NjAhkDtE_En%U=>%0ALz`wvAc#AYKH!UV0`Va&DXqyPU zg-~#BF785XMcc^56>#*-?vvAcJ7DH_g+QSmSfh()h_}HG6Ci`x@+o7RP-e}wzyEf# zoMAt0EA9z`+4Ibx*6?tu=IFKNRaQYqmIupa9h(~(2&;9LR69`Zzv3=nCwxLACpATA z9Jgyd4ai8ba!TK}2>6488m=euhloi_vBrE2ge()f-niGQ=8;H<<|3h_oCy)K zt&6EJ40%D0;@XV9VaL=!CNt-WiW~wE;ef)Ji%n^63rYcIfRBo=;Pc4|h}PG3v4>XU zIalA5@@A6+^Y#!Fn5e6>o`{cDPr(BlJe1{CJ+7ypUZ(-2pM`WfHN+P3Gl}LIs;a7* zp8evlx;v8p_y)=630?!PIUr?QdW{2p6X*9@Z=@`X6o=w`g*}kP4=~?F0JL8O=^trI zm_^N%$uCvsLuO2@m4Fky_Zm4L*!>5oiP1n86LqQydl{GDL1oXMtTGw{^L zO1)xF+NIbB-WTzmL51sI^v=WD9HbZT>qmA|XV&WgzKI+=qIS5noaOJyi7`6dDN02Y z!Nz+hjB*<+o4P<6-g}^Z`0Ms;vn{FFU$XEdy7o|&nO=G9C>sD6!-0u6RfU|9h6XGYW^O8FXem8_nqOJy=FAotU}(U!%;Wb)nV zvba*bUyHyWh~sen`%_8E4}gG95#R*g+WFhMa9cpBE3#d*N&G7edc+9M5ZZr*d`r!t z1V%q_l4~Vqs(br@Kg41JOy8+>MK8r_k9|5S!2!Bv@Dru$&UKetUn{PWidJ%E%4B~4 z29w(z-xTsO+B^G1)?-ye zuN&)N)Wj@ww6}lAY#P}n7J6UaA@=i-&M%se=*j|;c~H+jtce&^+niz}qiH4DD(Ov4V4@qzo)6@$7bi!l3je|q_$bVnH zSSWEgLBpq2#FfPWZu`!s>Hn3{^t5{oXPRdEmjKW%gRAL|Eg%NbxS--!82<%|xF-u$ z)uf&=V8NG_d9*~r+@aRe(h_%*uKyG%hYn@~;f}msiB~obyvOv(qC(>w0IV9DtN_XL zLey~X*|My0WbDeQt_`N}AjOy5;9i~K_X!_%Sl>;ehgn3v-%dD;`y+M#U5N~$9GW9< zo1FV=+WaJMVdGl59G9ULvB|jMa3$IKHnxrsM2en1r4BU;cv;d-v*k4qMoxgRI?#;U zP!TSXDWYX-!QbsAf2JA>(`4_ELW2>Q46cdckB^VHdeYH5ETrBd&u^Z=cp`5yXmEE5 zFz$iAO=`}wQj)Rckx^elLINlIn?Xsa{E7qc&-;XfX~xX?&OTZvdEd`qOBUAY&s~9( zGHYXoCR{CmG5;uvN$3E2zEGo}=fR&=0Fg!EnIVgOOW)Q))A*6#9}ZP*KZvbc{hxMo zTiayBz_1;VGD+&S=`4UbhP(E+ynekbI|Xf)gcmv8yC!$s$RB_!+)-{@mq1IC-EP6) z)dfuZ892!ki*TJ8jrpiYTEBGPU9SYm^HEO!75}zUj$OhNS!&73O2o%+fGy2LYr#Y6 zgg1{)sG}uFHhAX*>Q0--Aq8cjX~>HF{lW{$_+cQeublsoxT0!!O2OX851ht5bK1;*BsSAA|fKH0VayT zaSM<8zZqR1hZLRzl2VT}*9JEcfKW&lmRcOTC-cK0tP7b9bsd-h@*3}}NlOoWpc(nM z7zccn)JjFOM%m%d!dQyrzDW+*XRRzM)qCxYU);I}Kn7o6vf)qdVZ$8~)G!%z_4{vK~x_OV|bBYs%s)(?NSH_k2<8jR3qNb z4eajb&h-Qc*-7$9(~1e6hK`O0iaa=%SbktjR0&IN1lYJ$SIPtEDfrqY95(D7mAsc& zV$Z)O)JzzFf+%j8fKs&KBQcHpptjdsJYX8>F0ozI))g^Csvp`sYX>TIR|R2Ll3rOM z#KBka4%Z`3C()ujgRj4&^GI=HAv)ZgX#XB*nZNf(LCSj#kQ7RCYFHtJ-|D?jm-!BK zf_D|8$bi0@7(XM?Qg^sLu%Z_ki%2dkup%S^#ev+)$hI+8eN#Fm7t1=hSH-91=W7%93i9q)iG~h3DBVB0CllgF z1(~dq`qtNfPOspk>u({dicwx5%>e;kl{bn|KAgr)mcD%*{eNIFQ_0tXl09^ufaG7| z_%NloZ=byynbu1JzOt>C3CZ03%Ys4UwDjWi!YSENo?eW_GUTkRthOTqp1Wn1SIpi< z-Sy?wrly9wjk1|1M%vHoDkmj4*MH`c@NisxmH||VS@czsKgM;%Bgmz$+?D}v>j@c@ zEogR(x%G-wiy6nS#)oKu{JKAdQ=TlUl*f0AL7g0w$-VTFQaRs+C^M;3&>a=R z9Nm!penr4``#@*#^!`C7%k^xcNxlN99Sug~UNlAnqJ6A*q0IW$g<8Wc8$2YcY(#(E@SHlT6!k}9 zSJRmt84P7h|%USyofs>9V!T$Z8um$ShgXSh`_kE0;?TNgt6lDHcB(qhX< zgbSZL0C#C+LymJehx3p&(6u)LrB&fund;2w)bBIrpI(zuM4IjjeXp*m`8)jEYz^8R zn*CfjP6)7pP7$8J=3bzybnERri11G~zUR}u8XehwxCW&6w-CJ|A>^za_+*7PfWCoX z{NHN}d&kK1f7t2R+aF2}hX(@wdUAz-djbA``=T+XRxzjB#Ry3O8_NpsJm>S{-389g z-z6RAmNUMKuT|jI$wK3=pR0ul(}S$*nX%~Bm5@J&)W@3^H5ISQ!8`8bH=?-Y`M4~o zX_4U_sf-8KJX%Bj6>IutIy?Zfk>XoZnd>LKtf zOz^P2LxOuHPZnL4>&%N9Aa(b9rXK%FlO}a24a9}xgS?8By3pX~IXM*2QZ-i9aB~tU zBZ)DG^B>GPz%`0k$GwTFye-yH5oM~Rjfph%@8wD;VF{uUk)^={N??gpFJ@ujJiftU z`#Oid&aHG}{{$&Jxz^~t6kMUC{Oap(XMAI=qTxP&Ao`_kR`ZpB_%sY;NYAi0fPsFE zWX!;-WkN?&Qy}SGhnson)I$Afn%y@)VJUC^dH!CVSza5<1`6Mr(&DFXFP}~{a|hNZ zN+q3BFP5y0(QQ-BOr$)QNqT(_BFM}(VXQ+nGS-i074Qu5AJe5LxF~dwiGj@faUKmCl>`^ z{MjI`2Al4LY&&A$HUfS$2K{>1yF>xXRdI86(SMK?Q;B3Q->0a@zoAGElSq}9pfr6;Z+*XmMQHpUXqpWSh`(mj-9PlCkiV}C zR@ViUwUaAIBYl;6THe)7^*kzVH$0O=`(}7HX&Cn8R9+k^?bE?5)sJe{+4%}7E>qCd z6O`UzxF?KDj?QB}+GGMsfKYvjG_}ZaLA)9A=}eb&IKA_4bdxvYbogGYNZ9|QePK$# z5s814t1F*`gk3zexBP1D*ZdE2jo9c;^$~@dGHGPVKEf8ma-k|G$3!T5O)NEJbbI z*Dh&dK~Ik|^i^%PYyVY)y#zUagyA-zxHV+H3uT2qCQU@%C;T&lLCh^AoRChtQ*E82 zE5>(tZ}>pnDM}qLVBCBkBG=@9q>5YD88ZN`rYf5hFqpZh3B#*!%c%+OZ#FmEAM^o<)QW8AB zkTdcj;Rr9I9S6l~k|21IB0z1Uj4^8yy(T0w7O5_stUhp8A6oF&7N&mrWo^f`E{PLT zp==**J-wVIA^q>QL8P$B?2kLx`hbP5X>wOQb&EA5KKZQ8z$OXUWpNF70>MX{@tlDE zIVMG0kHO567uX`V>~zQ%*F9-vRlO9ZoRE+J(V}NANg!>tEX&W&$1zz)4UGoc^69_v zHDmulTpxJZci&ZbdU+XP!(FjRdOPFDxL2t#Q5&1GeLL+cD4y`bPTp_W6Iva;>RkfS z<8m?m5(Tz*eBs>VCTrO17W+IKWOiP)<7sa4vFk-Sr%elVaT11W7s$>>rH>5@LcS&0 zGMIXa{;TNFmjfm8(H^N8Im;@o7Juh&UnbFz@{|m%iEoUe%f+c@mWt@nDShyRSpasr z8B!PottwWR{(;^qSusLnDyjf42@Yd7j85Q2!c|$68yF2zndDQ%O|GI{T#7?1>4ANB z^QF()Kh1N0JzF=!q$Sd*cNxVOpl@o^3q$j#L;VCV=Rw@6`5!MB3I^c7GPJQc@@jE$ zWYx~;oicxJrm!&+_UvA4zgo!o$K?Y!#vAh;%Cy@>RiH-uY;m3~8F0Ul@_tL9S~e#7 zUO_?Ocrv3^P0RC}DY@{X5|jk?(-gf*d>v4XJPLBn8|J+^QVpyuALXHxO}_+^dN=BS z@x;?CV9Mv_H3;6@4{i;o(bzgWzX6jKy~tSC10@k?u~NNS8Bt1m?15G*;?>Z?#bUNP zVj2uUlx(tUlt5|y_64w7=4S?5u_*E_Cgxn9=5%B4Fx2MQK=0Xp^TUy%hOh&sb?NF$ zysKx~oll?_PkeV>JI3YfEp#BAz8?@VWqJf2k5*RZ0NTArDelg=a+rlL68se?`|{ea zVk_rr;I`X1CKHF>dID-}Y&^BPOoYnwgucaX`xJ!~rd>A}hZko;&P!a)7XBVgm8I+) z{DlbWP2l_d4a@oJhMp>cc42hvx|>6FP6^|ENc6iRQeqiSE^&@`RM@ z)$turtg@mxP$2BFse&ZET!NP*^I%ToyN~PD!00h>?Tgmz zS{IQcTI0t4o@?vw+54r*uEk2X@p{L0`+C|zrh_rFQ}@Q;Q}?VydP%8xV-F7fPu4j` z&+qbFh6f9#LCpgMNF!64=uCn2r~X7x9MBUb>;PBz**v-JalUWXYu?l*AVOC z<{wnGQ$+0w+i!p4eWAJ2z}AuH#b#Ctwl`pX&pTRR!>btsCPK$W7gO+qB_w=QqoKn8 zUR&DQD)gf62}#LMQJk@PWwFsxaVi@fIV8_3y=P1vYVQ)@7U|&gx`1Cm`9a&y@4)b%-EM!IhuLQ(l;rQ@y2f_nl#N=hd;XbVqFn@oy>jdPlOf{b6r?Mum5W|N9FJnZ z&s@EtGgK>ha~*m9Qd?r_@QzUya%J@-t!0{Zy)YGn&7%Pxl?jE7jy2HfCM9iG|(Dv9pL{ThoL zK{N(b8vnM##B;OtW|J@%|JqhzTO}bZyF%h0WB-P_Nab3*z?c%Q&epe2>EYkez{`FR z78Cyb^z@{(?AD2lg`o)A0gN*_#RxO9ei0h1e-Be4no#p&X~{bg^TGIwzV%Zos{&u(M&22+_7sd3Q5L}%8yGVKA&93r%z@dqBvDo@Jg zUT|NuIXk=V{kk7z(+rnuM@_ZHx}7qX6FH#aBLvOQHU?B{j5_b}K-T(O&bxOGvY%zM zH&A8fxZeIsWmUHKu3*TNzY(lk_35xi$V*tS4u?C_HyG-$hCKfPkiQu4WrWA}B(Tw7 z|KJ&tMf7QNvqV7nKW?r+b!a@2SdEthw--cX`Y(eXcXdpue_Z#*6AIwN_IaGUo;g(% z$}4CuTwZqi!aiYENa7mxZBYa7gj(%IFM&yxA|tp=b+ ze?@8aBX`icgtkBhAwDSLRpcMo8B4zW{lak?kX6;>HMXyh!k8aj#4-{c#!WEq#8j=V zYiO8|FFt+PW)JMb_yj1BsjWAr{bTs` z@RE^VwxY_q{;J%>$ah73b32&Gvnf)_pGZSh5%|HvCih)2lZm6P?Vr|4d7+LWMO#}9 zaLD54RxHOx?wi%kD8Cm%#U3Y4-om*%Pu5OeX{*yK7kRq{eor@|lg|(eg`Dy`Di_BB z4QG32=b$wh3?@&C8S2W3o49$zHZ`%tW=Tf4pf>ipIDL`uR{LrU(x5x`?dkqdSS-=~ z1TA@RqzY{33$~K;^wN*?oIX#`SJWGOi1a+;=nwL;8L!G$*q3u@^XQB#{auEO82Ei$ zyJq8e-ecJ*J{XHwe*0TXxpZ*rZOL}pZo^wqH}FGCvec&Li=b^w|%w@s1jleDw~y zhHtySg&)QaoG?h8g|iE10s}x^0Rhd^=iiHqv5reNH3ZCQ!&fk!3xN*aY^7EqVPS%d zP@9lkMO~%liAD~kX($vJ-5jlSwAttb*q-S}Qs*CM@1z}N!xxZM0t?R#wqG*2MiOy! z6=*BeP*YH_9iac)a%KA}ZCL+jKd_jm>Z(LtLE=bcg8O{_J6DFnN<`4`icf-x0nX3! z?NK@Wmtq3i@Y$7S^@Ed)#^Ir%A-ZV+xwDVPudGdqppmtMh-yYYe*bH_`Y^OK<5|@3eD;ciE1a@EGR3aUdFf@8yCe}tl z&z1)90=gm?jk@fwND>_b2x_`<)cvT*SFy+L}*Heg7NNIIA=%Z^)d}Y3LJllH#t_78S zw6<2u{#54v{kh8quHDHd5QTj&FE5|?^xEh$ z?T3TVgC*@638HR&eH~@SE*e0>y;Ytr#M>bkgIQ&4k<{hG~96-MIn!qq}%Nk0GaZHJhcOzCee_Cycw0EElU zc#@d*wPkY`carflMnm^}yU)cwpO-hTj(|Cyi$@}NS`&=GHIAJo@6di$V0<^k3f{n$ zCDXe8d0~i>jj+hlFps7F@_>{>F>7q`56b|Ppea?&WaTO|{k+%JXyByW#$W;Ei{OO?nkYRV|RCprF*n3~|w@c&*?GFaK(l+Q~36 zh|}+vG=$$Ru{mbayY+jacWJe_4@S}tu1)DpX6JQXUN6oW+y+#baID<+pWx}Yl=#eA zrzM3wHOyX8zT2C(@t#`%qJ2)TqeI2O=dJ_`7f}-V2Gfo5&TbDA*K4;A`(ufNEOqN1KhBixtS8E%Z{{T*|&G}DZnIt9BZFX+5JuJSdip@ zy$StXA|)B`8Qo?icJUU(nCOFqa5txc5aUmIkDK|6C9rkqbr-NdM`K13hKGlj6dPUm z*I)%$q~y>(lrKR(M^Bjavydd^g{1298XOMi&uEPib_(PV@MI)ffZS?(PC)*Jj_jAB z{~T+s7&uG>o*Le?IJ{j$;M+xT(evqo)b$Qegs)$(e%z@E3w%Ep*6x#2h#aUGX!~g9 z%2{h>I@2dG9&ek|&98hRR`l|gsn)=+(gjo4xT%l9t9y|1ouPmk?@bhXt>3x)Q+*(g z?N2&OQ45FMNSA0uyIykoU9M0n6AIurE7PRlASSq_DtI8n?ipdyy8~CHa_9GQ9<{%i zcOB(yi%)<1+kSf_0}mLrcJr_K%;aqq!~p}uA2Z9#KhTEeDrEot0p)psXqP0^wX@!p zgm7hfd6{7Oc=NIXYEgR}rMg~~xqUd<4e$Vd9H%u0`TS6e&e`uXyZhFFvA8h;B;dGw z&UU)Z;O&8U*Vn1nF00^9GbhD8^3>BfBG+^tP$#j9RNv2FOmh`I&$m3Z;^(53#9)*w zZy?yCcV2x5X-D;9I-R`Rzyan=55WPjfEefo3hb&~ErMVZ+s^;jps zD#=>CL&Lx!ZKgx}@kn*M4#%Z8XRZhT;>vI0^t>)GqI-oTx6PTmn`OXX#$0tx5`LZ8 zHFZbVhaZG71IVt2^*(L|24Zl`*=_6^m|iO zww7=4kD1%`1Mqp%Yin<2wmeThmE>TkzMgu^5!r7ByP(?NgV!)!*{J=QV{xdFx0Uws z80i^5)YH_D;s`C>tdRc*S*|kV3mY4!)%q6nBGWu3Sf*H_mxQPXOm}EA$Qk8g$Z5EQ zUM@e2&~(OLAYK4kcA%0V)=H}ayD`)EZ=O}L3jOdroL$Bs;7>;#wGCTb)I&!{j}3%_ zX{CZeNXV#J6`$@Bb8BZyy5KgF%)QvlKPv7&R&-*WBa2M3a0ZPh>Z@|EeHRQ!$=&H| zPieS$5V`4YR|u~|r`9s+4^D!X{x>}AE(%5vnKGbarX+{ZZN%D%;75O8h~>`~s7uIa zw6v7o1-E}>*GEck>z@gp#gAl>8A1Gv|7+rh=X|8Qo$>~-|A6B%#B;PrJv=;kgoK2` zJcz5Sgs*%TEHlSxg=XSy1V+~TqSk)z$~f}mejue7z#h{6|Hot2OVvNyw;58netG3+ z-Sk;2^Lu&Q%^0$vEM+Y?x$t#AqMfidEd(QjMvSzyySs3*^JUrQolSZ&Z_K*m6+iSV zFK({_AyHVvC2C#4>)~vN9Pj^1Ble>+=$t&prji>R9aP61kBAZWRmXar_HO`(d9u$ d8i{&FQFMD0ks_fs02UB~L=D;voe5)85?0gwDg>&H*CiAxeLKhY;{@^koiuy7OCHY(?pHlvL>?9i7eT zc-eW_IqAjl=;-J~oXsqR9zT-)H5~X&l-|n4<(Uu%hr7EwyE`|#qq8N)13^JS4o)r( zE-p6U4mOCVgNum=n*-#|#U#Jyd1MYTb+&%yV(sWaho0BusiUilC_O#;L4W@ITBrT9 zKM!(%{Mr?;KMoI*XB-dMIXV8XK`z!7f0y^4QRq+ppTo>Otp6Xw(4SlkI~T^q8btt! z3Mo07SzCBMGI22%u0>^+1SIK!}G=gyVni`MbmO(UQ*QCN7T78jg;3 zVyf02=63ezU+K8nc`ru(`}sdZ(Yp{*ake%G#ETZ9*aH!c|KGL$yJrY<)rh-+TPsZd@uZ5+#($R=O_O=Qqs}R(OLbOiK#hSVCO^rx%9txpFc}h7>h6^|&2?0m7lX_y*PmRTdPE;b3ABgPW5+9%Mba(& zA>n20LL+*R>XM>yfkCq*d$$qZoAdrsJ@aS$+y1#_+y0g?yR2RwVPlI@*ZDKYnEmu^ zTOGg9gR?H$sl+p3Q<;Sucbwn8aBYoG)R~#Qw#v2J;(XQGG$@%{MVLz5#;%-SKAYB6 zZGJ|8cl+nJ)AtQTi0Xrz&--s-Vns238y}zE42e8=gj>;9!tPq& z+InSf{^|U2Uz;4AsFpOl#Q3e%Qt_@MPuzhUQ6+1!TcRxU-a}{iZ%?1q^K&>r_;?1HkLJ1)&phMIyItWI0|ZCv#}q zP{`^eaoqNV9!m%q7vYE-x5nf(IQ8g;56!ZXQok9iNW;QffRrSRVP6&mnX#h zcW9tOWDyxtGsx$c=X*a!3QImjE|YXF#3O$Q$_=td`8k;Dy@`!j--wRzO$}LZFm0U? zuC72*yj;sj^=b_Qmtx@zb_k`uy$pSXTYkUsglX{GGp`s*q8_eP)YY45SeB4SA1-%3 zVZ;+*aNLOY9N#~Tpw3=zu&`Tegi^yrACz1ZzXMp)Fg-Y-D2`h~oH)l6(;qo2c{CVe zz~UN+wKN!td_R8peJ$HU18Mu^5QaSN@oE)iY1%v7A2@4how#R4v-Qzfs%6aM%h&$b zZx(#-=R3;Qgv?zd?ZTr{$ZX*Wld{4-hS<=ygh6-U)}MjXuLJD;1)eA^igHUm#lH=#~T@T zKAP8bdB0FIoCx)HD8*+x~+}`ecE~27U)J0SMDywT^KC=;m?2Bf* zQL6j;y`mQJMtKM-Gk=sXZmo=II;WpUpr-Cu~uI5tMzn!b~o-cx$_N8Gf2 zUxRUH2CN?KG2_<&dkf>1kw|mEUUGNq3)zE}eTdOzzin581vh{GBR&xpGCJ@4O`^uD zUZ$v*Pzb))PW$|Yv#Up6JtKEQ#xs$@YB3+8&~}0pr=*%sgL`iO5Xm@=g4cHR*=T7BoImXaR$cBJX7Tf*sx4P`{ zSA0E;?o*ak1P3-f7GW~63Gc3=j>&IN9KJ(OwI11V?FC7mPr?!}K4?2WC!LDHu8rO3 zyS8>@{xtrAML%lWQ-HXVDD{=Yfn<9gy<3paxP6@KAzxQ{LXWR|%2A!J5FgsTgox=B zM?c9~yvFHH3MdN^@=|`2tb5x(;eLFO*TH7}v!yWpMoW~;v9I%@Q+70}jH^wkjO3-Y z;`-gYJ>@=&dx*0EbLqisb1k#TVV#2R==dg|2HCJ@R*PXI6VcObsqcH7>w88X!4fG+ zoIe1F6bwk3BK;8+7O?=+w^Fl{4;qp#Py=&=JD)tWvh8GN9-MIbeiutL^(~HF92{VG zoo%k%@T0+T-i*2XGpItXYoKT(Mk#f}Y+N$&5*wzzgDh{BreDv(zNUrE>toy2!yO%; z)#g3kI+B;2s(^3yH$(#*0edrF(l>EUVE*kS;iSLSe25u&_+Z=Ub^g+o5J5vR zj<)V^Oqb1?s@g5L;(S|d?c+z^LHuMnV>*Qb zLnc;+B-W$i9eJu!yceiQMf`C)(=uHei3U?`#EP+|*R3^LKJL8Qa>t=Uak?5@ zgmaYN$C0zfSva-1SlLc1;7pn@X%|j2)9>CTC8mmenD!K8PcbvKM&oF7>n5xLHQdQ_ z*WKqUA!lQ{aXYzljm^4)y+bLL4rbQp+BHHbB^h8S-{sP~a5NsKx$5qR(8Jkd{8^g7 zdsPqpH0{0;pk6Fujw1|71o*PsrYx4ywC3KHggaxk!VeUx-P~$?*tOf?+cD02q+`-r zc#(ZI52{*6W(JxvS1qA#WOo9<&w;3gD^_!r0NMfSt-QakOCz^qKN zR2sK>J>RqMXgp8r@fjbQN2*g^eNpCA@ zP9S$+om0Ye`JPGl{2>Kjf^QRbZin$_*dGQ$*=#q8>~DL8V}R5Fv8YRVgo8uz%3bY* z2CGZD-iF=55mLmvNaFDGySxI2% zm7?fW3Z-Z_Rz9)0J$O7*?)^5zk+ago&=}E~I+TP%`}ctVqz`H2XKmNZ{MPTa$y4;;p|RyX?*9ga^e+xWi|-E z^OoLRz=KH!;wh32@}Gtc?x(M_@!|?Q0$uNYeVeVD2ba@@DuD{*!E_Ace&-n znf9{~6@P&#+p0BFDKiK{2Tn-2zFE}SWB{LQ?uV8LxzMQ;B9ok zp`GTmI~@?J!L#q7jTe53>S-`!mz;efolbR#FhX@X@3*)ClQ)7_$6+asl=BLUm~y4H z%BZuvyR2GgL%QDUI%TNVCn}@!i~FzHu7T4A07#GzDHX@+B(u_;XrK?Cf|(%|7DjKI zZ!EQKXVy7r#QmV0n&8!05*xdUiDdzP;^)18EUjDg_%vz_;qD=q7&OK)IenvTZvtv> z=pI``#3zXX#ls`cVtw6FM9iewEFQ`>ZD+)x{eG5dGR~Fz{V`*s_DF!yJ?%n=pNRDp z)(WC>G~!ZNt^ah0p52e{)EkNKAYbL~T>EsgeRMjj-m)a_w*GTM|6&DM%zzRZ>0ozM zYk?><;jPu?%>CYOm3DPn<4|mkCNTsx>|!>uK3=~99jLOz$0E+T4#n=`Pl*aX8MZmn zHZQs+PH&1by35bDF)>jiv3CcR1n|A?I6#G!8G_2w_-K}vqoKaiv(2{8|GfY4WgO!K z^RyCNP{ke9H%(SBOw`jeXRo`yJjI z{Z{KTKgYy{h`~HyD5aFY^}4C!r*P!`Hx;(Xcftx&aZ5OK5p+< zgof7fP2ciOUM!X$P8S7rT;J@n$O@bvm2N!@;*W(4J`JDRJa{v$U+Z9D zK$o)Bo_-RRZ*D(&dY1Ns4A|8Li(;r=8|ln(rt*#W2|GKJ*ShcSkyK?0f^df3H-88t zP;8t4^PsqL3W$)ZPRvE`!Wsz)z&d%tWW-9yV# ze@>26>^>0`M<;88WLEkU)9m?kC}xY|7;)GK61V-g2Dzts1-d8JBY7MWp;k$ciEhTb_qS1 zx~UedLxRW~N{-EE^^RsozR;R?>h|04JOa;Q0MUnuiH@JVQ76mHM|DZk*x=*V!sYOG zvIvguvKHfsaqmy2(fJ-hKw8-DZkAqM@R+9@9_x13J6e$%Wm%)Ul`>G7 z&I{VT2P6tjklxn{#8&!^TRhQxv=L2ijq-e~i~C(?@<%G~(t{d55r{Da3NP{%u8mllTpW7kxz?xrmD?(wXzXgFRs8iMqbKx z(PqayQxQ&F5$Y#83iy51#~lq{|G*ofpy7Mp?Yagw@XJ`|d**T#dhwfZ|=Xc5I%I$T4IuZ%Lgph7UYK=VKRE) z;F&Wt8~d?f#;-922Vn$?7(2tbgip+sddGp7G8gN;qK{D`ryKCUsD z)z2BZ9SEo z9Vtq=JIToh;G55764qy(QA+p&YKO+=MmwqGg6-wUfd{opUun(8v+2KD1!90)86tGW zc&JaxuvSJpn?v-(G_pEQELoIK<}XcZZ{)JxzpT#>1i@Ot;a%=;gFn;P~-LtL5Qd?vlmPk7Q9pl~F<3 z7L^2mN}Aj&Zw#8Zv-R#MQFD#>zxgonyqiMRgb$Vhx=|= zii?@S#7|Wbr6=YzaXrKd+?bztlKLUvow43&&K9VAWgciTr& zQqJ{tDY5z+Eu?Hjx}AL+t+P>&T(<_a59rGTS{W9~Y95~&Y0SayFuEaZYT7?}8L{D& zQN36G@QDL}>yXvH{tz;&)t`+IQw$Dpt^HiC!H!3C_r+2gHD8h`p%Z`IYXX~OdPf}2 z)o?PW_O`{(9d(CZbqmbF^~5<2K-t1IHMJIzthj6y%}hPo9urnX&nf*DlFILJ@WOYJ z-K+HV9*+bD5Ev4qpmATTXvuk{&&&#ZVYezb=R1_GX@Vw7XYEef54MX5L1bb@4a{Nb?u2&wHFeMQ_09{8-I?I70VVu&rqhh$kom7t zJElzr8oJ(dB1gF?DMcmdcnH-c2Ox)q{|c*gTA^=?!b0Awv+RbaOQXBv0)DF-_O=cj z7PIB&>&u~mR+>{8v9Nj`pADo}(tEjvAFs}n`|4|-Rlly}k$5r!I9{1_4-^mgU~z-+ zf#j*2o}XZ*M&sW0Cu&HquX8)iHfG%8&U;eSXyH$T<`t4R{F^HFlDFahlF*r~#lAC@ zb$;YbU+M{VMotPZU=#&vkTf3<8#R@-OzSbM+`4C!U4P#cCAl9AFpv%A4{T@+U}D*T zpZoxvrb~+CtT>8;`9s}F$pe4UZ*?}3*X*sIH@^k^#WE&7nSw;X^J}vIHy*x2Io+Tg zf>#srCw{2rM*n>|Z*g;ndLF|^-tW#OHx$*(z@Kql%w;Op`PJd2f%VV(T>{A1$iP;2 zF^^Hfrs7PKfl)JD;;d*_Fy& zu-!>eh*p84)ACPB+~NF?8y$`d(ydjyyG4y9XxXLB0g4TdE^T=TvR7~v8+e5DdF^NT zzH%GWS26CZ_vi*&e$(R|w%zjratDb{wOtFW|uOkf|=iXP9@Yq~?%#)h~ zguD+)YXuQt+qOXSkw?by`8fD^6|sj@15U^L-j#)s`Gn|t6(-h-+LKQMyM0Ph{cy9j z`0F%H)ANeQ6Q?u7`gm2D*q~UnpSMOfkdZ`qKC^EUIrVjpMWtp~&%_F4oZi+w{aILg z+^_V?O+ggcUI5q^r@b_nqN_C5IHo^EZYCNGd~ zLT2Y&I*mg&+DDaFl8*N?HS(78uJ9Fm9{L(qDTbebLDpW`le2`W$gEcg2fBsaR8 zr@xo_totPU;GEgdwl}h#*JFNbF(82ASNX?fEc@>LGgpagmLs7(W&Ye_gS#G|DtW+Z zkANB9JmKNEPv>r&x|($Iu3SXW(bjTLzCnc2ZFnD9nNEUU6b3W^tbf!h9-Fti(An(- zWxNthTRvjQo-aA=looJyM->;Chk919a&kR4!fZCtnP4I8EVj19OA4HwGT>00F-X97C=&6M{dPcy?-DcqO`(s*-Lvy zJB@ny+-h-R80c0Xr`}$D?61xz#x_g1;M^-_xjuR1RaewC8YYB=2e=bHt{N>_1neqM zX|ChwT3j5xLS@ccz?({IUy6u>i3j3UWYS!SZyCp|F;#l1Et%VqwVW3B;n{)2PHAA*pRb#Q(iq%oKId^gD6e?G}u@@!ml5#}R zQTSC1V3Ot}oC5@{rA%K0BA<2v5g>^esOFPB)V$tUbXd=u0thdDV8bm4SlsgOQ-d5oi|^mUw*3&f7ekw*Is z5R}grzDU9d>$HS6DNw>D?y&ZTQZ)HTO{(c50KD=#!y>!kZ|l8p9_ci?sK2Ky_Q-U5 zztnwwV?C-O-SgtWOZ5bVFI;P`WV4T}8<}@|NWaO)$Zn1NM89t2)r4vohRM6W0G9I; zrf(5P3CELPV{Kko;?utQa4=v98=XSgT@(K^QAk{|_`R@ns*LQuu1S4MG1mc+qm+~s z_8N~b_&~+)<#Gp2J;3zQ%(+(jEU2p!L8g*ZnZN|6Lmyh#NDmHNTA1$g0|2p>C2v_DpAj9A{t;Unca{xKsrD zUHJ>dMN>3^-EVdf|$6S#j!BL;>fU|_&#onM;mv%^v~in8>u<4bb4O&1yDrn zrpLSbYdqhn!}qxQ$QqevIH`S7{&z2OP$>nzQ>UbDl>=Tj)MYy8;rxkyZP8QF6(9Ls zwYJ#hLxH|eYQ2cHlu=H*qIj^c+()&n&of+6HV0i-5JK>MK$}3R8L&gJJ$_RCT(7v5 ze7LVIkn8TW6#R*ZF52tVp6@#IB$gz7 z8{B=G6Dpii+%zuK1r#GH)b!hG5b>BN(Z`+9@pg_pPfuM&!-^VkDMciiCFuHs(qA@w zD__Dw=Z<8VA7GjDg`PrZ)x3Moy5fgdJSS9NU99UVwwdTkyx@Ez<0TQHKDwiM7oUD~ zeSW+AJn~Hrd|GVKH7fF&vp9v`wgu}^iniR4%aw&w$=L*&=RSsW*C@?=IqQ{L z%pOJB#KRY{;9VV|N)Fnw5m)UTg2fK9C)V|5DWl(WuDjWJD3R6dVn5$m3UrxLjv(ZE z3_TCwGN99=Z~h#e5&S`c>rL7x`}t=n1QqW^5`i_CrD(fq*soK~uD*?>Vo@xcXr>S< z>%Apfet5cWOG8lk9&`a)LZt0J9G&~7OWs@}dlqM+t4)qAN;z`xFa|}U+@xc-%xDP2 z)*A0?YBLS@T^HOhX)MeyvPev|lsmw^m;fkj%j=xJF4+@}?t;u1x<4*Ep`wvUnUAUl zeRi2sUS7bgU~53hlJ+3O<$RYH-dWCe1U$NbBi0>H9quH%9HQoVj^ zS?)Y9CuC7XN+K?Dvy~w!-GhO7ltATa^Z>Ve^aPza&+>gay941@?i<`-&aKNRb|1F? zqaL8Vj5Q&|m%<@+`h57BdLCna#ePl6PnGkkBkrr?1Uev+Jy%}(^8UQ`oKS8&u<`QE zqPm|S6Lp(^spaH5@9Y>PP{KKfuV7+yRr_WxQxv}TQiezmEg$1T?1RH+h2k&eFiPB!Mysa_Bo)h=*76ZZh2 zd3LhWWQnx%%sEThn;iwjrfCZ5PAn=Ry}q(>K)qrM*U#J7Zi$-t5%qq-(OCib)U7%g@?9 zSJP&MYoCp`HpMqh8^P|$66u;6OGOmztKiMcoCgCX=qa1w?A~mXbNB7t>O}k}8xvay zrwZ&>yy)}iNYYGKZluREOEFg0l_t*i@-W|$2SD+6axuPr{=$nfKxItdoBU7qKc0yZ z-qK<>g7s}5Yy~KXlI7nnkUlgTosM}mCw?De`zPZ2Dz4>}6go@A(pNoqJwPPf>;Tg< zNi@i8)FpJ$NO|G=h$d}sMr2n*GZ$dI@+yke`n~LSh|{-IV|0VU&x>Wu33nBaDvM5? zK4Oz!AOIbwEeLQ?BYH@NKhqTvJ{Q%R=O;DO`gNxgr=N63NWW{c0-mfVsGjIo6gnZ) z+aT>%Pp?u%8#zflx?^ut!Tkrf!B66Vq}owL$wAnFBgoe%Mbdb5)VmE9s>{m#$Yc09 zu%hd|Q9C7NWpzKaC6{bV8>jP;E+`4hwUyJEqDH`y9-dQE45%Opy1HG~Y9G;(dTZsm z>)zzOUheq)9hVLgulhVgpnAM+DSJ42QCLMfL`+^i<56)xn$z&wMVE;a{=gd@XWK<# zL_XUm%>m!-Ya#+BextS=5f>8p!agSl53pR#q`+tQj>(sU0;l}1;|vG`B=X&-x;h$M z#2d;B_e0mjQI_uF#qU3%m}#@IkM2SbGd%bDsvlC5 ze?*uzNrq(LFRNAc1)X|N`ulgeABR0bEEs2S3pN)p6k|s&H9n8#-iOnDyJUIN6NtSe z>lhX^n2$hLWt)|m&iMvFiCM@uXJ=2n4Qi_dhon+3wNlVVxYm1UW&+}5Q3-kAhZ0`a zdp1mnqOSPy%{LuRgl04>)S&6#!UO?NpYl&VLhOnt6}K-v7g5T2#|mVT1|4w0B7ZuC zM%RoJ$LfBIqf&2#sQAvKOcC#FykEeRC)a#F(g!*Lf4+8UGB0;^M+#s{ntKo0#UAs_ zMicPUmOAb%z7RE()EsfGeeui3&G8Npu4gCguq)X)6V@kOwC@I{CE7*n#gV$we><0MlbB8mL~+K~RT z!Tbvx>?%rHEqhmgleWN70m~-#2SPmSSt=uuVnK@to7MUO{R@S6lD8L0`exe|i;sOx z9MjvMKqHLi#)GjUF3|f;t;B}1_~#(|F22rVCU4&qx|1~j(%TZXbPNO+NGba^;B01p z_KYj-%CLkvd-M2e8|3B0^Ut~@-#d5iOV|eKR=b>dvIjdv)mju9b<#MsrVP$b4pZc0 z-}ohWlKbB#`ll^Ik=i+_6gQSWhR%n$JL*Hc(^BScUI!+*dMU`Q=>qNjRaPV=4%HHO zN&@MqNiu*=0ALZ&1$;B-fCbE();}@rd&E=@3?h9(S8m5wANH}XOR=6JJ&6k?0dS7ytN&1ht%KxvL9EL??`ALF4yt57n91Hf-Ue+ zA%=I+CZt)GZyEsj`1A@j7ls@pIKk zI)bfVq0db&6RdB%H`zDO^|nr%=@1%>M{KNb4~qZoLX*HurAVqMj=sFTd3|?xAV|79 zeuxr}l^sBH0qYII$qD{OZTUsqphJZval_%J3fp8O1?G#ehC|1xo93@DTIK7PU?B7( z8RFQz6b?js0a8%G(gV^w(!KH8f}`P)8b|p^8@5+xm38jJvx;LcX@5x_1Ns3AU|Gkd z1}UejU{h3(;}M0J$CVZ4q5ur-EY^S<6H>rI0Xa!V6U(kmMBJRdTWa~fqvWLe@3@My zKzgI^+Fl%(%5$#zD6j7hAQDzOVjNHMF-S7vgL%iHRIbJltG~VoBiKm=QYLC?sWLkF zS8f>4=b&s?H(D`~Bv1Ldkcw84JsO4V`7r3Zt(*03jFJlAexl;8t5P9#;UTaotz0`z zsvAgEGlQd|nc*h9zskcQWBZt6+VZBPM9e#hY%E8dvShu^&1IcnNZlX{pmd!t@$LDV z%Dj!H%o;g%-_xCF2!)-!{A;|JSn}kXHJ{=5;~Tlnm4#xd2T3G3+0}{wRwq6?wCLr# z3R^YLd`DPlbb9lNVkgIVnSD=#o~Ze+coqPz0gd}O$?-Wd)j3g*Cw=b5lFFW<|LjuZ z-022nUE+)q6C4>ujZbY)E76?IhT+xkk-PDqRD)i_lMVSUW5T%#y8szgg8T0{89CDkPssU>s*k*5{IUu^=xH% z-Si`eXboQG;ZxM3)x%1+2I;uZ(S01}9ffFTeUb$v!nTOhxcj}{y5D8*Fjlex`49?r z-BU>t$dm`0kuY|h+S3|9^LNBoBqkDimWTq4R3cIsd0@Y}3@*qSlOtOEZqUzS##Cc2 zLFwY`a#>*Mk*a!Sy}cWO8f2)pfYh8}1zlrFAwSPqHVOj3vVKJv;F$VRGn^<@(hTkk zy_ED5^0to~FpChN&xQg@X0!e6<5Bfc;ng8=8X$c*KQjb?;E%}O;vL+YL%zPXu+LjdJ7?l(7Ztss?aUXY{iqg;J`Ib~Q!aAF|1`ds6*=tD?0a^xs88?tbK>>0`D4 zBLegjdIaw$7hKBC1j;H>9CqkZM#1PowpZaeZDvnCP5o-p?7uQz(rAX0rD|u89u0@d z_I4~#S42Hj3TX1QVv*O?Z)-)GPn`CUWbHNud;v1OgX7iCEV4h%v6>8YL`mL-1>7gR z73#fTqTJcH!x^<#nUn6OU1^b1^5AZ2@NPUVoQHj)vtOgY8Sqj5P1&`uS|K4P&nJv) zp8bS`_letxzKRCoceH@>Y&SalebVFKp~uy>Fk&IkN6YK{B^E05I}Bhz^T^tlD0Xq* zi0vO7&qGu--5P_1q!QSlaZv_M$1eu9a9`@7xoK}zZPpf6CjGCNG?~96L;4;J)X3}* z)^fK&B8(K4tATT#zI*Qr8;(M^`O8Lal@?OPTj$1%wnK#bPAari(C3WjwxPNX5^FmU znTLdqXs5pLp&3|Vq5|@+D`L3l>RfrNFQclx+3Ak(l}guX@`CJ(6cSKw4klDFCDMR& zC9h9XjN;YGYC{rdp%3Uf$ElhrS!UBsFU}CVZ6(joKj1M+%)iJHP$XBTgV{Fbx=z7w zr-t~54FJR5qKBned2L^M>=XBxL29qjQhKjJt~eciP>rs{66YvEw|qgor4)6~#@X>pvk~ zWV;_X?7lEG?1ZVEbuY^x`j?~O+*N)ef~cm&y6oo6$AO)< zTAaX{ED8tC$c*o4r*Po=VELI+Ql2$|ssA`N(1zOz>2n;CkTdFBV5}%T#Y^O7gZo^Z z(AUcNi99^Q5rDI=%+?-WFAO_BP(zD`?hUALor=ADWaie5Np&?AzpbLWk*ioMt62%G zgYi=l!GpQQT{i)~P_cx>TCPQ%G=qNyKRVX$$nZW1%!oT&C^=`1v_^;ERWZe?la(?5 zsJ2(`SiJ{*(y+C3tB846uUcD9)u59qCRbNAjVkS|$+#JxjDpR% zY2;RA<^#dj=Ptm@V&iYw@qAwg?@GRu+ z$dF+IjZ;r*3@OTB-K;Yy-hXwncI8Nq`fn6%z&}V@Kt68%Q8S)Io}+BxWcX@XYJJ)^ z$<;qISswL7 zkgqMm?0$^7_ue`ELAO}qP0tQE<9K{6&hwXDwmutqUrKCDXkAIOh#s`i#*hw>2rwJ3 z&DPpL(!!qp(jq`{aY5r#J7007Nd4_PmWtk(zd5O4|Ln3l5E^1Gu?3jSoMdj!^2$F} zjR6V<^NEXQXml{x?h~dqmDgs(GKeXkMuDzjew4TiE-xwAEHi9aK zMTwAG?abK`;%nSX7w9*stw^uy5!m-z*XS_J39qr*K~xBJ{sL-1MZu(-o}F-(ii(Ag ziZtr1(P{}lIoXIVHAyQ+`|;UBS&Nfu6=Kn`UK=s975gA(>yHKLYuFh)P#2IPgbvi!X7oABY zeoa8MZ%pHO^q(MriPcHmzLek0Ej)Fv6{&kuxM*96g^GIrQA69y z<^OJ_lXMf+2`^|Kk+DSTrq#NIkWyrsM;)=g<@L`>A+dGaN!|IGJuHOxoBCp6Wido_ z=tflyCCiF+E$~#7y5WA8gOE?7+jbHpb*dqAmF6Fa!hm+-9MmsD;%oUktP|_Av_>pU zo&r__wEuBP98N1l(!b(e8~C5?v#>^V7>=CSKPe8ffas^yRtA$^%hDI-yANrtm(H%@U!SR5a%%{C z{MVKfiGNO(9tO3J!ODG)rk0Q}Ep1zKT%5sPX~siE48OZ!>EB2_K`_97WU9A)bo^T*F{x(fpOOvJ*iWwuZ~ zaDl?2xK0}(#Z;g}%)_il2K2qjb?cM}d*yfG0YLNwZ0Jt=SW~-+Up{pwTp$%*UaQBP zzsxBrvNpr&pO2CxnAF{GY_z8k%^g-d?;bdhRGh+~OTxY0pW$5VR$^@^k?i4NdEK7C zW%W2aJ-T-7%V~(^S!cNGf2_gD0Ss`7In_3?B2+zo@xXTe zKb~C1647BE)iL#^lLaJ+d zvdtpMq1&=K+M~o*u4ts}jrZ5@EsUGtys4ArcaDEbL8cq@Nbf4wz`b6F^W7~y{O)lSAOg}mDM}96c-Fmudr&_zG2>fN zv6Nqv4rvRAi$Yu-3|-@f6cZc_-6kP+YcpA*E{{*$Anvn3?@WHQK!1Z*=alO%f>iY3 zd5{@`HM5E@t=PA}b>G82WT;|W`-#$DXC4`;sgm0`asisChOabq`#BWa}<#Zabs1^%hE6M zvvIr)Hq@_vgDnO~lR2Wp@qOm@-Fx=Df=>W@ZxoUk9bUCYwE9F)&;{Fr8&bsF9bK zos-wH^MQO-f5DN}@msYS_2*&&JL58i{PaJG1~`om%O?d{qr6(!2Nff-aUFiFHEc4G zneHFQAXL71ByA5gRbPXT}qM<$Qhre5)7A6m(KUtrVXBAKL{c_))D^ouF;ZseT2e(>K% z*nsI^2MG@~#fr1O2{BNi~Y_lF^;nhV6JS>de3Ds9@quL9FrW&HA$M zpj7i=tEIpQK0aEW5i8$G@rj7TO@nkNpTCmo#5aS?oF<7L86@y&)&#?@>p%|0Vc7Q6 zu-=%O2gMrU%iZyQPVky0tyQK+7xi8=qqc@B_D0ud6MlZDjY`TH=`k@Ger0XiD#k>^ zb;WnPOQ@_*DxDT)&?)@)Np}HLheJ}i7(0RzlX5hA4AnFNh61#H$%gqQw*DwM@j>_d z<^PLJ5mzx&_kA`7WlW9vQYdyOYM2W1IM)?jG%y~&)Y!%@KKz*vUIQ`v2=nCxKt`tMQsgJ{yR?Kbb;&R^S0*wB^A+fFw-^J992p?&_pToD$%l?i94qe7US1Um&jad}kHnbq{-F%=WTd(*Dc; zXqyY$=M*WZN}asWs_3V|YFYJ><+FzQc1~xaF$kEt37pNB~AU6a*F^5^Fm?jp%|?nuN8K7tsG)i=}*S zFfa?Wm2Kbq5Bb=={T-Rs_iDweF!C1jXc2sqH`~O^M5ZsbeN1Jzl~%Y-)0!agPdbs- zi}ktYTLUNT9$e8(xuu+Y!w?z?(4g9tk^jU3S3pm%gez{tJEHZF!SOx10MOMsmAtjc zg1CeDarPe2W9<39;Ya}_Dp{C=TJ1lEtN>LMi%H5MG;%g) zG|r3V))%;EBDA>Y=$! zaZRGPw(L4Sr49i7LCY@?2bk#m4DhxU1-jVUf%>`h_~FsCvhn=hWh_u%ESpw-y!L|dmC@e zulR)(sw0=6hw^9sBcFFZe4TFV&|HqUz4WZtC_h&HFDD5IVEmp`Bh4CLs{Iz;3$}k1 zV#%hM4eMFWL@3Fhr8=ZoHWtRHzxuMD&EKu*NElPvvT8Hd>8*E_YQnBvjdu zIv(jwVJG;ooo*if7bf2jw=c-p4&#@8ni9NpCpDJy{4h2Hk!^Dlo}R!=LwWU2P{aUv zV$)M}M`i=>Vi`rc&*N$K$|r=)%O>Nnt9(Q)H^=d=ENtP0hEHGAL(pD)SK|rr65D!d z#Mg$2%@0;?4v&(yI7OSvj;wAz=@km_`?ifmRU~g~nfk|h(Bdblm0FU zNJ6hg800GNZVC>;`ycOV+DzUg7*zY-D!IIHQpZb&AVwv>dd|cP=v1_9XHxF>;=P_oRqp%;!_fitC0K}S z6?7yx3?Bj;j(;U58qAFm`zm>wvNu!$Jbs$aXkvGuS~YE4VpMkC#{3JUK%m4aW|Vq9 z;gV*d(=~Ho+L3XEa?{Rw@+=Q4&VMA`{Tj^VAGmE(lboZIU^G-BK{GIS-HQ6A>z?#~ zEzja1U6T0p^Q|ep#B{Cuifun|rzbC0xe;h()vcl5xc@~Zp`StH$2Cf}iTA2(dXRCN zOp0Bi(ORih*4q5ikv6ipmiJlK z&!_e{vZ{vgBwfCWDlQpRpZV5X_ZxRs{S@bU*U^|1NbYniIb!gN|8TzqbLcbnVTc{>G)xN!-6JRTf+KQ*ZPCm$%Pz;Mf{SE-agdI)3zTTUg3=IN zlN%Sw*C5yCDdeOUbT|3VIk|7AkA(e)7~pij1M|f@j1nf?>$B;BbY_ufEsnI|JMaC( zFQ}1X%LeTKkLGKLaTW%jq_CBSn+GkUUu%drE%+8jdec2$OZhM60bPl=BkrXZwJ%MS z)@r&X(9k&LFTPe1lyS8{6wL_x$EYi#-+4GEUcfiV-H49tDD~JDesf6SW6f&drC~Oe z(EeL@}pEwuUqWE{_ z<|KS9XB*DZ>-|4UKkt0rQ{ss6Z32%saVs&`aP{E-vn>|jOfYw1zQs7Ar;5S7x&R=% zdWIw_DNd*9jX>Squ%qG0zt(4=(+t;0)vjoHxm`Ho_EcBbWa81G&vJsM2;;;y_n)1C zV{x&NHgvPsSNu9m4qtxdr?yEF)}Ab>c(AHq`>%s)yaIC;%QyBcOp$o&Iwyz~EN^N$ zzg^00D`^!Oai23?i8jxwTs-d_{Dp}SfFkDg6?T@u$^^A@@hD<%WhFZuMtSs}ew*^J z7V=@T`K>ZwmkFkrsoK5{nns_J^2%q<|fv0Uxd>-&;HWwJS{pE2Th1W zBQ%B#h*1T`cZF@$+0rh|*>1gX!&L9=xS$yd>a65944Lr81?SyqVRs{;D1FKE((^Cb4g<$B z`F$mNh=(=Ww)!p!akkSo3328ke73?_f3sE$sLgep#WB^5;nv5%J&AXKdy+}Acy<2? zFL{RGo)S-I$6k+MyurdQ2k$V&FGdMv6=XbL976urI|l2hcgrdX6g&vMhtx-zdDE%u7h{@EPqw!-$4wTzO?Rn z^_P!8Cx~(2x>X(`SKzcVlBYJNA5NC;d;auLcMv|RLmzrC#;HGVzKWTvjMR~|p&uaViP zg4wC^`EOIgIauKS# z=#cTx_z$%OE8&_!);|s`o0fU%2nb4dmw>=`V}b9t)+~N7cbv1&-q*G3 zoFRdf^0I#;;8D-y?7V|;%I|z5;0*XugV=MB0@nuzU$$7yp;WUZOHO02{hMuCkmJpkLp3esXX4VCdZpsVgd5H zxxn9?TCH|Aj3ei>y|Jdl9p9VuG3{iv(QJ%~yrQ-?6NmZhEjG9!T0rrgx zdmXn6yr$eB5|m9|Oy^TOYbS+T9M&y|cOd!OmnXzJv{gC8xEoJ*3Y@XnW#Ay(dScNO z7`;tDA5h7Cydr8zu6odu1-W%uNX=mZP?M8O$AYlayD)d~Up<63OnMtJ3Gn&H zous1#?pyf-+dV-;ElP(6xh&Q?xpMm$`5(o!Sb6?V4A{tx@=(s3|LUcMZaNMmjs5UA zf=~^@iFXH)v1s+v(tYw*mZbp7R&8S2{=k;jgO2q2Z=oAw`2MRi7wMY^Q27l_I#Z*l zIT-&FK#|74X8qvjWab7FHc{Fh_d32vViz0<$tVVWizjIw?HJK2%A|J8QiV^)gd~7Dmoyr4n4J8U+JC+JithLm zua@Lfch_)wC=d-h(At)OWh_$q&}_Hxv0Zz?=Wjn~Y&TU?<5_-h9HetcC1G%0$KMgu zksxJ-6}o;VOKUAK#ND;JQvI%6;QR8ZyYpnoLG$yeh5eg@pR)c1IP&|IXQy_?wm|Tn zTF8e7QIgF$jPFwH@HC$dQ2Z-bzZWQLysm~Xh43KIcg(^l7R|ks_LpD03#zV-nCJ67}Fy{E9#|{ z*9OxXUJ>g(&q|h_m+)-gY6%vglD}V(qnjBaU=m>-ahIh0QI@?6ma-P= zg?C&%W&{HK(9v%Ch!-K|7c$kUB7bEAq;{URCI?j+XfCEN_OU|+tgNC?INX*0MQL}8 zXyd&qdguzsn;0F+#HZS6t2(Q0EpuB4y$36WjksixGB5FaIBo07q9pqajtt(O!T(QF4BkB9Yl zp>9rdS&CWM=v;<7bNkU-HEsh6FQEdvf@Trn4{f=Gqk`6lwBCkj>38V))N`ncoq{~0K-8QSf@)&wSMEu8%&KRA4aR<1p1HHQAt z9T${T0UdTqK%gYtDSc>U1S)}$?ro<2pZwFDKRGu5`>)`~rtpO%Bu3P8=RD0Ga+6Ga zUG>NO+gkxIcj?D?pxn{QMq1a94;=IGXDWUj$nf^>hDKsSjvmzD5fs0D zj~&+0`sZ8Vx?y;cI(2(T9MiYQiS=phm}-#x?7iFMOy&;o{pVFPH<9eC)`T8Ajo`@Gf>Z z_3#ZLqk(1_SgZOCJ(Sn*XXt2*2yF4RVKSoU`n6;CKNuh5X}Yu+LL#YX)pD>5uNveL z`yJrk00V=kl8_jD>^11vpSsR>k;tO`S9CD4q|sqskqPwBbJSp_xYlYog*#bBkjVX4 zH-dWsHcJjNPL>c6#&~!ek7dHc5cGDk4Tw7D$RFA!5Mao#K>oIm`U3a1)qU1Ws@Xsd z9_-Ril3QE#Gq|nZ;xJQ&Ok` zvEXXeu~|nn4>~$n#F-O|V?%EvXVOAgz)5m&TnhJw%O;9QT!|TzhhMy9og3Q*#C5U> z-jrG79K5mYj=Pp!iL1@*y{vTi6L3Ht3?`3p-%vdn*?ZR*SgH&6wgYpBO2BHKb&~6s!-ANX z=BGI>BWnK`dGusCU270xrSuk7K519yvb~;=_w}<(>HD5c&T{E4Jj=@tj;~ADO93*(+RtroIGXD z5~F|uCW*Be&nue$S(x*q{u+A*g&?=)hI0oixar^#||(67+@_adYkEKnF-0MIflr z`v~t2mI|WyA5?%3T~7cvwuH}A^IPJ|^PvpluD{*#V-3DgQwzeLp+#;|1l7anbrOO5 z@8*O-;Us@(5-719FH~y3^FHZ4@@v4%=hNtmUoU*pcpD`akoYMO3q@zVms!jkHzyvn zHLmv-q6`1qCkoOp0_MoipPtd(UiGdet=?dziJiLeZvFIz?fnF{D$jVaA$r3&m(mQQ zyu_6_k|%CUyefAypV2S;z-SY1yJSoc-+%I72ac*>)rdZSi{CXIoK*f#r3>`tz6kH< zRg;X2%g^iv)7PYjPZ2J!^P8;*nG-d;JZ{$r{tXcL{dPXh-J=l50Vj9)PI{dbc&9#E zdi0HZ%Vldk4e%=UnNr*5Tboh_`0OLL{h>e0_82{M;`7UoW|`-jrS zFPLs$9Wxm&u5(s_^W6l2GX}oL+0Xq{a~$fusxM0e8CjhgBXn%?uWnMX%-;^nA3ICX z3qIbjI|H44WU=m_{+t+rE-gOe*9&)TL#StYo({KrCVT^SASBqYUPDw=RljH%^a%0$ z5Bg#^Gv-fyLfla;XmW>pHrD#3>!06EI8KCJ-;Q(44@7eT>>&klz>L=jx`FC-h0}UzKL;o!bhT!zX`aGaUb5%^N!Jg<(`3$&K?&J@C zo@e_DXDGEv<9>&D_a~CMN;uQ@qEOmzYr=^PnZlOFKV!u1XBw97Y~;JQOSPe_-^{gB z=;o%FNZiC(N_BF zYHS}o*$l3I#ydkIjh6JY-66so>Klr5mrfj1{;@T+Vd_&E!@oG?^5YgVd_imgo6e=C zreZ@%BX-BtQ{UzPkFHVrDeT0@3+$xu4W_be8x{weZ+>7Cd=zzh3$mZE5tnrtsaO5i zS#yHSlfakcZfI_D7fbuBcUpDMA4NBAhP#(%-edyyzfBD4AVlx;cj@)tTC2d*LKA6V|tObBdNBSCMsMPej0D+PlH zD)7YO@J?bq8$Na9@FfjNvB7kBM4U*>n7G=lgNc#IfYD#)r@d8+7HBTDve9V1`jFlo?7Vs#ErVd; zMR>X6!Y-qMXcq$A;6P6jYYQbt`d+~T|MykEi3C}xfD`P!sh=pc9zDiXNRkemDgwfF z=m*Zfqn2Y_y;}{a2KIrK4%JCt2LWlo!d`5MhkJe3i?9q!tn06WIj>Mpt1_|mRsUNT6$h%sPCMzIi=TiLaTMmJ4fMTIuF^OHoYM$z%VbdKU? zx6$;=x)ZO}z*p9(HYofs<3fM%AdVYMK!MvBF`I2o^ti9_Zp+I(zvSCAE`FQJ6h6I@ zU}~MK1^-krI`o>aI`x+|D8cKV#SHb5bP*{@yJ?NXp0o_Odcbq-fMe>OL=RNg zT^#a$Xf4(U6wSmTSGT364|yxa;${MV;h-wXD;F+2PO3d<@&;p?_&q;!%&bS=RK|hfxLki6<$24!z?tUj{KDhlh&4 z&RV!BQz#6;W(g4VwkUNgDknz58E86pg%T@?-)U%4V28y@Rx?W*i1nf7WTpep1l|zG ziQ`70%mfL)c~s*a9~3B#dPc_2pvjNaCsSX`S?DS!?lc^J`TFTa)NE``fAixo@-n%9 zD^}GYp6=<{#j{%5Q=kOl&0VQo?d~)!4nC&o2Ls1r{*Ji(EC49Fsga!P0W}htZ$M(> zwVknM?41B4vk2fet~ihCk2pk-(vUG0Mt;F!pz*5OG|962PM09C-5Kcmn|xmtDK_yX zFW%BvJZSyr8QMF}p5Q4dhocuztbaYUpM{@ll(wjyjcw1&&vHQB=d#4yw8Y(s&w1p2 zCvh}zrs=N)79i1nU_vd^j7bgm!?yI10C`%@x;wQDVn9^xR_BWpTc=Q}z3(5EO{n)u zBSY8UeV=Ic9@M@$O!w}t3t)jbVHfMho76WI(i@98nHj<8H*Jq6Y?)i1o^f@Ko8J$- zgT)&Fm#Eeev8-}N4>a&T%6j#t3~cIRA6CO|nd)As{jYoMKow10Tg1z)CoF>ooltiw z3aVZaaQK8;==mTA(CWXU=LG;h`BSMhb*|@1`%!aDK6mfCpnPJz|7wLG+iKkvVspxP zbGNw=B#PUGGV=P%w)<&R``RM@zSoz^fJ zN?y9-kp#W9nlmBB#sRSf-!sO*D-3p z)D}30=~I4EYI1|dpZf9<7xxruWo=BVpr*r@>_~^PIvQTet6!2Bj4ksJ5$}(_P#i-h&$CWQFfpR7JnLC_#X$bO;DaCR`Dsl3k17Aq z&DRX%1!h{FFW%=xzZj)Ts-hwdK<0?8ihG4)%n^C}Ek^cGgVEECTJ1%FI z5vdG3V}v-FCpKB9Z)Cw=%0?COCkDey`u_QlQo$1*73QQzyvH)ag4Q^y`_puUzgfR( zUo3&kiMfXq{r3avMSQrSxF|RK{K!iC_;GCFjzzH-Or90-^EWsNlk`G@ZE|pTF29GA zn*Qdo12h{8+IOwPfa@Kblxq+aDeTfr36> zDAa3~k9n!TsASOTe3wdzff%y-CK}~EAQW2|m{N&$1C>(T3sm8%NTQcAM~8`>j;BS* zs(N|XoH(9KLIFlJvo$D#LEE?&pU?c@-#q;`viX*R;IjKQ{1^Fe^P9N!EpJN5Zsxc2 z#4ngQzL$ewk+amOd)h*&3u%S+(GPb9HY$;O`R<9W!_QzTL4Q8c|G^HrD{ra+mpW5E z1?Rv;5(6_8I(NAzpkqEMZG3BWPPV5 zkP|O!PM@vZGutF&i{90(^|?;}pEJo!Ynh<_o_- z=F8h-Qf`mt&Y3PCYxjm8k~jHj|2V`r-aDfZw(8$0vLQX(7{rcnM^++dJicy=-bHFPZW8{4UN**m#2`kIc?_9IgYd}ljpK0ZYK?X}y66uc z()^%Ox-VdG_Bb}}t`uOfBoSbuwk!59&K(u%X>!{PVVlJ0Ocy-3A<9-soTpy_i6_#{X0&ji8)d>P}Pn;L#_Kh6{)k~!(B0^R{f z^{F9(3fU>r=Hvi9x!sKFfAkQXv?2^&ILMi&+$;4q*^t^qCMl7L4$FP+$RltAr$|9~ zMegt~*!_1loPQ}*YOBPs^ie+OhMM2kEA#?hE#b47{w7%p@*TeIV#4sXS<2Jgc5U(p} zBdw>JlfatE~@}(HX=9yrqqy;kh-zNp}rm={`&;MRhs{=Y*6`@1 z!hfu@O-0UBbZ2i0vuily@whwlBK}eI6O7<&3O)R>43BKc7AK=2pP~m<|4XU3^ZqgX zYZzKfGWNR|)WZvld5K75Zh`k7@tjd*ep><;tMzf;gJEAGQ3|xR_4mS-6;QE%RPq^Y zlb{HSIn>uGBVX-{4LAsz2-g0GoLI1<6}8RLFwxOB$TQP!IG-<<;~E|S@7fq(K#i@Uz z56WLzr_ICVK(zes{y2e}HDH6C{^Xw$AjO8a(pJk$0G;TSCzA;Mior zEs{-Kv$G0A@u}DBy27NqY|^;;2FA1Z*f6vnX|EJtzAaMNwM#m%SN}zRpmRjYsf^|} zb*!?S^DZhetbOb~70!6FlYB5Of+GaHIiQ~mJZ-*tPsi+s2gN-}gDHcdT8Yo*!B0Wk z!I)%DA8QDiA+yV}Ixzi7zs7b6kK@c3iP(6An-)bS&G@^I^GPH6L+qOKLQW*d_-oej}8B}vw_IyC)HH#QON2`jxL?G5Q4iSflzzli%t zn-clR!PNKWsc7YCz%f$XKf>lu96bkv6L)modY^E-j4AZ{@VL2`+lBRhGE(HHC!GXO z9NeuYFxw6)%17Z+@@_l}P%>gcTH|i`QRDdt&6vDG*Ro^$n45hTwrXwud*P!=U5;9D`o<$>YOCOl`I zol;ad z6z*noBF+CII#IKo0w1Oi3oU2#pc6*$F{6`3I~yIu?qE&OXN8~R-Jn|C7B-pz$1a)v zt~aW%+v8MAV->_9rqFf4%a3X|!|ic~^|GLlfIvToC>p-dw12^eeDG9I-839)Bqk|s zF~firm4W2*rNobG5(-kxm9?tJ{TCBdhPd8GY&?9Wau~$B&qWxH8ue(Kf8k({CahOF zD&FSwN5(Sf*z1^>bz6klFOb4CBf75V=vSr+5|Om0@16{PNJL}e&B1*s=WT6)=!u}K zz~%MWA0OQmmSmitT5ypteF1}(+HlUXI`N_$Q;0-Ko9#XIua&#+*R@$|g#6V#QZ661 zj7*;PeBa+KmdyVN#xS44=2Ps4r6kpn907UI1UuGJf=#5t>&`wz>V1U}-E|}{_B2ps zdC4JbxvXHyl)F|2&gg|-|Ab#&>`!2zjA@19C9O2BQ^Gn8N`LIGihnw!J;>=G8a&~r ziH^TfX{jKL|7z6IJ|NNC+^JpF)BIZBsy5 zqlh!oPeK8qVN{Gm&F*(P7BwiA6G9tEnm4BsH?k{PuixNb!++TL(5CLP))nl{fdH9Y zLk0|^B=<*MyUyOz%HhR|SL{s{+FF_|>QS>rY_(YGy>PZ_ml}4CWH#>F9&aiPf7*>t!Ho zjGaAVR=bbdNj7q&o(uX)c-Ev94ZDIrGV0u*aB0J&$O6>Es~kg;KQ#pgRz6U-J$PLc z(IRwR**QP%(zP{Q)_!H5wv}a2^VIy3yPxo^;5d6C0#X6~tW5!NTAZi2GI@ab;JE0O zB~hE1_pYcx>!B%*e;g3)>OBg2nHD|bUSa#oYA7*=7nmCHqCV3?VnI+=6sihWIpaZ! zw?>_8BrF9N&myIBES?`3KBL>AV{H;rXu%IAW{-&$(N$5cK2lHue(8>{>|uzvhej7k zx70XkXvcagf7+=jCCrV359?QPjE*SS#7`0xhp8})-PuRmCnUo@;V!=^HLhPmSR#LaU99g z2lsxtOn}bM3Z{L)H;N9E=mnUC;@oOcZkU#y9fj@ZD@0XTRrs>8QJ*HU*U5p6WO{rS z`YwFXB%AH6j$<2P514TN^RfVRY1 zHEre{W%Z>x;EcVKBFKC1*L=J5U2IwC6nrCiiM%KidLe+_ZG?s%9UD;n3P+wjy(hvvcFM2za4bCgq)g5*Y*zj2)XoMUq|W~J`Ct`{ zwyOdzr0&`Nqg747&8jzOQa*@+n0(t=s-%67FC%N9d{ECR_q7 zj?g!ribnH;%l?CIs&P4rM$DRu-2a9vTlP|D0tn=}rosw|4P3(IB=}%YlidBM6Rumn zj}07lesO#DV9RaR)Jot+13sj+8VRaL8GQ}jSJw7+qx5Tixir#oHVQ{soer%Z>`>VR zl|GO)c=@hQ6KpBz+;gbZxb~>L9X)E&d9`S8{*mnFDiujUn7}*4h6y8s*C>&G>%`yn zh!G~CdRXF%#E(|LcqXG*bkW^-3HOjXWbfrb`NzJp@*F69k-{5E@=a$Zs!X?yjMgK2 z?LKH^=JYiajRU@}d&wQ3bQ81{3#mnI#=MSz=8cuYosa8*$=VqIUCzn<_;xm{kkaIY zmv{j46mGzVEY8LNJ8GdHC!+&QS+Wo$M!Gn!@gF?H~B1CGvX_ z!B?+`*a@|Vv}{WoqU`S(ZguKU%Tru=+EYTRI-^+cnT_TEj%>!JR%jv3l-+l_WSa+D z#Szu<&MFn9Wcm;L)E3vdzeQ}buQ8k!drgdeNZni(6(I&Z9jGy`d@KZW!OjxGh|?AQ}ee6dE?0!DI5Ijkt17TWnE-h@`Qadjl_dh0F| zT2|rDx-Jo{{l;A-m{4OC8T9I{S1+Nu+jx80(8mRSX1R-pO{9C`NJGX|D;q0~A5pd*%_oTrY*&?Djh59w z=Lm^cx`s3;Ae;g~a_5PFX8jzzBCVUbn#cAfq@uAp%_~;kb?G%mHuZd8ilA4Mn)YUT zJDBcwT+T#^u|E+fIGF&(9-HK%ayH0G7z&YSxi*6o$F3jr(fxQUS$L&i7Sza#S8$om z+wV0|a*p+r&{cvvokYBBd8ExceQ^H^q21K9aCwN{H!~D}*d5aDKG^P@*U~Lu9J|}q zDBd3U%Y%ppX(G|&txA?-mwW3c4_}A?g}dr5ACO+TdQ?LgdPE25*UAf#Q}l?8f56#RIju~ z-R0)}aDyX<{|k)QTQ4TK_X=PXOsjJ1k2!Bz9uG;-3T+o#r5oMowbEo0lSsO-ftuL% z{pWDH;Rr`VR! zg&zVipDlj*E8!RlhSh5uI&m?TI?8LAO#C0Sqz{3IiB>cWXwXAAC$h=ia-2g6R=pG>wX~<4ibK8lw3;>oN;#5qu#ReFS)l%IOy3$FyyFIfT zTvF`~x^%y)n))*Olt|SKnP}L?)gpaa0xcLw0X*24sbD_W>-WkUJW%0G zvq0-J+FaNz$GzN`Yb_QB(5pGfcf7zTZihi{o*YUX~6v}LU&=JM2Mn{(^JWHR&R%w%$2hW#_JOKH+; z>-Wp0XqVsURU2$<5&MkTcM?g~QS^)LKH$Zb-itZqXJ6j`)C&0p72y&0oOyA)F)Lr< z$bOySswt@yKF-`Zc&r86^xueeSL+u0N_gVLn#=L-_s8(nBQgPT2gwI?EwMiBTbp&c z10j=f80Z!|54rutCqk)8r(8xH^F2;yBPYs|K%xN-p->{=A#cyN~5|hk<6!IL4C<$ z>a+ND#j4~(F6Po6=oO$}`fo**d1^gO@Sf(F0&~j^|1ei{YC0}=jDSz(=UdL6=3ApF zFOS9UrD8$rp|7l+qV%WRM7z*%7CH_Fg0#0W5>`zIj5Z&Qq&=Xlsxjy=ev_)!?G!!AwIj_{XF73OSX zAnu#7cH4?7)tef&M$ADve9`f^))R~q%DfdqKmTUH`aR~iEgppXx}%`mk#xSgv6=*($KWJ>;;Qn|IB_lcbV{ZMnFrDAQE zW37hZTjVbC7{Y1CQ1xW$iPiqe%0;bgsw2`0M~KoxBJROA2W!?TyDpj~W|D8BApcS6 zzH}EQ9N%M>TvMwsA~*3WXn&fGM={Rp#hwt=G{E)CLhumR%!#DSS~Ew}vWG5A_rD*f z__rR%NcgvsN$3?x4qj7YJvL(688Ow*b@%yFvh>4^gH9X#%Wk5#;j6CKjz3H=23~v) z>f96@3XXJJnN4`#l$TS|Ql7oeu=Yw~;M4N^*wBvfgoNo-XWT?wazr98az6Z5nu*UA zxIIeRqME<(g*C?yeD#l+*w8}5!EWz1imQheW;yngdw~(``$QQL(Nll zH{-)nSK>n*qg~p4ErD0xlY#7mDJ*nln>v#4^f_5OXQC47^HMriO)t`y+V-O|;p~Wk zzg&o=hcHxgACJN4t%3~hrPQ5objp`5pE01NZ5Eqdp(2bnTObFv{?IVZ+|g80FPl1e z29__fB2Z&+PnUR*@R$!XV z;jdJUEhw>^zR})TC=cNbi(NIUbi4B>(1?^(N3ZuwT!$?J9-rbsUGkhUq5B%1^0x8Q7$`A?FcU>ZSfsGsy4d|Yj z6`#=Cu9(E|M~TvEq0Lc&&v&YAQEQ~1sZ1x1zUw{7k^4L^1^vm zMvWx1MM6*T*AH!jC%?)PM=ulJLt(t8-D0`wkilDD5uKL~V`o#hYfb6~DiRbEQ3PkJ2Zb!_D3k@nmYl zEzm1XKcm$v4f?*Vtx9g=nZgq<94l9b^po1L{$4^Mhs`9(5cb1WjZhzq^zTV!6FZ zlC&JCi<3VfLlrbW^%N}keQ0hq7YVjRFpig1No1@bv=J0d+;W>CsM&r@pvqY?YxIr& zr#=4+Y?lc9MthCBHOLzgwwc(f1j8S5QM=sVn0aX6X3+aJ#@x{+cQNn0P*fh*jbU?$ ziu1}a$oB5L^O4Mm;K?pY@y)SD6SJ9o-VBLox_Ik`nWK`rE=^?U*Gqs}xCu$<*OY4U z@0O22gV=auPd~>EIQD?{3TjgksAQ4sM-wKY0w+gnaT*bM&Y#bzg-AYha~(aJp-3MY z*KK(L(9B;^Ns_)ric{ejh*xBCHe2*a=oT`#>_zxM*uP7$D*}oEOAER9BCjAhTfEcX5sb}>RRV>dBJqMO+Y64 zcbIhJq`t&NJiaDIwV(d>@@YlsRaD*hw~^XFLwCdmg?D3GAvXAi_y?-#>%|sV8yIk3 z-~1Hs4TgaRG4Y1^55Pfz*7##GXdll^S5nsvymWQP9V;vINE$8@UvfChEGJD&2}#&U z48H!&nSegnhl1QXE2C&0?|^c;BFqx8YO0j2O|>o$Fb&+SSdh8|y2!+q;A0m4dw{qY zSA|Q+y3J~3GjR6-LB12*nyZ=O6aaxks9r*%iKo8eKq-=f_q`s>`VMGUIoP*nMb4Dl zUN@k^+^Oz^Ca~~^xi@%};HsGLzj_P{D5*Ek&_Ga4K@#0kmj*NEjmn1kWa=qGYP##I z&WmWaW`C=AJxs8#>sK5gnwEmOPB9Y_kUh`Z=V1O+nAzeLM>^UZL=&f(n*u7@C-cAe zkouL7vSgZ$SI^jzX zv3#dCSV<3bofj*!X=@oc4l(U%oa2v{o7Fi zP-TalB~xomrAzbauGwtPaTIH6mlvV}c=vVtnJ9}wtuoS?Xg6L$y=8CsN)&$}``?Zk zUwCa0X|`IXR%%i9apZ)c+xMcdWmI zNTal838Sz^X`iw!68s5IVO%g0O;Oiy0MzkfFL0pA#a?3xZHCuhFStn+;3k~M9R@O4 zl$9SW5n%NvjNGxjJa~kokYr0b%52L_8*;vG96v4z*wmHz;3w`Njeu(r+HpN1Lt3i_ zH)qq#6T5%G#u~n&eLhLLK;n7k^9x!kVd%JS1F7|+-mF(VY=XUm;i;1;dIsNIMeiQ9YhKUxwig=_RKIw4)M`Qdzel)QlP!||^j~yGj>?GOTPZoxhV0^vU z&W=yb=MVkkPy*{jOO@#qJV8P8;+5#rK7`|NQI>&sh>5!jhIxkHbzyJPPw_?H9^*xR zvHWr>TbwwoM51l~Y`59O`&a4iXEJ#Vb8kRLn{tl!r3g}2goeJ4?58X2hG{tucBkr| z%T!)A!7ic5RNoo_^r<7>7b=W%OwD?T;GC7Gx9Va}c5fDQgd|d=ULMq-NA4bqI<8rpVY1q`UN8VQ4T*!3|}-}RQE-@HPmum{yG}Gf{($RE<7BE zQeS#w);p5KrN~qYdd^|xhJ9&AzlAiJ9rSlC%Ff?uy=gTA3@c3hxxXW+GUbunU~n6S z24Ku-3~xjalGp}(f^`ax1ad3MR-9| z-iV$v70auZ>=w)}V8|WPAlbffkNcKQ1*I+`su^lc>*We^jkRqP76LL~zM?z>w@<;q zg(;=3Dg(X9iJov5+r&a%|6u!sFvO70;Ke@Gn(l0cuI5Qy@b;44x&NH*JB8GiVQjs~ zaPiwkYexi0h#mQ3P(MVOI((?^>k)&OZ<4i| z&!WF>#RikyQCc}zI6jd9r(iU6*A2E(2>`g{V1X5+1cfJvn`T}yM~@2IhQh^PE+2%h zSf;g^|MFyi7GkhrN=d3BSpvkZhDk7q7V!LFK9RQG5B`*iCqZnh;((s5vxA`mSkVk% zMXhA1CV!R>2J5p-W=Ze$a(JY^7U`dFWT47cLD`t6rHq3N>Z4&fd65kxil9{zMP~pZ$CP`Cp#_) zWr-aXjrJ?gd@4TzCR{``=|a0BH2S zCW#p4=UM$La?4TPa2%(a2K!z~_H+hzaV2rwqquOwvX10TTtjY zp(Me7SC%>ON5^`b_c(@Vaw*PX_+E+dy{Ep{>1c8mrmrfP{4q}EVYvO&n@}-?XG{;dZNTM9OZT`Y5ziE#= zoUiuCL0%R;mW+&PCvnh)*ECX(|D=W}qfz0HoOJbepQ}kw!)r2|%|@ece+uGL1UD4C z<)7K`4tckJJ@igm8vGtv@S5l9d!F1C6;UkPU%CX!DM!aTdaGIYA-oefd}P_UgcK~K zv+!#J6tf+RG(&1ro)p<$L$q{!rG+h$>2>e&YPxCBw*P{_lxSvdm!9@;F>$x&&^b+Y zGc8`hg?F4Sg0U|^mhATgRb$S>UZr99tmDZ0j2z)cQjE#@oBg&!zb&%N^3`|#RTKAd zMt8Q9BEb#s^%Ur-yyCAK_E!#QRII8m+A_9wc#J*bcpm;G3|m?lK?y##YSG{g8kI&| zhum@u*f+^Rn}7jMJLy#BizwcQ>no3wzH*A{+|#svyMHjb=U5smkYgY8a7ESfT>2zDK(_SSm@sM`X zKk(uA5PO)P0PSVZ!jgpMuE2i5chJ!Nk5r~dCLihz#xGeG=u>k$ z(D{yG{d|F-4GZ<-M7@*7IEU(=W?uVL-|~95qKbgIsl7cK*-k@IL5XA~{%jU|6^zcR z;(ChoUK$PE`4&;B{Rok*|JwYhSgYEIBYDj|AFNJUM>@rZmQAbj z&X+w$)AJoY5zcalwnpV*rLoFs+fHA_;!?83Qi7SS95jrTRPHXMB%EK*G2Wyx&N59d zrZcqlI8B5PRp-qZISnK>l-(;yoUm46#$7vjKjDphf-0B9j1;ZcBNM6L$`>63sJ^;h z)X33((xzVdG2o@w!>DZ!wovk-g znE*0Fuq-eS7@T|5q-dMRr&L!zY`zoaOV$xgb2o%068#hl(-gY3ccUQG1fq**zq3RUw)HKZujw~UwzOHPvj3hV=(&_ za$do?dvuCe>F{t%C+m#X3~g(V?j_d(X3bR-&gAv54k388w~R4%X_}S^0%*E9}h;)_|TlieA49s8VdfwbD?sP z3F?N4emAG!7#oM+drfXj3Z`f_(o12MyXp5Eod4T(Bn;*+>h8O71Vcf$!fpmMP)SeN zAc8#C?y}PP`8x1&Y+~6p5y={Ox&tVR^>xj6`FDnx4E`9m=+*|s0!xyRF530TS#}U|~#4;CwWsxFT zxcq2(y+lDdz{XqB&~?JCF?Crwp!IF}-57#Z)IMhg7G36n-t}kdoD>)BBQ59n|pYGgNXw@o7&CxaB6o4+*bW_M^fZh zNYWwQo+`43{)ycVRLC?4znXHF@crBpny=P3Nj14X);Y0dmgk?>Zx9BN)qw#Y!@kaC z(M(G!qUG}pJ+Uut)M1YJTWN-!37icf@}Fb8>eSS zH9g{45;%mBR}kBD6Y`snIk6dAr@pfAy|~RW9w=Qxh=872-Od*&V8M6F64uAnp{Bp(Z_PU z%^BsX6iSllYXiB?j(J-3hXI@c2H^He6Q$2aS{L$a8k>|wUf5OiiNu#w{t%|Fx*8d7n3+9K6#wziJK|^LO z>1&v_Jceo#Q|eQKSgkum z78PJ`noklbPF(t_=t(TN8+~lmz7Eps?($f^B@6%KEA;UAQ9uN1zIq2-e2ob8WO6rt zK|O)tMF;duF}Vu_uJ8jg@)U+!u<}{2%J0&;pZb^1!XI1ciTQwGmH{_z&d( z>Fi$m1TQfuf}H-1|7+^Y1EGAscr#Q6g|Y90k$p>r7)?lH7nL>p(n6VUM2t#yrp8Y8 zkbTcGk$ntFA-iM`W#8Ah$McmAK}eV^rg&N-j+InPVT5fWd``N3C5sF{jhyfGyk zPKA15n~;4KLKQ8Me|NWz`FE?rFuj~)>f90xp_SeE-pAcUmI zn(klde_&(m{pFRL_bfVuq8acc^ps8bQs|CYqJDG8I(+604>z*X%fgekj@<&YX5KyZ ze{sW5twG*<_w==#qiu#(Ck=6mfeIHwk&hI6jyb0!Z!_9j0Zeg1eMeCW0<;IP;$9{7 zTzw*Xh3{Rj!=1%VZCj$9%s)Fp#Gp*C^;n4h^l95DndA8Eq^5N+K-wC9b^JIDSXw;q zC;wOlXmsyR7_C=dG9juSaT_MuQoKj|vx3X~zPD|QgTlYDZ~>_4z25AHx3sXx+bRsI z+uLlPj%GGRSw#bt7}MgP2&j3FP+?;ZKfg1q51vtZB=erld1BXs-28!Y`WpPd5aGl> z{`LSKiIp%rzqqBRx%i)OXvJP*1b3K5{KK{zGAt{ibPqc)XBuAlsa@ z&Euv&hdteymX*wA{Yi>q+;|hfth` z)P*|M9yQi7d-oyxUVHlcC5{b&;s^V4}i)0_*HoPlR1L=jq*>yo0H6RUen3 z)N2}B1Vx#P9(vrT0{Q;p6gPA(nQmKp?Xybh%n%3 z%h2cf1bxfeXTmU%e>iQss zU=7z$+9ys=_uWVS?T}TA72O#@%2;7+_g>7IA5Q~AzUa#N-ZfXv{vakHWelC;D;v+O za-_|D=6@+Lqul671}LVZkHm^MuK&mt)8hS?O|)GF>z4I?W0OL6pYsqvX-VDa-2B-9 zx3ekKQqG3uQQZN34l!lPC=j~FFvs$~OW}Sm89$X(aQMirB6-WPgMb5c3YkiZqH52J zNKY}CHE7HUXX8p)?{~%76+utaLkX$nwZdohM^jL6m3Nne(p4@v{~MG z%NotmyKcere;qk!r&K5Pig%RAbsY9f+Cu-8;4H3>9|WY7ePODX z3^-u?U1?RYrzvz~AmUwY%BJ9BUO=D|_Pq(rQIExRCXoTy+Kg9+Vl{1umfgANbjmB` zR==I||2X}>MPi|9&p!~hZkXm>NLP}4P2=%h)nbSz=Rv5TR7N;T3!H~+W@$Nn|Ax%= zO3ak{GJe%=&n0tRT|10PJ~O%+dDHpJ_d&z&Op;Rn=!X&Q6Ny+BZ|{YthV2hz3Y)ny zvTW$G4zN`sdlNILz9n`HhYsW(KbSXbhXjHV;`-bzJ$%hFvzwO1j^`=?wPyxu6`xEP zIsWQ)#-$*2d09pQk@Z&#v6$Y4RMp(iev6x(M8>!qXt)dT^F4tHCxukgvPjE6MudwR z>1WTN<)xmO?Z&&954aU4^l&Ii+5L6V_K_$u%^K@4_Bz1NspSU#2fOudccv=FTU*;j zO2H$9}Fxru;Kk6k$Ok$Pn;dq)TPV>hU@L} z6Fk$W@3@G|+_LBG&-tIuS;F_}ZPUJm&l5gV**UFmF}*)|2zr0?WDN4N4_4t6#(=7S z;=Wz-pBgk;XPa+nGsmj==}~sjaK^`hQI$Ftx?agJSkTqKNGpmM2NE#9h2@4ZD2642 zIHqdzw2ySL;t`Q0H3AK91}i5wLLs4ofyD1&q_Un_KQMK_iOua(<9G1!u=8Fd?M%wC z9z%bF36xvOZfLqh*iBp`zB5c*TF9y;`t>7gt=48tixUFo3Z)3W7w1NmAHhzss8MXP zX-Rw4N79hwwt!bavUjIg-4uCbHegsISo)3qui6Q22{C$baC&1ace&AOqYrJweZBh5 z${gkaUZOf?X=yU8MWjd&CA(NHWfK772fYhJH)7?nwAiR;mr~qJ(smx1hb+5S3@E?o zh9Rc@a?au05Uto-29?eMGGrprP) zX$57u?XZ5H9g0+5ecRyQ7qtTGIh-F*6PZ#me?3AHbSIBf{N37tK*X4}`QX@Zn%*kK z*>(n4+gXrk4Rz}mI9O1@_*fe4+UjkJTew~py3bFFy^Q$#?=9HMEqQ$wo7D_Yw!rmY z^=f1J0z5}5BY6mTCy3YY zCfPnM;F5u_a!}tBKg(SkELWd{9&K|M$8_077myRRTR&Rb&#A)8B1dSBBt{vTrg;MC zd#`8#NiSA;oPMtH3~3 zVvmW$>N7p)ZXpoGXdf?hSqwP!m0R5N$#9C={r*o@oP$2vJ~Cg4J}WyIVPJ#o`?%*haAs1uqPq|$i}(IA`twh7%9<{z`k#2X+CY^Y09;TG5+_m`YdqnUjabWqXf)-B()tqR=D5Nj|vtlD=hm?}D^y#xLtc z?bC3|N~Hu^BfPRB*NmT7&CQUT#9E^QF=#j&@Wr#L(q(6$Ou-f+Hybz6InvCl&SGh` zhS#N-ucdE0%|o&uX%h_29rM(2Oz~#B#Kzk6xfI6}!eC!6Kq&PWm4_+6Rjqg! zloQmNWLWe@7E4v->qfkO&#Nt9wINThfo&(|^lOY$%0$IE+qEJA z$q9g+LF)Zi6$oRMzRi3PLoeIgEzAd>Pz#gsY%`J0#m0str}xEu1+w5Z>ljyVN$vZG zPdt$ES!|*@#n7}Wx?Tl_4lTlc_#Sd)30IR(9#?K}Nc|>qY z6IU zd8!Jgt$#V;3pWo%?X4mp;>JwuGXIC3-Y?VBa`&FtDtvBPwFE&Hhacyr_WljMxwN$V zB>jVrhHIQRt?xr(Ui1jAdN_peoid;A7YcuULlX!O5CUh{Xy~}KA!<-{-jL8P{M+nK z?)&(uKU}W)+xY@R9>T~M#H(RsXB(*jpM0anaws22Q3sS{`Vw}V3bFWuaH9cO25E4Y z9AIAAPcGyY@)eEIm5VMZGS*6t>e|3lRC#TirmS=-4!B0Iyukr;k%D+F2JviRxt12C zv>Q@Ywj3BMqt_Tk7ZL|f|2A&D#V!aUGsMPV0s%bs)G>ulFu3#TQJ0X+V%&*%nIeRu^EbQ=ie+ZC$JjOI}&JB_ntWu&esE09%n{P2|`B= zhVixyYQ%+b)B%->at6kB(9VbDabB@%p4yZ?$D0?moR8U;rNmgD-yy?uL^)2I96BX5 zkXv1Ak&|*J8JLsLf*cX(pSr;>th}4H9$A_FD7n~JjsxCV2Eb`THUyO3j(LuX+BsXC z!n`iMN|JS~l8o2}+xN2NCD1z@Wo@Pb=;byw$h%vwdL+MBL-bo!!e;ow;V6nx^xM6`pB1;qH&)I?Ia6_iL4IA`U;6 zzS~Mx?;*F~qn2PIxvTmsTcMqHp+hY3w0~A_l)G_UN~kWBGgzlr=4wV=0y1S^dj=2{ zow^1}l&2sLa*5fEsq-GLUoO)d7kYZ!8j0$8_|eqfbPGwYa4pEL#ejV~qSncMnB?-k zKdx(SeN|qH5khDS@{go%Jo@8uC;#_lnC52!SgutNZoS4c*+qcxS*Pg!Wm3kpe1vZu zJFh}0ZUudNbm|w&b}^{soe$m60bE|V167vJIv-357H7?DcJvT&*hj4+ZAi8g4Oj_vVEn>bxK&N- z4yr8+nU)xazFBz9jXN?CTl6;C660HSc1sM7C!hQ2b2q<;>TJU^UR#mqxcke)OCubZ5x z9_L+%a11%sjfCWW4f3pGSdz(pRi^5NSyMAf!pgG=<0Iiw5vWM=tT;s83;LGAx|WKS zYqze&bNzWIi!=Fkzo6-)_gfloR&p%x+ho3F11xX|lgKeG2~DMc-o*yr>>xg7N54%) z*+^|7Cj{p8-4YeeLUpeHe;X00YK`NPTgdg>66Okzr?LIirFMM3M#_mgV3UYckPPYb z+K4Ydr7=x z=B=?y1^0J4_kFR~fnk|JqMR=A&a9fGl6v6C06W$?yZ*cDO#x3UUJj&y%K`?-w*+V; z()R$|$P5C$)=S|%GYrj1G1fuC(|rCWXt`IqdiS97Z0|MN2Nva+ZYx{pCULJJG&>+Z z+eI0mj4dqQNjEpi2ezcAALN-5HYZ0CNQ*cWI*??ptc4TuCah$`oTvaRH2|#UiLP;p z03uw00vE`G@0LNrvH~ikSGfZ-4Qy#i-(o{cu&T^nv8Rla4*J6l>V$kh3Xdn&9{8n8 zV-r?#dNAT%oWym0JAaruFYRjdU5XS~A&;G<4nUA_esG33Af2|F@F6NJi>|Pkwxvi6 zF}68lnA!1eOv%;_l|p|rfy_VZD{U_ME$OsPYJw|B=(Mu$I-3`^m`7PnWZn(SH#52K z3L91?&1a3IWtt&eD&T{bM>=2S>?t+j^whZ3>-_Z8r6Gv$05a?SrDXGNMQ@WCH?sfS zc#tg@c2Xro5w1Tdc0e_2hK~{Hsfl@!vhS?f`73?}dl_4m8roNLTKH{IqQ7C71WvV> zzPDIU^;rjKS8KQiyxyH&jshC6_b(y`xzizGZ={HA39S9s5pU@hd8SQqok$y--s&IV!(eiQ6{llI$fbRKJ>eyE1ncTG)X0Xe8U3z%hKF5rkL@@ zVUDJ+)b}23@VaWm=B2h35Cb-#=IugvTq>k-I8`9TMHPXp!81|_o+8*}ibv(s zlVTJO(ZzRHaV*u^p)?Lme^9OTuVppi#=!1p?{}8nLCBnf{ABs8lCJJqR8mfV$NhYn zR=?c}vf_73Vs=E;Gv-l8%|71%)8a+wv^bQgc7oOgUmlZms?j2#rE>3A=vHOK!Q)%) zKbvmrqbE$i4XMnwc>X4lFVomn17P)hFF{hs_XeYa?XcbcqLn9#ht=p;i>cS+&~G;! zm<1W1;VNw|U?eqK2gLYOTPH~sElO(h{Y6L@CtN1oR}7?;uiot_=Hlc-b|nhdy+ahRF$#2`TG5nHv-+qH-99YXt~e{Zl&?{r>5(;TA|m< z-5b~zoW{V!)DDkmqx_!gF%@)QvfZ4Pe^YrTt7wr%!f$Zm;j56ej@N|sg0gnSck!R; z-%8+{le{J4@j-E~3@r*ZYLOn?F0C(3hXU6)oB8*lUsv&d?BtYo0c5|mY$wUW2!tN$ z(^}|HzQCD65?=G723=``c8qX}zOAT?S1r@z{p6P=w=2&?xF8>GhOzZ~^gh&-5GH+P z_HBFk^3O!Bg2(G9R*fY7H2#F_N$%k*o9>ax3PEQ4z2=yu@+Xz4xRdWoj+0*B8vtK; zPhL334=x=#XF%>XeoK=tgHs?->)7Z$Fhm9@ut^cm2B)71E@7N>rtIz3pk6vNC}C$u zkS~8r_G#TPIfdyJEV#_ZLGJJ0duoXngsRzklxas}k*Dod^R$^J9bFhP{0Dmlo(&)TM ztT5fHRMK{Q*;|N6F^fu{@$eL}hfJz|Gf@cxrSmp9vIp&-aF&AFr!?#vslmO*1Nh6P zMy?wa+k4xfKjvyVU-h{(6@;O!)E`UF91hJsGLM$5G&6kIGAk)2Q8mn4;ZUy%J}n&b zNb*>x9+uE?LND$pIn^-zc3LjFf?XFsL3fY*a`}hP_htFK6%Zs z-YgF-me1h8-1x(q-(T99px9k($ZRvMuoik_8d_O?PRmA(!JMJhj6wWQH0sFV(D9s? zle6lbftQ=>UPT}2f(G z_MMfW_Cgdw_~g=$h=nusJKTZGHWw%%N@Y2P3gU^;sWBvF$6U6sI{1W<#Xjq{zw!es zOP@83LD@wAzX{7}K;kdV3HL{WK=T|J=uPe9%1c)(XqK|hZ4>q&9&X|0D0OJ`kUe6+ zNQ1`Kv6=dlKo=f&X+q#VMbG(5wx=7W z(@b>$;dExiiAz*(EWISV6)sPJAfU!-{Z(fiV*S%sM?@E?okPsVSJocD3%C6JN-O>L4w1 z@8Sm4+F)T?3VK$POkoH6Qr>45Dz1ki!?ymrXBP5=uZm@{SwKr73n!6x(|mmGEU&IQ zF$1a*Qzb^rtLu+lk5`?)D0Qek{HqlQLX)!ih9gQh$JAFg?#wJFz)$m*A8RXYaMwUh!LNuf1-fwKNp*upeThprGI>E6Hi2pa7c7 zE?!;^#0U;IA4hj{Zw^N{hQEmXLr2ca&BE2r+1<{`krqkU{H2qJyEr{Pa-skH`uP20?&SL1 z$;n}uua zh8&L||7ZNapZ)iEZ7Vk?dk-XeT}L~23Esb_{MY0Eo@MYq-%AMoGw0tA|9gV||2E;@ z5C1g*sRofha?+QxbFgy!vll^LVKJ`%^_%~mDeGkKqy-w_0-I%nX;qa#j9VIBdgEX3-S+(&wtts2=z4G*=w_w4&-smb zj$E~_WjCMA2V{;ZfWUw-6gqA;pg_UT8~5+#C&yTUiDvfMbjMo9J|Q)sLc}JqM;grD zm=+(2FR5Z{@MSACIyfG>KnS!cD?OM>lx@r8Fg-6p|Mh zJ7Mr^HU|Vx$dB02Dk!RwePIHL@sKL{Rj67r4Y0*0Q}zit<1m28Mu1>D2|t?a2M?iR zT%2qYK9&%$hU(=EDg?XUt;0(y;44_>2cGP1--pCZoHnZC`$9iO1@5N8I4~^`geoH? z8=Pd1hg(bp2$R}ja6af~DLfZ--+zXc{oV=Nh20pqAxn>yU#Zf8k4CaNO@5~Z=C-(J zM>z)$B|259)PGxqN4NAjPMyupmlfJi2J;=(5+Y?e`T{g6_2b3Y474Nvgx(SD;1?hOuiUIYyb`6X%!#h`FQ~YCDEKg0y@(9E%}i zuG(zaJZP=t1r+x$dDK1tq*ylmzWZzog4Ro;*nB1sa$i_!(B^p8j@M>!~6|TNfo}uqlG3CRJze&kXF*%uwTwg z7U0VeSOLxhn(U!`jSz~>vmyqO~`@YD^1cMJ{7BkjZyws!1^}1v|SUSXiuyo>XD5q<6 zg>Or7Wkrn^(qh%p5Y2_|t7#$rHX0eJ)W~g0@-7t6U!%EoY-BfJ$JpE)xEB z6FeIx5=p9t`}ia))}mhoPsbg0`$LONDtRri5gWUdo1SJqjKcE9Tm!l%OxW>j9PjW( zUaiG^{YE-!i_Tlr) zP8)gBhz?zz1R=Kps6Tnk(*9Sd|A=%*I~8YAWpzYw>eJZ0&R~(zMX0iE%$~xJS)R1L zxp@^V>cFw7!S!K2w;t(4|B+Bka)OWmzJw3B)6AvC^l4ATKtyvsiHFqK(Y_}%B;ii1 z&O`PQ0ViO_`3uHI{SlT+@e{s`FU&Nmzr+!)Qp4tBtOVVQccwNEc+-g+^$+e6omnI>q)DQhHc$iCn z10Z7Y=wf-6TAiTr^X&YN-I%Wi*L3i|l6<&Msf>1AZNbkSM)1;XLT^2gh^4-ODTI&?z>}WfYUhu#}7~~4l*jwPqmez=7Hzi{yUhFde9if z#SkXF3~<4f*#HRvmEVlX$gseJ9|_C=%}l>_bA!;FK0MiSRy3@m=)?Cu`p+<^=SwFF z9C-+LFDeTz9{$YqBbyU>&Q;mZNjG|pbf@c^bb3bMD};BhsjjoANtl5=v->X%oGF#O z)Nw`QoX<&y*d>4$7oE8LL0aVSyh5)S#T&kiOr4bJy>OVVNS2r@B=C_S`jB@_^5SLi zl#_nALgl9&C~)=#uQ|6%VgFQ_k8JFtcHg;ozNfMRSO_=fC(LF1%74KoFAo}eh}weA zIPv|4HiH7iVndqOZMHP-9i=xa(W}cQ(^5g98%Xvt!T0tjC)!Qq&p@b7z)R_2LcICR&&v10(zN5XnY zS=`+C3clKAuO`v*x>wu zS}7{jZIMeCjgcBa$&eU4d;P=BFED5z0&nW!BqMQ{)GD~~Vl`O@^HKB22imcS$iO8} zR(+XD42|V0S5fBKXp$Qa@l_)zq4XQXIxdW0--{=tLM?qupw`l#D;EhOCW8$Wl^vfO$c?2~lwr(<~4ZxEV z|8V4v-QE`xP03PCRRXE8{w}xkNr{jMP?i9*=fo~cz?WA7$&ol)lLT6*8} zJhrL(HLn3W@#2Ohz~ZJGhG~Yy$i^!BIb>*)K=u@BH18XJC-rhQIYqSVJ>PrGjt7Fc zN3M#`x)nyo8=m#H>dw&1BUR6ea2i3I;G5@M!ugu)I%~l2`{34X$eXSyYNbXeOXl`+FUFJh znzOi5r2khTaYw)9i=a&IOQ5q=;+9^H>JXUSQzUc%oO$_%93|Io7+Xi?iiIpzlcy<5^jn( zuV9Rfv%4}vTm(fw?nqjG-v*UV?TyvxPSf%u<${7FjAFSR0tUsvE`RXhTSx!8$>@;4 z4RvX6r*>$9#2Kr%tVL~ zZ4?C{!Pxy=nu_o00>+Q#NJY%@e7EgP? zkiBx@#_9#(0W?j|7~9E5wCyFW(1$R(V}+h?9^$e)cUn~vPmo;M6Ou#fS8?_0PhK`= z0U<);x^}dO=M(COu3wy(&JQ6&BbEiM<19W1MvCwe_P&!P0CF%>JkMTvyrubM`I9$DS8taSZwChHTWogH|=QZ>(d#TOdm#MNRor%wr!ry~NWO9vRACRXtyyL=7A5E{csp36y#l8&lMX}SDVF~hAIDmhJyh1504B@M^V1zu-=;%>pU^J-4AqPj2XAs78 z9}19EgnqAO9u`^8K(8{8uCX8tZoHbCq4l7ryaV4gX!Ar z-jv^e<}XuC%)>w^e~1Xxe{_16=pa~7Hn4sesf8r5{5er*1A8#9G1BRh_iEV2 zQroLt&bz`E7d}X4`jiuyBrzez;2@aXQlbUQ&<9tDZS?;lGdsKzDjLH267kX?{aG_% z(s2nHp3Q!RRDv8rUI+`q0wmxSKycVdO)%PXmr7=JNXfH8|uN&jKtn-5u2s!NO= ze&rWAfJw>BBSRDBJlx9)FI*=Ihc# zixcI~FHRgSx94fHJ8x;}MkA1*9U+<}bFOG+A=0X8BZw=eW{eR!f_nGkVOq`2p?Oas za~vfyHSEb_MJNjlcq?c2G)see&Z?6r*n2_1FTj=sfxm&c?Y_PtCT=p!H7tMFKsXW8 zltXRJljlLeY=c183CqA7O_=)goX8IA>~?o3PWwk*|EQzw?7X^9=&ZVba&iPV}?MqH>kJC$6H?!7E1gBZ^Zy>4IiKY{gLf6Eszn9}1va0R+W{^E-!` z&F(j((Y-9DjpUr0kkF}}%E658j4{Gh|RM$pssKKxW@a5S^k$A=7U2p}iH*KRkf2>9=k%(-Ebe#m_eR z2m~Z^(t-Jr$?Y^XT@b5+*-!MC3-whUJ=hz%9B7QMO(LrPhA#jgUP6?-`^wi&25i{< z7vQ-951AlWoSqbnb7@`8Q*#{;E}xdl9cLku*|R^$pr6eMf^H&{gwqQ`pG|16et+zd z-@BF^zLgx=wlB||hs#Hfy4U_B%Hl&r$ix@9bp?=^@121nqx>DKcg)0Bn5FYDs#844 z5WF{AXW}-tB1~`xp`LDX=pvd#PsiJxvY23FnE~#096_HJ3AGe%zW_mafbZDh=G?#J z=?M}Om<)P2WD7wHDh#h4mLuVe;kCoKGHqhSs|no0g@gu&=s_g zin?BugO3?yKyuj7MMfs={UpA}|I7V}F13*IJ?*1S9Y})+ru~>Y@o|YuBfCyc8xx1eHOkohmO&y1bF0(uzC4 zHa_N{BQ!2(hx3_2>zG;e2p4(VNk1<>;U8Q>oFbsGl~5Zhi)y$aETbXZ1wM3GQV&PZ443rkw!@$8C+-h{2DDW#b12&)w56RBa%PdjI706I}&BXg5wYh zaes^u;F!CJ&o5(_Cw!AC>Er2?tB*F|EDCQ-1nvulRHdu`GDHMl0stdUH5xPPsd1So z0(9SjDO3Gbr?ua$aHHD{HBPN;gb!dKw;GCCxZgrZZWOq_<1>HBqtr5NG_dX*QA0+v zMPVrV@Ubc~kRjtum5iCc0nObT9}czp(=ccF2)%h=qt}_qMR+12%gi-L`X6`E7X~h1 zeYsG;{;6EA9{EOE=SUb9m+ECG*gsX6G0 z@kMgmTj%V^BedQKuAh^OF(f>cQHX8h0Y5aeDw>^K-P+*jz(GGY!q<#3(W&i$Qv?_B zr8hO9vg&`+dLWNT)%$lEJA!a)*y&6qdIUpv70Kp#eHCLmrOm&A5ZQ)BZH-~SZj;8V zSJy_)V{j!iWABA5Gl=PAz+uB>vF??W&p)cQeF8+H+;`#`&T!q&ZGaW&#S;-)uRXc? z6K!8j8C5EbDx8R<-O0V`OHr}5(tAthq5CkU2j~i2S5%!(C8kAPH`f2hP;BjhJs87= zeRwB|&aeBX=l}t+E-P8$vY#kr`jS9&T5bqSDHr~1If{eYB=PBL<}j*L{P0pUxT?5QpMU5(Z-<&tM!=GG z5*0LS6rVGe%fyvL_2@o0=jsRDa_g89vt-@kx@a~5ohtp`nstbRt&n8~L}d&HkjW4P zS}@vBydyo>GpKQVO(`Q!Cy0d*+XtY(sE4RPa>I+Y7W{KaTE&Dwv!Y_#d{K!oDJxobq7W}7JkbL#)VsXCd zqUlK0HuK&%-A{5H2^2&k{C_(u2#SJ`nZ`^4rCU3tXG=d*b&lBG`q@{sA)*i9bW%j{ z7{Qj%K9@1Gr@d8hjM`Z`PeeE6709xF06?J^_+zwTQbItPd)I-fmCn_JZK{5_lh7dI zQ1?6@VOFUY3J)bfi@d4{nvP#PNp@9)8Ud!%QWt$!Qp)Uq8x$)Tbb|{P6UyKdx|t3s z4{g^N?0SLv^a~JKOr(`#L>x6ZHu4j`Q|_i6zRdWBDso!ug(CHjmWrsM121ZpcZiw& zF5pIdn$H&)?1%ts2v0GI$9h&QWoB=1J!c$s5V{?Rx?SsN_ph0F0MeE;^Rz`cajPsv z`s0sr;37Q4h9@w3>+3?C9CuZ|!$Cy5Ug#mPDUU;pa@9>dW2n;GzdSWeDh^z^zzuis zR@hvoO-{6hNM`I^K919oRw9(dMNo;9jq=S{bYAkUD+#7|Z41rV?Gk6m z_&^e^^4InesQwldOYoV?661!aRLV8csXg;HeC<+Ryuqp`U@b}+1f(?s9!9H!8tH>~ zlB$7U1s``rV5$aUsxoECVb!z6$}#?JuV8=>{*rgtN|IBWkPC3-!=nv<^k`+!C4&^- z)ZHa`0*WG`R(+M-mYQfBM5r&(OOO*Ge}7+Puv@KL*ErqFc>8yCcUs) z8256_y|+(~K=XaWcXmr(UhDl}n+njPf5!rlP);hu=b1*7a5l077u`Ru_=4_RP(aiI zxGPyWjuZj3Qt^|2TYs384%izGzne%exXKy5S^x?;zqScnH&L8B?j-kE{0hC)OP}1yAg@#Ur_!&4)&Y0a3t&f=A{Iq zEY0DGymP%g=MLyD@#tM z?>D-eK+65HJGm!?rDwJMvF#>L#eBUeB0>M5hYR+$ue^U6n$Zr>OE`5S+h4o88{Y7O zxFQOH29mVL>RLX-e}dpg)t zn^Ak(YAfzDZ+X)CZugD^D}s9T;?o`CP&k$*et%sMzwE_GV^?0+xj<)`gnz*KiZ2mK zWbgGTS_K*IQRu*2KxeTompGR@b;yW00J~TD5f{SDrhbD))jW+3?v57)xm1ICNR}5C zVM_1Z9I$Qt7EyZ%&hg7k)}F}X{$sSXJ%|(XjuEQGQhQg}PYF|Vn0`8+(EV&@-jG*| zWz>8dGE}wQv@ssXv1x2>W~N_Q(6$wZJ)8Gd1D$+C+%Hgb+7O2Sm3kqdmI?{naLVDh z1u3~0z{l%v?ic3-IhC(g)Pf&X%^+jSjIE7d`1NYFU@JXak~1M;00M<>ndu7VCy;>l z2mJn)6$cbH3A4L#k=WP(4~5^M9@Ra`{<@i-w{UQp#^CrBX=q<}?Gl6pIUu}}=Evsx z_t)M8gvQw;VL9E1oXhZr%NZt>`ganGaL)j6Ff;*?fZSk`qqSpv%H24zugpF#XraSM zfV4WQ-sH-@^`(ac=0St|uF zSnQirH|fddBg0*|vZR!eqa*$&;V1_!Hb$*5a67po{35<0_TgBS`PUqSv@mY}J*sO}b z>}trqUf}x*!&I*o;xZ59LDS>HLD}+8NjWw1&S`RKBaQnMLu?WBeFTOUzwopF`Je3m z&%puJ3%CgF(*$gYwh_qjdJbBapWL@ue}9kV#U|$;U6k0j92UEuoZ0vNjqrbt za738ms1qFwrGUH(OE$JgNJ_N+>+r(SQOD^`IJk>DrnNm`HOrsb1EY*~ zITT#=pRk<>TJ(Oi!Wy-(RYrf_dqmM|=R~nw3qXKML#L^GH;SY|NAmFU0yNXL@^pQ~ z)HiU%2gj-X)+$TM;~yVJlzOO9AL+UHTV>wQ_F~75EH9KDO)I|9uTlu!?~gy*XY-=o z?`Q+e%bjViWzvVcov($qVxgG}}}I@E`wzUKBblK48zg{Dtosmsqngb4ykv z)afn7Ufy$(-bc~Yn#fb)Mj+dfq0}|*!n+D6G5N&y>9i+yXnWnoJGCCw;LU$f5<$ri z*fXqI*z_b2?;T{xF01CCqhcs0q8m5DW7-fE;T#riWJhs2D?!^gJiC1amtn`HO5{%a zCgROwiV^z%qQipV=pqMpvVH)Ao12<|hVNQ8e$|ovq)clCzah&?+V$S6AaYXtxwZPt z1lREv?8IfB^7Jy9ePnp6iG$BnrT|&)`-8ZMk}nDL$nB>%dxvAB%MJNK{1u)N%hXq7 zp{W7aWjPpOfG(4&oD%VPxb6@AT}j~uwX@+4Z-po=3|u@%-g_8{TSZC#LkkSRlQr|- zO8TV#W&u?&{M6S2yMOAHj(gcA6-CB*5}ofUVJ;K=V|&7Yn7{Q^zsYQR3lnX}ovO?s zQ;Jzyha&JF?dRGdd_Q)Dk6mH$>bK&f1((d;qc*#E?cN#%L=I^nJE@O`Je3OC?YmL> zqe=AE*bEC=iY9Gcb!BXb{XYz;OQZDKJSsHj*u%@lhY2ODh{xX`FfZyU5=1_GX?}70 zQyTx0A;vhlys8~Pzq}JZDLRS3ZO$x}e_3PH13T`_Y>{pE?bd6>kM7aunY}6pOE98hor|;sDQS|x))QALW`bUjX9xe7fD{u# zJaH?6@hjQB+#L}CbfU0ilF+g>dEln4n$i+UUlWK^OvvU&iuiEbzwt`z%+AaBF+Wz`G zg`)+&d=u>7F)JQ0;nE?oNdYKT2_aDbAwHcAl8=y9_O$er|H5!=4zk$KY9kQ-!KSY& z{prC!b_R9>nq5h>`4}wlb9bC8=V7X3f}%bgKE@@T@vzt?4+C0nPF$0q9c*tzs$0Q>v?c$mhh2~1FU8E~2? z;#VFT$6|VL*c>!^_NT!6x7fFK*wE{Phoo$&>SLEc;sQskeaJoyljuTdyM zH_8so?6s|d7d1mXIDInRbZ3~5_k%@W1s#9a6P4inU|NF3ZEb(5g6)Jv=A5%w<>{H(SJY~HUT)TESk}W!vy*IL830UY1QQ?j>9WLxcz2dSEO6@y^12C?o0{0Pz1gT%hF{M(<2M2 zdeZ*T9D{Wks{!sy&rr&hXt`xnA?^uHHMKdPMJQ%lsjt0tlk1ua+MSAZ6Eib2{$LIK z>dH9KWq2s(vFq|^3pWI-*M(>~1A0Qx(J_}2irqa_xa$V?LLc&)nd}FEB4(v8f9gD_ z`^>sxrW!1f60&BNFeq@$&cKcy02Pz5bD%ZKRTf_i?eLUIFysm(VO3%6P_}f~5-L)q>n^ zyhi3eH#C2hU`FHnr9!f;B3JjcOa9>6Sw5A~8ndy{%Da@wWez)I$jlpUafoL;Gb*>7 z=f0H&TTlw7d{a+-!FH*0y-2^V;Pt7gXqo&)6^)~3X073LS@#R3YK@$#xNAqF{F{OA z3gm|R_o+F>DfLT2gIz8WPp>dP1>wI+w~(Ra1uCH>^^>SFC6_X!zBI{lqt2KSzPR`B z^17#ubvsLHI_jY$-e5QJof`aVdiSeb{3}6D=Ja4eLBZ7N5bogG2{W@`f!%slcUZ56 z=O|SjM8$a&{@g!~q^Wq{?-4xiVQ>`9@&gP-rjnOhBF$~4io$i8DWxB`9IObe+z#J{ z>TsN0>$Us-G{>azr60|_Wxaa$#LgwmdrzeyRM3>LdvfN;YKx7Q158NKdV#p)1rewYqiS-Z)b2*l!?%%$Cm6o)ZOIqHcf}v76@Vy!nD@C1kyw0zN ziS83XIHq8b&Pe%zYN@$mk%fotvww~p+1E2n(hYdG81dk_r0?8E;oGF%NXBWA_bBgm z3o2PJ2$E5j({(ZG zHa6*YhP}eq5kcX2GhmPeE>tIx7vHWQdst*2s#S^a{)-C;u{X5*knnCd!)dpI7N4}l zA+-;dJz6mCFcz4nx|-{z^}zNqMoOnaG~^p8pYcJ zX{t6arI;VFyMuR?b5)&asy~Db!|;5xf{S&h1)lX(+nkxOf*uypj!SFogJ!@)OawEJ zEe2(tMO8>8%ax!_Rd8_PgeO3vf9D0sDSEDrkxkYBYOf|=kefX@%d0IxZ|C^**ulYb zNV^hTL-i@C&Z1$+sCVcIOZF)&50mEDumSQqW$Oj2%T!xGyzMp0#J6l0m%DG-c|#N8 zSKYQCfOp@|-#|nmtw0iwto*>+^ZONs`M!7w71{e`vHNJHB&xAXCP0U6pm zkeLiqGHW0Hy1OUuWwhU*H0JKtkDosI$+mu+sj(Rm))c=u4H9eE3-3BtQyaKD%Ipk# zW&-C4&x->`F#((pJHIpFT36q@Zn6;w`}q53UYwsVJ1}#!hx?dQWv=BIAoBTwwT>v~ z;FON1<{PmD1q%l`W-Dwu#2fV%A#r?!i71QWG|B3d{J)s!v;=|NRk~RF^IK2Och4P8 zXo9XY{Jh#)PB!>=pRWzu@6>26xQ>Kg+&k_bu~>%01+XMxWtdq6@orU^_gcOI+dc&x z<9viK@{I~cZqO{Y?Z<8VL@Rv#IeNz}2@^wY4}d^aP+-~M$#D6zkV6gA)8*!^`%VH zXysrJQT3@(fJx+r!=`qOiRw3UBp>VU36lsu=GKJyqY~}$5G*V)l1&);L`DMN?jWx$ zJi;4(3C)JyzPkK&sVBuD#sp#Q$gwnVIfpL|x#gdF*M~JuFlZ!rFaB=7y9>N}+S2i@ zi&|Tb8EA-snrRY&i8tjo1JOV2<}+&8o2;^EZ#ZgL3s1yy+Z5ud>T$D~C^v~=uHoZ~ zm%QpG^u~FenDX$w9NjWx^QxW`MZ;}()yp4He-GX&Omt>xYSj5p>`?mchTLwPtbk{ z=d;T^`VbNw;88@$frh2}>pLB|P+sH+9#C2f1d;n!bZf@NP#{IQ+N9Tkhdq#F?i`D%u6zW~rc1Q?j6@ z#%*G5=3LBW@VALc7S_%{q@H9&=6NC}{jBy)-RJDdDNNc* zC0?UhJwU_qP4gAk;*rW_u7}48dBDS5JdvOA;Dfwu*H+epq<-+(F#2C(c)ulHYHs2!Gy4k$M&D-p6 z5bzsgfE|PAl=I9t_hqn@y8RV}%i*K%C?j;?sjpw+?P1$S&y94OO=2A*}aC%|GInaI1QqnH2llD+GXfx^wz0q>5{Je{VK(p#T4H+X3WhquKn(zZ7vG5h|iGp`OZu3V-S8S}{a zGp<_Y(d5UK5>ijrZ==pY?*8n4tEuxWu7TH4Zs@=p0mAyLFD&;wAHH|SKFWH9x1jvX z91eEY%sYC=CI&U*FKyZ zk?4(P7lWZPqlV=&eOyNt=)~uCA;urr(@Zs-NPtT20l~) zytZoJ+qE}rvy>?H_v3ri;!5j|x@azu?7Q=XQoz#0yNbntZ|ZnZK76XWx&BMFkCwMj z?oVBlRO$JT#`;sVM$Z-}=pPi~*@$Ahd?e6luiX2ryIk_2oD$EG&#K8dzV-5W5Q1`p z`KS{=IKN3l2;Rp)AOgDaJFW=2eSEAcASlZOF#JShpk7v5JTtaq6{1U&qOSYGPN6uw zija}GJ0~tk!(_K)`mT7~?Rs?6bLI9$vb1wg-LRU#zQ7P4_d;;-jUNlGW`{re$vcIE zb5TWltH zl7A-OM^+7^HyIy$D*^GR?taARwH_|VWLaIj{7P}`T25q;lfE`peqA)mT#R1)7*)5p zt@ugTBq#}MS#Kq+;L`^#V`ct8t-4%D5zy{e0wfNHsaNOL*v(KuZ^`MCN@Es$vYuxF z)`A8&^zEPPNN+H^BbuRQ*cD=$;-40#)XN-wOWMBx(+T7THQQ&ZcfJ`JL{^bM<1~;e zdyf?_Ul!AHIm!Qe%#3(f2<71WaA)h(SFeMPnZfwT%$w(r{j!4sXCkeu`P{kcBS>{; z$Wb3TST{_qjCXMeqnOV_uQ2`N(%nO`P96+FxD#9u#bRGzu~JC!L1`+#D9idSdEms3 zTO6S&Kl+7Qo5hmzx~X2^4&gXc#Tx_E{Au4@Ex9l$e$e7hy4l|D%JCKaOT=3-bkj_i z`Fg6Ejis*mwTqW}Nes{FcLFA4tX_bj+HuF^AzG~C1WEA^BqzP763n)aH{XbSGi-Wp z@z|H$+^=+CXH=y;Bpq`JhIV0nQD?6ac7?f{;3f6>Tzqp_oc_90tVg-<+#YN@h4xAy z_j&PauJ99TP4N!SEhGAM=FEAXNuHwWVg1bv}M>$Eh9HuXySG_-0t0$roe(w&$OSrz@D}4ymTwF+he_ zp!S}^5~oR@xCBsDC$i|P|JisI(UEuQeA$1l(D-<A(!YT^(gu?D;@F zd2ne<*!^z@8gq$IAG~!zNB@ty$cDkxCEdg}WIuhs|FItTHkCH*HtF#Gj>@hN8N}eb zeYxXfQ;!`_LjQwznYJa%g$<4Q_%0_6^S#M5)PXI&H@dHH;^NS0f-|!-I;QOFy{zt<9iVQ-?&z`d6L64}p2b%Zg$qt!25N%&?$RK($7zu>D^A zuxLbcjS5(6CB-=wk~$exSDMu@yZR8gDw=*_cDFLX^6{_tt#mBkqzf1z zw#XxB&vVcL$2_4Glo)Fvq=mIqJz289`5Qo)Ml#IeP$y(^{MAp`~oQ&n@~J;hT%3UL!8hC<)^B}95{O!*{Ye&|Csk^4PTw4ZEx@yq3U(_WM$6%kLfJVNF|@Dyo$`IMNWDrQ(q zFuPgT_$iP~zp4uwd|4Eh`;F*D*?aFSk>UF7nXG*cT@i=pFR9V|c0=dsPS{HQ`;0<_ z@)2yCyiqiGeR8GhyQF*vLFu}cTV39s&#NmLAc+nJHCiZ4BAeH;z(uPmC35xe_C;O-%KpG)+aGF+Ma6EG0>Y#2^q{Y*;7|Jg6Q``2k6>tR7I5=vRIjR z8T6Pyb`yLE*A@5uKW-z4lzmz%^2)ak1h%WSO-sM{UTDC16LeLqTl=6^<})*s$>}Qq z1I2%C0E?xh*7>J)XQa>U^gQ}vECW{u7(XIgW<(wu2K?2t;$`>wWoivYGDXl#%{sx% z8@b|f4AkqZasiqGA$z*y*|F=8oipKR zn>XkoHT0K-5AYJ^_ZW4)J^(%EN#4s}mY?#RSxX>-Ln4PYSCS@bK z3TvePI6K0jEyNTw&6j!gmNLP%PAd0JNq*WJG7~Sp*jP@Q@aFv*)0{G=x?0knG=+=8 zo2N5v9s#Z87thjsWkpL=ftD$VTgBy2ly!tBdwaLgtgEIA`W((=tMN_6%Pt=EJLtgz zCT)cJimwhc_cJt$F#gjcf#@Z+1lr0pmy5!N{btjiMEEPOt9zW$w)H&e|9w6;DRmuJ zV5awqv236augiPmS;<|a9)or&vIzm&LnqZQw^CVq19@J7YrVEE)>7h3yiwf}I@5t- zF>`&pl^t<;nvt|w6w z;%!s`0545QOCOucP$754@p1!YYgw{A;>{s;J2+syh`@R7DAQzyFNqU-c;}zL1<=%= zswjuhkrV`5g#aii2KUZ?@p0i*zJ6!k9%H3?;`0zQ9WGFr$AYB23CpX^%SmQuT;lqQhX z{LUe){4>Eg&5x|0AMYFr&GeGlMt3Dr9duucT~>h8<{Fk&vr``T#CE<(I3LgX_=HMX z@7zK7OVNe-IESCgi`2;qQ#}EDk!igE)_Yx}cSV%P1nAEm?DbXKQa(X{U!%FI{NCiF zM7@=r-Gf8vmnHVB4@6&3yepo`kFCFi!jvAB5sX$YRUXzj%Ra1k(a8t?9OFDpG|C!J6g)2PYqy%S%#z?Bp((WdL{Z!ep_&IP!bDU*hGcYgx`Iw3ch82fTO7+fr z^%VJ;fxwP>4xL}A)0gIFEaRiu@@c>Gp!e*kpGkbGt!GkHl3pw8>#EMbc=!Gdr-KUi zxy?)}Q~fiQEgAKd5?{NZce>a1c)(rwCeMYfTuv&a$!#Wjx?cHwz2o#rW^{$`?k@XE zk0|n;haZTlT{E2-q;JQ1!JHfu!*5gzEk(e7(gHLvsrx+5Lv`Tcq`h^=g8_a2{HU_d zVBN*3o=TYd)=vIsG6`c#d+{eBfs?nz)r*bm2}$EaAJgGA60q1LMLIv7r)H#0On6i6 zZ+;$sORC*{Vo;K@e%M9yq>*zbyv@Eqr%(Ict+Kowrh^Y08aRfT0!`nQVAoNq6#T#$ zZvSjt{Kx0d*D&qhsjgJ2UuhQhvD+dKyJR1@B$-d;$2QTHoO3EEwXMbds5-2wR8Yoo z>+XA3!%nvOD(7+2N2=#8KYJTLd%aU;V%Qs`-h(}x*q*4y!no!}0|-6c?@xLa`j(&xPA{QCZ8tz_-&d&@O*&CEUdSQ*s&`6jO!jbe0D2zPRLI-{pS3s)iA`(w}y{Ah5C5W;U-D*`*z=QqRcm#LONPv~0pm1hQ;OV$`rEulfX@2OkDlX{nX%{bE`;zX`@ zc+~SqN^(GDBvAL>JGC(4HU@{vW=tw+m6o?NtvfK0rL`WxrWXnvtt{Q<2#Ppn5+C_B z?lm($kDyjCcQbXlxpg$B9G%>WikOsmN%A z-;zgZe0Td0L1|Z?(t>=P@#hevt;D=~d&7~V1qDzLu@e$v$e$i*q)P zHd8biCUrf&yfsjkx38FAoLlH#8A00XWPVi3q|n}bwf5{xqZwD}%k>Vt<6Dke zgW|~jYVldwwTQUbKjIDz9J75 z1DQtO#$N-i?6o!r1ceSb)Mtuk0xxiu(V#QOB_sfVT+;}W0Dht4jK^z1;{6~XeQE!(xdKwpV(Yl`qwg?Xx2qjUbLJ3W&P!uaCRn`-5hIdmIs#7s~d+we-9o;f7QL1S=A&H zrdlS3?))-#0?;gf$|4+7nQS*I{y~N7L!%J|6_GgTQ}p3*^-6J}wh+OiQ?d8z zTidtP3={RFGkE;jP|RL-$3beQFYV%f!o3RD$3D4rXin3m9A}jY?-w$3b*~B+! z!R$jK+C%ov5^FRd^YLlF&s=CaNz=DQK7#2J{uP7w?v4G-1`k#81-b;a(Mtk<=t4rF zrqrAG$}FC@+EJq z0?br%ESK_B11z)S`J8D1L^DC4Fxw9HNdH^qKPyBkSTB7JU+xYP+_39?W&Wk^Om<(v zlRza=})tRw;m#qQtZC2hH9C-b!~Xcz84tIc6T+jj0(D z^-iqX4gZu^UF^NZ1>nzkNoh%yE->R<$9{?E{8yMY?%g@Sq9n64^YoB3*!zw^_VTd}&Rv|1~+vr?;7*&K0)3v=_=Y6iY?vx0kNT*WXCW7%2 zo}8Gdldko_^K*@|4M)2&A!&vCI}0YLoBs=~{MZ?SA3uCV)5GEeXmS!%&XSlI4-Z)_ zFFo`rA3t=b-H}ljwl22_l`l0ac5IB-SA9uKx!wDtp!HZ`k*)XpDkYTNp_czJzumwb z&q*mZQIK6S-_zpk7@|2u`(D)D6*IbW!a)xE)YE>cm$Yy#clXeCM3&L^0$ke&O}>%# z(m9%U5-x-|J!-LrL!D@Gyqn*l5$hCcAIx;~jbKxiwcRz-2%Rxi(fKgaS*&72`iIU4 zlIFu3&Pvz580qxvR^wfsAX!vSX1DbheSE@b60o@H9IZ)y-!kut%IKVA?DgbYDGF*LWJnmTG;DlRrON^f$NzlLekS}8MYQ9eqr(#dtv+Lp95~0G z&goovU`K0=%^WT92jP-<$?ruiMg5X)vq;ny%)x7_CK~%&a{ILq1?DL-185ia@z1Td+7*NROl%~S}fRAW2UO34(^5rVTu~L&{O0y#h$fDKw^*GN` zbJJDt6d1K3Azhn(hB_|aFT#AZDUC0Oq=bb!?n*PtQEl~CeeUls6O`kCmbhQSdTQ1g z3+dI{x!Oalx{OYwoL}k&ihgcasvQZ8V|QCg&T2;b0MW^r5CGbBl~<^Oyy~NsbEC`V z><7B9ziLp_$x0Z0KEo@lXxKK5iK0H;eLAPBwV49`(X$Oy_FmNT*>n6ezoGr5SX-@P zOX26O9d~1c4?LDYM}~Hv1Mh@RbY7Z3#N)?(%NT&>ki9reP4zr*Z3TD}LD3qp3? z%bFj|rA>xK9To)sZK!`>(i_qD<-Pywhhcg^xH}*Y<_plK_({5}5KVwno|IO9S8kr| zRo6pRIUU~rrPQGSwZe!2PbpR2s7W)@t=h;6)%&=1AsnxzM8EFWd`cXXg5%!+mM1o1 z5B*Xhzm!jck7EJwmU5f&wF{zQJ$UxB7$CO8&^WCKJ9~%LoijiTf6Bt12$ZlxcPtbx zdR8^Pvy`oag5i%5Ck5iE<{?Rev%F&CU22t8iOi5!m+pmmi~3lfYn4UKBMI$~Wok#G zZ{jMca|CPXW5?IPIKx8!MS`?u1M#JOyw$mOR_o_*;YVw*Mwf~9TrnT8Cxmsa`KGUV zPdnY&*8fX1+nd1A;2XyV!#9@*DNx_a6CId&q(K`;`txG%wA-Sw& zyLjumX)6QWC@3}>Cn}W+SP`$-x?7BdnpDP4%|?`%=^nvFOl3&!paprkW=oWSab>Zk zlJSxXW?lUu1>>kbS#c%Jlw3RC5!@#&7kT!4jcs5f@mLyDD)OQh?c8m9(l4MtX0P!0 zR%B1_eIu{~+2km=l2U~@>!0~N70VWsc}haC8A|FK^gOj=tjB%xJiP8L<;Vszm>Fg^ zc|8Jhs>xdr`XBkC7!B4X5liP`;s>fXWnFLl_au9oguGGV$yAWwK$;>DB4^)(Q6PJ- zK5^Ri>;GM&9k7-vJAg!yzpotCz&ln>CHuAPe|<2%a5-iB${9VjQ8HqFN7124^42qo2kh+7md=2T|EER(g>%*1@23X?y zl^MIG>fiPE$7_45OfLSkV@Z9`jC5d~^|(86K)lr~_e&r%N*ZacbZ_Ti<3_O8JRCr z8lGWeb7H5DvfkvGPa8cV#SJkuL2SeI697*x6XKjmw6P!J%nO(n!^1K}}yVn>fshc!G+ z4mf#C#u`O=voHG4Ct)>kj+ycWq zC$zK(W+vo7-`{80cg~n@yV{nwkE9VwR5679)-E{UNG~?#G5C_r$=-JL6CWNfO0MY~ zO%0G^V}uScI{w;dz7DzTjsDcx$*@cwHnbXd`rH>1@VweLEZ(<}iPrqr2$lQ4MyRR# z@3F^c17hYke@tyzDDbDgiNFQhFu`4C&OBz!ZR1G)q|dzHmn6IdZ@FqZLK>cZlqX*n zAfx|Dv84#8<_#Fj@<2=Z2Cg2kS6WtQjfpAc*m)|e#=8RLt zAv%PDK~8=B0?C=D57p{I;V0>|DUSS0o1vqJ}F+`%H|&|xQm7sfJItzIr92D zr(y|Y^-hx=roflSZnH-Skc2EkNR1A;*roA;&4#G#pHeky5!*3VO?Uf8;O-Vppx3Rt zMD+RWY?))&W6xFYZJYg1o_q`;#yZHtfk>sIsHNB2Df-Z@4~Ali zP9Ths$*3;WLl{~9W~+(KnIWWR9mWhEbIBYlGDkve`Dufz>0;FFY19&HLE`5UOIyBs>)ZXaT^{>uQzzp zw(6ko%&BokPe($I47z;nTx4Aswe3L2JU4eexz4Sx8u>>QoV+ovMU}Lq_V^$R3H~Y2 zZPMsEvyc1h9GxwUSY=n>G+zW4z;BtF4UhAkOzzXez4H+k#_PVYeXtEd6+m zX2W{jnZ6XfszElJh|};ES4gHG}Ya-yM7FC#2l{7qrf7Z0VjjqOcc)r=tcA?i5H_?s zp@wBEg@zp&OZBN)`Ler`^s;S5jhDPJ`&XGOz_oI2P3dua4DsZsXES7~jyZh{M-RC* zWpgXm79|S6DQLBGwvtCo<`riwp%3lCOO>zluj;&bO5d0RZ|0_A-PB@FI!|>foDt`8 z>-mQh6G^ zJ56L(%L$aS0WWc_(R|~IZ&klO%QYNo8P1L=fZ&N{9KPgP=(m_683WtFl5H; zF{&ycjcCPU_QLgZXTRj|>%m*Re32?G^q2=xPX35;xtl3svqb0 zFXKWzBky~^Hb;QxwCcVOgx%SW7pt!6^6FC&bLl!MPG6a+eUE^se*bJ3pA9x^8EJL| z+JOQg?SRPA+1Icu#wx{GPhmG8h(yTe&2PzbN+1n7=e6=F1LY(%+JH8qUJ`#g+JbYC z>^)DH_NDJK$azKolbLjRjoYeob7ce3x9X*Vp#CPFVdc*|N~b@~<4oHec>lOB zIU6i_PE%bs@(@E$tPLT~2V(Rr0`_m<=`=*4y*Duk{s=ymB@;w=oF+N^3+gCQR%6@x z{k>Nd?SLM@!?rb$S%i?UXXtC0wE>*Jj4lVySgHxoC!0`x#f9m>d{?rU18wSRoEPQ}x~^SSHTOES zpi89#;S60uH&i^j*cxMQZl4xNALOeH1qB$!!&HJptZ|OvDr9qZmNAg%Kp6TsMO7n; zaGkWWLfqw>uGZG$iPfR=5cvYA4Kb9WWjcAkpDZmi-U4lf)4VyDVJU zX)-m<`FX$0*#d}{+JX*X0XUPJ?9R7P@wCERRPMTq>~GjBoDv3qosfH6g`L|dC!)8S z6P>KZGTz0>ynF-cl3lM0|HLUJ`CIV$lJANScLXDsj%q>@I7(0E%bl*J?7zX?Xb{b# z2&fx<;aDXmj_P)Svv5Hnpv){hM%3Ob$*TN%Nji;uQrm|9^`c%z%t(DN66-Gq3ZnlH ze+Tkqj`&c*lS+zh;NlL)d%rNImuMsoRXwC&sNpXNRjFfM58hLuqa*Cp12k(%i1pl* zw;>dj`>>h~>b4WMT63fW5=Cu~{g-}=fNU_}E)OmzKWMY@36kA9_Mbc6KET z{b_eYd-zQkhY0D}E6#LiK%~iF1L*X8l)JFU%$R55(84IeN4%dw0diXD(g1v$URfyX zqwl)qZH9)YRl(3+Fe1|*@^u-*7u?9ZaG92sOiSQPKTM}vspwxPg{*@L^%;K_C*t=QUndFqj!6e@qf!Tu!LKi z+$`LZ1b_VrM9i3u#VG25N}qXWyH_VMhlo!Q_d0FVr~(HZ|HYGafcOp0;=fBENe;Yz z?zcC_5HCxdWRy6gqo5ej_{NsbPqdUD;Bo^pH1GNIf}Nclq+5^Ky#+16%I~}QheW8> zo-b57FDtF)$MS;t5yo!apqa-Ik?ZCJW zUQUySX0MS`tjldxtdR*==WNLKpJ44v>qWhzm9HB|?@OaF0e*;WXRQ^U#bW+@Z@@n}u@zg(xH(M~P2 zw$guO$)UJBnKW#0est+z`hAW}zv*=$JZ9O2%C`8h+Zj~K0lY`)wNF<~`>d@G=_BNJ z1GFo5+c6XC6k-UH+O|?EZKl*lm4MwLcxv1C%1v-p!^g*Pwo{%qy7+pUFe1y;In=W<~$$$e9gaBqoZ_D^gBGFd3VD4tJloddrpRO==C% zAOAM}7s+&0bn*(!h!wp2Ks#Wois9!{!TPU%q zaGR??8=OCKatD;0hO?&DG-y0Ixk;;YLc$L5D1yAn2&^7gS0qTNz>-7VUxV99e#+c= zBq|ZGY!KwiJa-B)`T1YW96Fv07Y-2zZ)@rn<5pC$)KFAY1gvb>S<*WMeoY5MfEU4S zYt9=Fb=r2@-<$B&{|d!MA{edQ;pL7Y_{V|?9K+06N*Uf!<$blhaNl|im#`?*tU(X> zlBS$CZr8npLWiw_hiP5(1J!YUk*(0eU154TaW6U3bQ$8&7#CU zRi<&RTz)MY;HGhbWP8xUw5}r3t*Fo6xUA_)$Q}O(3Ws`Ct_a3Ds<3`Td@P(GratyB zf}qA?YmyIyi<=#0gP=)B)K;`T-Q1EIl2_?W(y#hO;!2(*W^BRDrWza`RAtU)0JoBX zR_KGgNDM12vCc8z_*QHXDj3B{Wu+Fdh@ii0s*wm>O(H6%p8MQ|q z!BQC^fuJzuq7~|`&&l~#z|~Zh#BD<;W~7dcrm3EIPJIaX(t}p|m74mRsfj zku<|MGhyDIYMiOis-k9(X_@GXTPuGBEk*i#t~0+kXPSXSfiii(NOt@epPY{Zceqdv z(|B06;%x#l8tB9V4$0S#7XJ4HElkl?zHif^?)uul zF|;9A71P#%g?2b=Q_&;R1ga!;o zn_yrzMx)x71jl8htDkOvD=-#)TSe8&X)CR!58Mn@w*P{t=ybW zjF#ndm6kc2NOhf`mdZG~SLyD{>(d%O)XDGGeT{U;1j@`nMygcQcnmiIVoPOv(5)k- zxKL%c#dkgP&ql7XPEx9R z55{~!XyGI{3K|A)r9y0>VO~q2(HSq{$q^EA-}Q90Bk34v-l2zdOBe#+sz7{CrJ!baMkW{ukM>ZjFlVVWN+D_M*9Y_b0(P~@xlm2i zAd7hxJi})xw1@7yzhmW; zqW)V++}oA z!BG03Bglt`HkbN&JbW)My+>YUnQ<)b`TUe%P2MO~nNXUY9dGcutt`UrPZWdB*>!gm zey8_X7O>2%Q{LLR;!Io6kx^HAn0LvUd2ls}VGh$ZrJiA_K2%Ps;#a8qMML~VZBeG6 zzlrf%q=!3cW!aswGBESV7OCFvLdzOdZh5OwB)BGxzGxN)MN;Y7(V@$f%9Q)|2_y1P zJOWGxxx~3nu7#vXq&KbCVJ1IClX1@k9uh>m3bAT9?_t%mVrHykUR1N zOghKEvm#I$y6ypXJgpH57#0*1q!xxf`ktGWC<)brWMvIJ1(0`VbnWk|uPiL!a_zeF z%Gb>Ch|2r^!Zzj*B{5#|BsXv}HZHN=OHHx(Cc^$o_c2bH85RlY5-rTCem#hCwF)bt zp@#!%*Oc!0@gUc-?eqhSiMg1>C;NGed^D_Hb42Sv&+Z2^XY5Wk{uM-q4g9a^Qfpb1 zc`IEJj)8C9Us?}V$$nhuipzGJRnw$^7|2P_-27h z|BK7t>dZ|c?^q{3cRKx#+YL3)u6GOSIDV zz1pmCkYKSh{dW|OMDkYn2Sr=j{vx!2d7~zkg&2uk3}UW!FQGSNE-NsDPMF^WLkP>AMg$)ba;m z_Fdfq^h%GRP_tT8m<;+jv^zuC^r3HQ?!6%R3~RQ3p}K^%!9PkSEl`sUQJmZT2KU$x zl0hd#KcU_6oZ&Q_zB2PI@{mJdMoF=SG?KMM!H<5o^99z!6&a>_&6(SZg@p_NBt@cl zm}kw&U4I48aQ0W?Z$|<_&$%1YA4t#a8xN9J{@~+HwsAM1~_7l(7P_vx|Xx|IX4GJNp-WD18E(=TQRxrM zs6(or^wNVq%@_ak(6rXOe@)yP^9hUgNi0=4p*P`N#lMU}Ezx3y3;V6A4~vv0z0N72 zYs}cAKXPt{Ukb^H-KhE8UzXZ2(E0caPmnub;hpg0ephS2SV`JY!&Qhw)z(^V{BZr< zJy}jfT_g#K(+YarelXc=q6k+3q&wyGU|df!$Lj+?Mq@Sxm5g#8bGYOY!8SDt8M3*lo+s zDJ)lODX`JmrW0yXa5(V;97r#_|2h7mp6!HT+m0bW1d1>8O%1pn6FENi7jnO8&aTql zG_?fyhmuWMkV8}%EZLJ|7HhSrkW{HfheiUKL0QIiniNzu_3WJ~uH*j(UOGo+58Zc^f`5JH*dx33!4O=@!*&Q2I08FJpGf=;O;2a2vRM+Zi?>S(w=E$?6%F6gXwM$s>4;MP?2{CHMmm*fB2#< ziaPw&zivM{YtOL{RB_Ch^nTain?p4uM`noD7gttKyJKm7(ReMNqq6%10{W1Ti2!lg zbeMALQ~Y6-j^wz;D0|dU?8wJFY03rD>KNCU=;XpRbGCANJ+T+S(WQH0JbBF*i)Bg5 z7MO>mqrzQ~Kn}F;hh1w&xHBTq-Uod9)|~3fM%1flDnRh_&->oh6~qqsbb`*8eRn?b z!X|haucKu?9^Avm=C0f8}*28awen z7N)(ho-dDgTo2#wm`x1N-sq9OdUI=tK!-{C_eS5{74kvCc|K_S~$AP)MLBplO?Yh3&tKx&g{2ssqPbIZosjCM3dMJo- z5|9mid~dkl7f&Wx2IzPME*tckAP~S~QXo)>CE&_oLP3X6L_&WH8n)BM?=tlzT{Q$H zEdg0B!dYs&;w%SzsBfnAJVvg=NrAi`+W4$<*tAIsm$>5M*yvLx2p>NnY<=Kw{qzxv zE5oED`0gGVOe5> zK6r0FZmH~sbWBGT_DI*?oZ6+0p#=#$D!)5<3LbxNxf^%gx&K#wqo=A7osld*SY;@D?H z{XamPlq)(`6lT#hH(Gsgh}@d)!e)JJ?-=>8IdK%vR>mYXW8Md};e=7ShDySoB224r^m5jdAa+Xa~V8mX^$QaerWNX#G zivUP3&yNiXV2^crdalL=3O$y-!flVcOkAD#^qE#?xZ#O*zH<0y~(eYB&t59Mv2w-9K|PSKN6t=_}^Y{ z@7MXrw;3Hj%TVYKZ>1S8N2Aye7F9kld$%Dy|8}1^kjqbxEr{VmHcu9*8Pds_T+|y) z`?cIGBSxK%^>G+!#Sb|D?n$|M=)L;hyZ?)F|ECRek_GwG)p}l0%ARAvr<+p2%4?m} z1a|p(q$(#k1igM;!}P`W|GqziWCJ(E2(w4}S{rE-t zQMec7D`s_B_LoH*whCrx$c$W4AVEl#z{P;@74xwcNbJ4hvw+ef$+vpgI=bC>+u-yRQh)8W55Af9m@9xrn?F?-r-r=eM+bu?-wI3{kKJ4&w;o=sb|YeaS}?9sV%h<5F^0pKsiga z7DE7(cYIPzRnS{yNu;&)qRAgXmMN=JDpA;gbOK~pE=c$kZ$DCWR+M;Wh#-031RM** z$Mxrya_mTB@YjxA@S%#ktM{4t8(Ly89vx96u*j}4Q0mel*E&jNMPTb zDIRr&e&jlXt6QsL1&~YYBg{AhLdjgS-=eCzUU4dP`=R>sqgcdXena}C3=4mnfLo}^O^Jnc2-s22u5b{A>~0tR79kL-cgz& z%UaFI4m{udA$2Q;dz80p1b&|se0aO*wKa>~lhOn^e(CyBvrwER>a5dfHuy*Lr|D60 zSSfy6qcMN$(E2%wPCeW36d4mn;Dx>ns9T9^NX*GF!FbP^U8qStyTukl-VBk1L#N}p znU#(mGmF$4)1zfZfv&W!cOWLm@13n?F&9ePqkllowy2gajpBVyZ0rmA@fsg>Xk+c~ z)bJrAzb$LeQAxfguU0(zjaG1}D%EvHnapEG#bFH{%sP&uqe2oy9nu@PK;m7G9y1k1 zD!TPaM!ziWjD$6D^a{s24r|wOjkn4pYS^f}xurF^ zmrZjAeULzf0tgY zC}Z@z2~n0Szl7W*w9YNr!oA2XTwoC3=a>pwc{n-oe_UKbqQN?xP0uebE*{f7GKH~< z5B9+Lkt6f~!_D*A@vs_=7ZDa`pI-&6q|h!@=<>Y{eVp@C__&KhB})30PMm#->*S3f zkK0O*Dw@c5RT^^t$Z0=0*){emSiu(B?i6dmd9%CB>FWe$<~cTViQKiSi}5fHT?Ang z9Dd%z*lEhI#kO!VM9{n0+_kUvn@C;1_6`_TUTbTgra}h^mLGmg-pHm|uq#f!e$Ha! z#q~SMXz1E#+j8fG*QK?AlY-kFU>!-cG9Y4hSD#nkUx-ZJU)y|WKVoH{M}`tT7L$#Z z$=8KT5N2Uz%VOU2h<)TRq_kG>ySGQkO;%1yNi>Su7Bv#KYESKW-K8WyVnE!gk9+A6C zjz8nse0*A-_!ExCd({l)5C4rw)zR0V>{enLlr#lvl~W-y{>XE4JX7>xO@h_`{?(jx z#JQuwN_Qb9f)`~?f|#E{t)%L#s6pijaVI}+)}L%`o5+uik_g;tMOc%`X1`C%W1z32 z!yyV@s+?F{Hn7LH%_Z$wbl|VFWR-<_-0$>IJHqdVRLQX@_ z_p|P9<#Hfc7Ngm`>?dbd>S=Cd=imB<_p|rEvWYF1xB&Gi853Nrgz~5t0fS~)Q|(X7 zR&au&V@;%_9yM}?B$5&qh~g)gAJwm0cw4gK>=v-T8?V}w5}oXb=c37wVhtKsBT+<( z+<*YjwrakfeTeC`%F0Qv*9YG~ZdPsJ!I?zqj)odDg62BxYpmXwoV9P|xUWgyNqoe*8Y0beWEQ=K3%U>hn6x=tZGPqmupR@iAS~ zlZ09<0>;X@gFN=U&c<}N-L(aCATlU_{qo+{{#|(%G)2Jp?X~G^S4;72>M2(jYhE!q zDIX#Nr)UTF`$d!+d=|RMFP+8r-kONwB!sY;#3jU-9~8LmP1zc#5XbK?Lz^1Jy$T zrAML3(cLzPxc15XnJYfCyx{nE(7Tj39LR#25d}%AMFvFQ6V71+r141`__%-^pz!6g z%701n4DTpU(^PlF*>`5OVQ=Ru!N0z7+Cv?NCnKCFOkDLB&C72cl4XDS59RDfYTdT_ zW)kVnmiluaMGNFMlbgH3+gcpz>*AYqSq#QAx&tr06Un5#)*?jBoZd%d{hRN2U|^;5 zRZxjmYF9<*5L{$Zm)zkJBJQ4Ba|DbNTBrgcJNZ6u$~F9=sXeL7RQ(;Cg5ki@y7;No znvZouJD*vcT9q0m2fZn$S!di!rn&QP;yhCh<|SAl$w3fGE}UOOeR@n^q@I+W_{T5F zIFuwRnh_u?Htm}?+1(lP2;KZluv3X0GL&LL4o1w0Q|I_gV{S6(3@hfNV-E{W?-JXP z^;x3SZyB^ll1<7W$QwOz3QUYCrawV}&~xB9w4&w6o0UeS3e#a#(SA_~`D#sFn;>#b ziN?`0sBU#v7F;Ou*CdA!hrx&zsS?@qL)$``?k9oJ5+>)|< zSb0TvH$%BI5V}fkj$MmVhI%Hs@GYz6_&^(A#S@hCxuzQF8AaNf0hEUfQv??gZ+*ho zN8ex9A3|ha!egW1!i%k+4-Po=lg+bmjB$}+r6whpbN+LB2b84pKCG7n1y!dGz~-0w zFT3cI>^gKnk2%5~U_F66XrU-inz3y3xM3;@p?3?w*48TX-JP@thBwbr1NwD{*NIXTaq?ekW`HH2K{>-pgB(7_BF z`U;!XxYIRW^IvIaMbXZ*76hnUs&ckAzCIHZv%%_MjHiUA=5iUidMW+6YIG!x{E&|; zjxaDN9ERim`+`ZcYQ*UIDImrjfwNOCHhY1aU_B?FL!g!oSBUldVW|CK9kvaF?IBLc zIyay<8Ed*MDdypvtPvW-sGsgUK>&HEjF|g z?$UVLl5?COj1Nn!;BWJYi|g55{b9zyCU1uhvX*p<;q=w_)2HY2U)K7zvq;?0P%422 zJJxb3z#?MdQH*BVSt>!&yqIp4&{h@jo?n1CW&Nn8crKv>Ggg9&&9i@jRd#pGG!~8}1E)SqdG()#QU#9fGB-vz?*9A?LmMPOuW6Ya0 z$WI+ayrgjd(ZXfM9{=|2PY&zHxC2IgZ6$`E<#!_dlAGSq*;ZZ)A5VmRU6aD}Y~JMm zHiS5dpo4jTWyU?@N~@_zEYN(y?OYP+;FmF}x@Bnu+d|LFthAS15UiG{k7Pe$6 zFEPnie~^wVt{GgsZ?7q3;w#7cxmP@5o;cksP@|P}x(5$-XB;wbtx2#_Fd8xqYCi^ zLrUINlh&v)eaO``pcI6#Jf^u%#*hg7^TED#^~qa;;2YdTbO574X$%(M@oFXWa!E&} zX@k-`p8Zdx6iZ4aj~F_j<+(fwh;82Xx4B|X_TdxIo#Wpyx+hJa>Uhe!#Bal|TF}6+Y%(29bYgB`5QcNa1cPw(x>3`;6FDk92r?>U0g z8OUcW2;egko_`n5E&CiRrU?gx`>jI5rHm&d^z!c)n|4wqbVLqWk}50c_qqi-X$u!da(AK=iY4k8$|Ucr)n!?CF(BUum!MQY*=L_+F6H z-OmZNG_E>hg8u;%3~`eSz_5v)z1uP%Ty^r&7%CGpjf!|p%d!=t4xOWC=6CR!D_l`J zLkjIAB#Uw$m1zU6h!fS3-sBgZwdLE}UX0lcwp)JzTlKNQRmgR61 zT3|Di8Y>?)$w!4$1~4Ki`-y!uRvBTi zSMF-t+FlD!u>3%{Jg@^$o>%JGs0@D@Z`5&>^bG4pYYWFjrJ7@km{G#)z@Sgf5sAe> zMx4x#a;pnryJnDLB;k^TZ*BSr#_Syf^i8P*UIH|n!w9>L0|E#C{7sd6?@d5Yxhf)ARkH9m z1f}x>SuMkDA^*$KB-K;{RKA^V%D30Ilz)e?Xn$~IhvN;z`7f#>e)WpK_wiZK&B-6^JeF~fR1HFv=s)gaAI zmD8qG4HFA8v~94;|MV6|;;FRNs?!TvNEeQg(2kWIW7%h#>}&NerY$(gEn(O5UfDg& zy&iz;eg2PyTRPx5!pO16JNdRD7tRY8LmK!zq(F`|lL}!jaf4@-1H~>fG6{u0B4*}V zE77TXv3=6Qq746yoE8Q(op?rn<8b$-s}XPrG-7gsh9kSJMt3`6AA|=y!=OqjsC-{h zVJK-1mX^RD53tx`*;%GZJkp2Fx8N41qI#t_K%G335?fVr#EgrzEQA$IDPXq@8ovTw zrDV`dtWYd7;^)E5XMIwIrKq>g{!@}Qj6is(XS(+NEJfww?DTfx`ufLvkZgT!Ru<>F z(=r-?tslW5Dd8$)iVZQ)q8xucJ4hQ^TP=*dk7#{idQpld_ku=; zR)DADyK%0pchsp0NVa)!{-5_j?LVehY*{V-lJAuGZSp?RSRoa9x3vCv<5>wSQ_!9%=tcABH1)>yB71?qm|vxA%u)}g_zHYbB(=C=Q~|x zq_zoln~3+elIz_K;c63y4J%P_A9YZh$4RrJRsT@z-ABS6fBE~XzLxXV8L4ic!;Ow^ z*8xd=&vxB6 zo3AChdU~9OPjWbN;rOv)^GGI}0)cNiZMUJG8OHC`@3{-MDuwS8jRmGUFQ*e{cXw?Xmp}uJySwXjp7)zsGi%Ks=)Skkx#!fby?0gfe;+p`*12_O z#Tnfk0Uc~I6|g8hIQKTL^OdmDhIQ$aQ2Q_D7;((^Tw3DBO|5Zyhxx1uvf(@FKo7%C z4t~%U&z2inn|=6W#nF{coL$P62w3s%M&gA*4@rH~tv%L&gkY%r90i<38N>&;lx(V& zUEx_$dLfyj#jT#q7PN~c3+NvkAGb{uHX+uGL4UuLj=PIzpT9z%O?)*?XrsN?%7vkM zJeSP)jlmL4=AAdA;XdBN%_Ep%4e<2+RFrG~98tb*-2IL8DAqbz$j=_wd@8z`&{&T3 zIU-Q>=;1|?dje!pr)7}A&xfq2GPxj`4=L(rYEZWhT$(|Gr<(e$z*Kx86;owNs=JZx zX@PM5(nHW&%O))B@OG`YgoWT&DKUG>)hXoQ?8lX^kGM3KKnl&pQ1M~5D-q8+kJy*) z92vyO*SL4Tqg-c%^g2Gp7{n==_>{_y_mt}}!7kd{>R*59?r87xP;8_%?6XMyDf>-{ z{L?HFIqU(te8k~TPpY)x+qvn`l~xS%MX{+F(U>nufS%Leo_P5f`f-lAnier)L|8v} zs}ua1tHy_*3;6wXxn}v`43&hhf$fi+=gM%o0X6KRK$yd{jJzBX)?9`~STBR2Q>#qC z{cjr@wX_WMvIR-0cf2T@GgLx`78(!mEL-czyIoS+cDqi}{{1ZXMj%jOH#mP8s-|T-+X@H+7hnJ?#5^IpFO9wi@BLOJZk3tDAPo z+M3S5X_nV%w0DiHcBFKk9DICnOGF$_O7skTi2-h!Y;2Q(S5W%}mZ3dUn*E0V!33s~wh6l{c;l8%b6k}B}|X1!CeTjl9=@(E7%ay415&bAGScJbE>6#cxWOuwpP^++;Dt)+I* zI<%X~$W4!pWqE^rR~5<*=)d%^RaS(x1Uy=JpNl(8wTPu!T8#Tu9WBH^VyCjF#fdbJ zej6TiP7BXqpW5x?`ukY~{i-*~;kuQrv%+fe%b|cstwRK>ZN3sRJWF-Y7P*YJiIe;L z)PnWB7X{YtWrJ5v?Rbg-q|lR*GJTubHp6X{M*#97pQ9(;>7zW6GA+f0#m08aB*!?TcBW67a+2>KZ<^a*S48?&Gt}_cTj4iDA|1**TfJP}qWo zAgcA;{QF<37&Y=4wAy->`~m)XqZNm(d1^-tuzZH?bo9~A3p{jxmYNZp<0O%cKR&(; z_I}Y&ry{2e1bo{@zU^tP3rN9=wR2mMdgpztk$XDLkU|vywt64aEo5MktPbtNrQWIR z^$C&&IJ@69?$B_QhvF{*tYK`VfYs}#9UJaKIe1p=>3poT3Z%aa&Efb{k@LsFeM?+$8 z3^gkZH*Kz(d`kBYi`OpCB7PsH7r-2(qc$s^@LY!uG8z|9o(={YpCD%uqR9XSIBmgy z38S#vwfX)EMT5V#aS>@`)J#~^yK3;mjq{(I#zn_uX~R~AEZXNL`}y8ds@Z|>$s|0m zZBk(|d53oY(SEA%n-BJTj;V;@4Vw*1a-Pk;r&QO0CFAe^#PWM=CaVV+{;NE(ss78r zjjOMo;Cgvg%j~MF4B%#y^Boo|g4Jb;XG}lYutR@U54Bj__P<}{g5}9A>foWFM%(Tl z4hHyWc6%l&O)9)-U{*N!J0G2{_@pYb@yXkh1~CP#yJylbJH_&+NgMil>>ylGR?MX+ zkpa*8eSQuZC12=BA2m_seRUGdk1T1n@mh=90_J>#G#?c>u)S7y;s3xAoCL4!!^08wQFNOk63aB7~V) zvk+5JE3^`0wB=`9=f^3sluO?FF%4}AgPN#%K%S5m_Kts$WAGSN+-bBpZtHGC|L2eh zlpB$@`ww#bOJo)0;RRM!jo|@BLJPb7U#!w}7(u5_rvZT}^;U40)rNQD-9q?zjB->{ zdhlM2zmDbu83X^JF>TW-taj#1k=V6xbqYd3qOHOGijo@>EN7RweRN#mxf$sDa>34F z$zCpjKP3J_g!r}e{KWZ2vpD)Pk&T7axxtQ&PlG`iw7K+Iy@m8+YpKz{7-lT7CjFmB zmH0W-HR#1IaAYg22+!5lFR9`Ic7wl0IrLe5L<5qJIVoIjko#3qXU^C4$$6IGxKrB5 zMGyw*VSQHqp}5h8tB4~0FOGJ-MIZD3_nf_$F(mX`wbAnqQhCOYp(V_Wkp);f^;r8y zcz?Ou9U!DST-`C-{a8=QP_du=XJE9S`3uxO?OoF<>M8Neqj5 zM~B_N`RA1!%f?6EogxZQORY$Z(%VoyWzO_XzXz(S#v4Hmc1u($@Z(%r_LH zfU5K%KoteU5|dSgqRhAhQgp>%HZ=)qk7~sf?>|YNZB9spUbcQ|Im7ivRZScb@3c|L zZ(7i-TVs}9rx9m+_7X26)~{O6z*9H2zq<-Sp2;Gkbq4ko@d_5R=82ECW4dJAk?|@| z$ihK#znFOkt%#5tbq9O=$3G|LG|bl4C}wB4Fj&|{YUV+qrz&=kP7cv%6rAcTjGnx} zbt#6SG~&(;J%!zMDv#ou()?VCw8<3GeYu;FvAbZt@E`7SI_H_BKvDB6IoJ=-*bt_= zTGmyJHVy>B6uV__smshHHC4j88*>+CPCx1<{`!pTY?^c8%p6_FZb1l*$xw@v-DAz$ z&X8;v$hD_@)~a2zaUgMld;j1bhv*|WII->TX+u*|7y^~qC$nHJ32M!l;bO=v5z#fZ zoagg>p^(;kMohi9IBUJFrXK+R^Ax^NDL~}zg>X}3{7ivh@pk9A>z5SKAW=INZEl1= zA;!<(PkqVmi?q79Dh}R$?uZPi$6(YU_iM|{2am&H|ATUIdsQ5f6ethz8;N`}JKea? zF=dUBo>kia`S$E}d^$d-FqwAI%5ztFHfgg;Nu}5OG28g5mq;pBjSIG4IQ?43mcfw;#7Noid<9U1Gg7{Tocxt}lMROFXK?Gqm8b zU$%J^)~h8tU=b_Aq z$%)*~e^HQ;PB(35{Xk@8H4Z0w-}Nwk*JZxDyLz___vwbM;FMb7ED9xR_v~e) zIwMoWIa64=%{-Cu9LzX!Q*(CT;2~%R5mH3#!pKSV1f_i+eUZch{xri_N_W zsGZ|fe(n16od%RCu6>?;!GFm07dRA znIa%7?C|%JNJo#c@!=$Bav!jvyQIdKVR6TMCn}b7RPj-H33~Nac0bAmR#C@@pn_jZ+ z{>s9fba=W`nG0l8e5ACm8<^8_a>(HMnL*x;9`=^YsrK2R?+oJ`Temh8!g7DEJ47Rb zu{U3Zvk+!F{e0tgMU7#ifdCOef_>ef^M^Kru$GaS|L_@2JdvmgJGsLN)5b~?>6p9o}4sPC3$Pl(cl0;0PVjLi% zpBUt9U%}>w%h%07taxieF^i-eQQ@aVI6}oa=M{UWR%;*OJ4vKYI1#iMo(#!AEq|AT zgwER0HQUT}4bbK9>D!?Ag&n)<6PGW%r8oN+4&DYm@A)N9N0-PD+V!>S(YNyAnwZ$z znQ;2aQYKp{h^GoCnfe=4p@}CWa8PLe>M3!;L03W~42AzA$U0+dTNJ?Gott{=oHQy` zgRygmZ*152gr5w}#WmrN0Y5{j1vBVSTK466#);3y$BQGd4O<^;rohG0c{@nw8acfj zMaLoWa$u(oYej?a0*zdj0f(s|Nx;WsgEOoiOUi}sR#pdx0ux?(=p*ulusXo7Y}?0( zfD}ictn9&7PJV;KJo|$zMq*L!yZ0L>@sZ>6xPUUkKrg-=H|6hwGiIO$F8hm*|Q4{U3$HTevU=@}VzfC)B>n zL#3}3D@f#L!q}#g3oQl9PEZ~IDhZ1QV9w1T>XnaFV@-emS7h#dhYV+v z|9APLGT*~MHG@a9sd8;6I!l3G`j(k$E+X=AUXzAXA`~5`-lq5^D#3|&DQebJx6d9R zRs)N-G?R|yRu#17zl#WSHDqL);0Mw}yXP`}0;+7jdjBjjtb)OWHfOQp*}h9Z1v3FX zqR(ch?>t&28(FCiu&dF9snw;R1@-m_G78kf}(Y260n zkI%vRV_B$?_A(xR6y0a^9GORbt7o885A|s5-~W4wi(xcm6u5kXM5ZF{#jql=U{0C0 zA~)x?tlj;%5>sJ~7wC;+xKtZ=n=<*Zy5P4+7wLbIh{^VpJv_La4G`DPBn+xUGP(U= zG@wyTts?H{e{tjr*!c!OZv%fE)FabRNK$~VKvk2EHMM0c)i2yT^q8$|pYR}T+5drd z@p`H_Pp!L|m{ZM4!uyC5wvri(4HL}xY2rVsI+mD^2B!7J9B%o$!`T)2W?Uhv<>OuA&}_i(GZ zok@7XEG7SPYzH6k|FGH~W(-hm$ef^3KQd65d_Nc?8q=O&GS!Ll=xC&CAi;s2Q!$gh z)e){7wO~Z~Q#WcmJw_8kBkvz0buvgwu0}T`Hh42e6W-tzl75$+@3!u_JuZ^{6iV&; z&6__hdG_!&=pS_x(mYYGpjRD)G*BX!+YhKxKroh&w9%H+zBtN-lqH1d_gcAy>=J(I zcYmFr+VNGUlQic7GA0FDe9E0ZoOjroYh>KfT0*Q#73o)b{OtgRZ+ZUk@^KI~D*zEg zD=u`8;dwjT*w=Ot)P+)K+Vm<&ebxsnNQqBt>eVCNE9CmQ#j4P0f zvIMgXMW@&)gJ6dpJO2b8b%_=VJ)2m$%7;SLXFMV4w6V?YtPQ#(3(}ESr zd2liyf(rVtX*}kxD+pQy(#hEj{sf-tq)@8GNwu?Bq={Knts8LOP>t*ffw-V;POoB* z1_yg{imS@jJ*^BY#a%>oflWSp$d=;;xz8woMLllyJ1c*u06)XticJ-mMmPOzY zCQ-SK2*wemCh6Iy!~_KwcyoHI$<_%1eEf{_u6>08b+Y6v@`N0)LO_@7;`v2b^pcg* z;dt7kGVniiR<$DF`hQ9h$kPkJ|4I>R(ETVzR`i9{(kGTH;YK6;@X_3b%puW0B=I%t z)ZKh^`#+lNP%5yRb4O9{ndeYtvQ(DY!Wg1mwrMTbRef-m`LJVGV0;o)&E26CzTfT6 z5JHq+D;lf46Y&3D+Fr!80Tk&G`EGkV??@#J;W-2CHfjhz#o`6v`bjNiDH_z6MkNF~ zbd3B;sYjze$Qo3|TmI9l2TBPoEp-)bm6}D?YD91ZOkLFZUuB$e7nC%h5ufa8X_{|R zswN_?rEO1!vO}NVA=OFxM~ozdwff|K{zb+wADUEeAh~-<zBjYhzfF zIJ08v|FOk{*VgU>um&(*MIW)!48{8kqif_)*qU^C?w5`M>Cg}oAQjQ_sLP1#Y@Tlli6iX00tph+EYpUaawLow%^d%V&r+VOgt8fa&ZTD)2#C0jV8Y1c=lEu6W z@ZJ(&&>31c_+c$vHR_&mwU4O8BRNtNRMjX&w$6^WpriDQi!pUy$A893zy?=kVcjA%1Nfhd^w8T*yvfiu6p%Hh!L?Nw@ zj}sdDn4dh|D{*mic>Nn#`|C2V;2gn}K*l3i&FIqHy)>$cHC#kKU5Rc(q^PUVO@EWk zlR#ypKC7YE@4V#%EQ0ZZDV?4(5bv;`Co6PKq6K_`hwE#>WkTp(ByBx(ig&ruebFmI zA}2#Xp{{-APqmmu-WfW#G{|1Ryl05{IvyJU@AcldthHRov|YEl)t?hh`JHYQ2B|(B zN%5Dko%A6O)!c|F9EU|y0k9g={HIq#6vf`V)z|jtt4_Ej&5GRiOA^<-%*j?=)~|$yG@++{ zpPAgW*a2ocP987?k(%++(O258j)r5c&LY(#2U3@#9^4wpNe24MA~P#*o5R)52<8l3 zK?-@J^h`_F_21`1CdR~bi+enyma*Fp6w#O>Pg~#~>kNBNW-SHXhOavFHF@bcQ!6RvRxY3pSB0{e*1fq)U%fgQboSI z+)%cb!0L3W5+_}GIMhz1;e?tfa=o`I-4zP~xcL%W3paPdeu+=eGU_);@^m9RSzO6w z)L6On5=$A-MC1G9FnALc6_rfhI1*mthyDPklvH~5!^Bbh8j76A>NBMEz4O6ADYp~I znxn|e!z9GSge)^K7vx5SE;2clI4e6HVr;dhc(9- zK4kk7A}uBvXl3`<&DU+RdU1M}X6Mw>%f!Ug@HmL$OFMxq?2|UfaZ(~deePE`y=M`{ z{Ab(?-ri{W40;I4QpFV?@;9^=LhG=zYrT@XvmH+r$9B-t6(?-k+r4#51_LG)_akd6 z_Pf7Urb3D%vefZ9fPKBwx-R^l6gnq zGl9>M-KT$TaM`>_zn9+T1=W29d6${zvWT-BMVL%ESK$b64>s&AZ({I*3IeDPY!j7y z=eQS@Gb7vuI{S51^a+D5k)|eO+P@+u%?ILq`g`eXmY{Yj@kv`CELngW`&qlF+%G1O z45s;v8k}tgP8LjK6@md6{22toeSBb=26eV87hhHPeK}NvsBuP? zev`7($CyA!z`oTNJ=n}^cp`l)#w8!cinW*4%7l`q_$+4#EL~R91Y(2&bAKbBYA9hl z5;-gS%R1Ib!7Mcqf!DsyzNZI`?{U+)j^H(ag&AEOE&jmtrUCHR)GPm=_xq;Fmaog+ z-<58E`p*y#h|Z-~AgyN{U+qmS+9}hEW-mw*F)n;=iLf{IUqyT0eW`PJ?;mYGtPLqi zd%^EKn6K=E$?Ym#w5z*t$7+DEFjJc00X-Jsz40at{bD>JER3s!5$`TH{i|CLXJXxB zLqB!(vc1p9z|~; z;=YIPyfFJ{?LpbsZ+*%B$xRd^bgRE0!M!1!6~Z*ieKqoJ_|h}M11Z?nK*4(^#{%vGXWJ z|GZ|MaaK-D)*(masbc?|I+ZF`P?T-rA?pl?U@}rjF4a8iH@&l4Xrf6HSGSdl>U0IK zl&^eHl@hH@iEQeIf7G+d<0_g0yLS;$p zcRZiJ4avTjBfFOqy@`1|M%2-wbxAqV6#$SINB0Wp!U-h@F?}?RCp^&kb9NMRuR5L# z51g?Or2FMjfdtSA_&Ii~`{Stz0c%2>bbP>A0k^9lmr;u^4h>sO4Rt1N5H&v@$MflW z5KXR(*+faF*Cj{>YxOz^GB(~EFcnrmkC~`CpNNJ$H&@MwSI1u3U^*le?HF}&IL%($ z(jXhI&>5Vc^UIx$ADyPuDl>~kzJ@~JH_ZjWJZaMEj zKXbe!#yjkQr}k9&rrPvJ#laV>a7$yzKb-u09xj^IbzC>9{9S-3H{j+5{I*jilk0WZ zif3FGH(bx?Z#hCEq}<%ff(0#gY_qydHwDP{F$do_9X$HgIs3!{r73bQlbVV^JjXNv zJt{k26m~+Q-TEHI?Dci%4r)S@T|N`ASX8{-%c7if5=7BGO|YxbeogWJFg!-LC-{5s zx)tN&a2)`w?9P-8(SjrM^kW<3X+cBXE9+ONoz6=fL0KLn)qpXfupj@(Xnrv8oid{P zZ=-ZJ-}Q9p&rb>#I5i0xvehz%GZ$2j^Y}xc;87cVzon$QJlYNo*6GyCK&!3_b%h?; zYm?CthzhulI>jaXFPS|SsM!-)e7>hCAg8-#dtcXqSx+py*N1JB%@D2amxsf;4hP68 ztrQJdm-_`^v-o!JEK*{FuIi4n_?LVehsV9t_C$}^5ol+J_YM}$27T6)GpB&=M+8Z3 z*el;hVwn|XaEJeM+TzW9&YJHr=q148avKz9=(Q2*MBcmFXouGsAr~c0P8y`+f4n!# z11jdFJwf_1y>V&(vx+MwF#4U6yr*c}(`kEZY&PF*94X-W!o2-f_;ysrIN+s^tJN`) z#Nw)fyT8mp0aDp;f=tKny@>Ncu!rvRA>=J~f=>MVCXB>>qD0Esg!rjFHKfmUrmhop z^5!7~q1X-4-CANK-+^1T9bq5~P=4gj^z~X@H=ia_OlI;XXhYz37R1M-@ckWT)8%o} z4UpdIRcCliG{q8SLd;Csy+Sll#lFzLuc^lAK?{S2V{3U=PmE3p<NR9Xq`>om zZ=II=iWo8-k$i60??5JFsa=tK8Z5E_e=oP;sYi!)3v&x4iPT%+!yZ$9m9>Ul!8zRR zxvNuH6SpAyParfbut)}Oyy^=B-P|b!7c~$*7B6ZYirQO-LM}3>q6N;#9?>%|sEwP_ zMPs>F6KWbdmieDIqrf813pt{Gms7JUxkA2|vHDJ(bFTvr$<=;RtYid0G+qRe>C-p+ zVKmX)CJ+`06F4N~srWmeq;`h%JFInNq_t<2T#=Uux*UP?ODbQ|r_Q~&jt;cyw!s*6 z+YN6anqOQU_8FS3EYtSMn*ZUyY`g!_m=3W@wPlp+B;=Z#BVZ`S2yNsQ(|Iq>#wlz$ zF^l8je>vOFd%SeDuqJeMydmta-2NP8{G8tj`hl1DgIwMq6)Lm_`inT!lP^tshA0EwA0DG_iV4XMphTug{ra;ky~4H6Q1*0^Y3O+6pSuTVqx>9(gj1WVaxAP1*M% zUwj3r)IP*%OsRSlOsZ^QuK8^@hS-YMo zmx&d6n^9P6e;i>H4GpD^X_Xd}?%4qjzQ@>}Rjcx&X|HBnz;y7hd_QDgdx&a{x)J){dz>}%+P*IH3emBUgevV>gX9qELVwh982z)nj7bloZ*eU zZ;}J9!0n_5XTFC$GRj#3wodi~1KZUGd#*bM;R)g%U*qWw!W*Q&l>d`mWJ2e0%>pl; z&iN&`YJdGdB7`YVcxmCgbsq;N!s$+_-DyDF+ej7gE(fr6fALm);_`XbLYh2-%=f|l z?jK)d^V10#wSMi$_}n0{s?dWFCd2-9k;?aO>}=_7fVIp2IPGaj1?OwUwpN~7b{fJy z^*+co^@!ty$NQfeKp$J?K39uh-6RrQR>IaGK`7;~U^r9&gBld))Q`(9M;yGpa zxfo7}5%ss9e0#hPqKnJo18mL;QFif0Jn2=(AyGfL*l_qOBx*y%+*5fxs|&SE1B&j% zI$U1>cNW=1-T+fr#$*?{6YP?Ayj2e%Eul6%J;bE;J2st#gyabYOav&&WP? z%R6m~?MnY?AGG?L88W-h{K+0E2)9!HQ;$&gOtUD09SBs3V|oT81LMLj3CglJOr(4> z9Aga%Q`26qL-$w%EGAc-L~q8RWKC@V@A-Iwg&fA1XP+?E=i#^nI>f9KYVSr%kEiTc z8W@TteF2J{_&+oAs_^$L-xP_hC$jE)f+SSP{B6#1SVf;Uqaf!$J}m$JiA={>mtC^U z*Q!jbZJ&@wH)>cFB~bAD@4w=j5-X(cVCx=mPn>_D^G}@Mp${eRp|g$$)1Q$npkx!_ z!Jp6Nz*dYq(rm$iRiT%$-$b5gYj@Z6lcIOpp!{FsPY=WbYQ`V^m?$wvlPrx^vJcZt z5|zNCl=$(Ek(sZ`#s&$+KquEJK=KRbT=#bSAc zoTz65{#aj*ol^m37K4g?YD6gWWcnT#CIm)4#fb$=pxluzW>VSq9&8_|?_QMTwp~S% zIO+Rb%%ygJ=ypLjba|UwFOwusp&RXJgPbuN6JMO(AQ^(}IAJ|Ayf=36x?A@^bg?2A zNx2;uaohbP9{a^ZOz;<)AwE|rtWsVs!($aq|0$cGlAIOk9#J>cPd{NM`PHvKVrP*Z zgvr)hui?B+_xpOXk^o-zoluMsByo|Ji1(1-uPoBFXLl&Gp|9N?xi5dq=d_dp5OlpB zJ`-v2)ee|UQt`))W9j1t^R&_m1}j><&)(PA;*_n=TfQ{;qzsjy+YkpWm_}HmmPj?J zn|T+gg}!fL31{4!@BNbqRg3$of|Ad9ni5q@nTsdGe7N(7qEI*c$OU3Z?7n<~@zy6vTT+r9R&vTkV_kDSMPcH#Llk z3i5-@wM{-J7yI*4xl@N@jf4d+w}6Q#T9tWK9+h>Wz&r7<+OR-O8>1w#iz~9Xul>T2 zY(+^O4ubq(1Q@_MhoOmuA6PkrX{D)BIV>F{$aMWM*{7RUgSN-{6hAkE0S*%}up|aI zH62hjCJpQORE_&6Yh6Z|xo~D&k)`GFxBZ2yxk`ReMi2psBKps15CTS_)Os$;1Q@clyw zGy5ejn_m2cY)rXO$g#gkc2v*grG>`ydH|kSx@b$ApX-E_mtbz}Jxv-m0b$4vPfGQ}m z=Rch8cq~VZggg(hcDH;#Sj01K33QBo=6%8txglO5_C_=orzHD~`Ce;iKr(Ok3?8q* z(n#3hQ3o^>l)~Ei)741cH|Zk`HJESV;%@hVSM>E+q52s@Cfal-Z{QX)K!_6p!8s?$ z5D&v=UjNpMds3&Dz!@SM!62;KRdeTt+7kB1t{rzaeW)?my@bXQvh<a2q{4Y*qp-X9`D6hebc(--G%ba?FP#gW z$WqT$R>-vCsPbkQxL)W%@FRBL=&O|Ma(Ksk`Frt2&^tKt8kt1Q`p|3aHtqyfLkI}( ze#*KXm-VaoAo-ugob%Dh8pqv`x%#r}UT!^m;K9c1C%Ubn5q4A3Z~Hr2FyF8<2dtw^ zD_hgp$fyTWf#X{>UdT;195-aL5vylRrUvh-5%26X zwFM`OZA)6HsU@G}*d&a{VMY0;t54DQu5tXJ?BQu6__v#SvgH?`AZ%Omfe*TKOaqwh0-P z0@LJ{)uEfn%3IEE#tNV-(LH8`MD}Q)g9mVgLUbU0T}qc6@$kKn9AkfcN4)Z5xU7T=?X1g4Y=6Q1h4nofi06Utw6dXT_ zEc>X$W?zYZGs_Y2c8XiH%^nyNW9I{uS@F`(Ndi}ZBixOG;l^I4)pw`fm$5kRGF&Vu zy#+s=3)_?9B$t$V^5(NW2!|1*;3VTEO(P7tsP2_A@maAWcn_?SMISCx52WCPMWPnBm@UO5T-bq!r<K*;G&QOFXx3$4 zQtt(!&NpPvY`+Ho4k{0*P*!EK%?X!h1z+tl^V2B{BJ|D|Fijt|}w2VZU zDBny~=jz!^Iu!unoybYcW91uIp9L_=svqXBXb*3b>Mh)Gx7Z;-Fvm4#JH#v_9}!+< zF&e0-)2@;l=AI~n+<861Xe{!)c@=PX0f%@@%^Rc1z~Fcyz*Qe&Q~Pef5+w^3CQT&a zku;*c1E27!kc9*UPus;=qLNwt(rsLvobvvAlJ&k#DN32^5fAJ_>csT7IT?{#_AAij zYUbO_lA4AyDtS7YWv>f6H@5@M@zd+E$~eMfG9s5SlQh9iW5mg=#^a@2Dy7l){unFZ z4LI|Cn_vDoYtG_y;C#HVEdoy++K(%ij)6`Urw;Gu?&X*J)wk=6@~X$1sAj`wL@Lg* z9;%pkm)pZ4*}HQ(BdDT=7P{JXGHo8xsIl-*jl4!rF}Wgd&!PeDZ(UI-wv8!CM}OU+ zLqC|NW6u%T(aAv5#p9Xupd2}EBDG>3E?91;)aB(5lX;ik? z0Xy!OkCo>olo%-T2DWnb;SWl_!0Bp-nC_p+KyjOe&X$F_h|# zx`%xt;OiMcMdU(dA3M>w@Tf6BhyfBQn1SRTPiJ1b#7bG6Q=HE2)jC|<%Gs89g6Jsz zxTliU^4bdUy%TP5T3g2MMDPrI_T=UEnN{0=M9BpCxE5_WQtE&B#THfSo}h+E$3k`S zc)inKHE%lT2v7H;1#d)`5vwPzS9n=W@bV>mzHVni@W(q?(ybmEjeDY(G-)=ZP~s>g zlhDZ%iONbyw4;GT?$C%3js8zG7Vv;Dad<4TH#k8%PU!M-D~_~fvy->GTv5||>l4g2 z$U3$8W45?a_Qwi~NmrS_d)4O-)&#}v)A4_TSu$B%K2R&*}Kn8CodD1 zt|bz@R*Rb7MAT)p>oT)ITXp=(Nph%l`85pSAKdQs5{&U_=XJS$u135))mw%n=?qB@ zCbC2x=ZC|xirvB$U2$|z1p_Ib;ttl36H&ujj8F?mUhnEY3X8%9`!qOwnn(l1jsw<{ zEdOCH$4#28?~CG4gp6tKfd z7%RSD?@LXgFF`)Pv_)@11zB4zjsgG-4O20-2TE~2`vy|zITgU&hL=-tR!1rr>E^sE zOWs$)yy->EcxmXB_TRk*W0D;Bl?4~M7KeWuaFdumwwPXn8+1%O8Km{ zQp!@f#Lv~V6*qQ1iRc;#=bcWue&DvuMi9Ai84RWAxOm!>(@5k0#}DV zcxMtF$ltPI8Sb^Y`|D;IvjN&$IF|t5_hqXGBvZ0X9^I>cA1{rO0Osr=S6dVFCAXQU zN7Wez&4!%UY#&WDp~}C9%C4Q(P=X4p@FaU5{TL&LO&RJZ+O5|$o zK?qtp=bXFHuV&0T;M8fh0Gx6%LISlxX*w22U%bTo#lp{KNs|U!rfD}@J;mPv-o-e$ zu7n^gwn1m5i*{9AtBgFE-+BA50nsj?hnFt+?8$u>%02~8aQtaqV5dmWV}(v{3xG=? z!7J~r#N}D#sdB~!zRy8$SrJ?{J$)VPJSY$j{mH`(i$)pvVhb{GeXz75h!SiFXw;pN z$VZ#rYaARRqteK3bPn`8PAmDhdzNn3| zv2W*r(llDYZEKgieX|e@u7Mp#g!exX7q}9?o)|&n)*Grn&iXbKFzTHkCb#*OXQ!9o zyw%EKQ3V2v!PRZzDD_NZRCa`--W?6xY_#-)MBsFZ+_hQ>HsRtYqU(P2JB||AH&D4;&bZZ~XP*!m@&jF5wRzQIb#*LKzC@aG|89=KtBKiyjnnmgJ0at0 z_B%&s48qDEHVt1oHWHy!PGzS?HxKzXESDAUkA%XdOU<(nl@a`^{8o<+W}1YGr1p5e z{i<8wLoVsBqX5Wlxztuwqyrn{=q-;LiVVr3yrZE)l`k?xcb!Z#`N|ME9$@lGoMB8a z6|B3#&Hm%jLA~|Z{L9l=Q>Q2oQ7q~2t7?6uqp32_@LhBU9|zogHq+4Cev_`{qjjk` zS}$Yj6FdMk&^2h;Zp~{&mfy6Gizu?})jN02De4af_r^+()@+kb>a~ree*8+Fjg-j$ zV#u#VyKVC+mBb6XAxZtOi2~{#iP5h-C~2exzobkZ=^t~u9{6@|DKmLHILv5SgJhDM z|8`Wq5c+;3<6u0_&?kChz9g3m@?F%Pr}J89@V81gN4x&U@xx+{6Ik18{;!SsgKLQs z*%qDwsQ78)NBTp?hp{D6#SO*iU;;_vWgDlJx3on?z~`Ql zVrcPACDNt$2abn1X%M_oh$dbuc{omB$LYuWdGvjo>0U966gq-U)f0beXI8>qp+heM zrY&7j7b`hhecu$952bA9G&#wQ0@SdYvX_YO9Ef;AC;xgvwC<4&D7MH$p~FkeYrAxv z>C6E?gx9A;iO8-^b5CGChWt=D4<443vEQFPfCCL-U4M+eucBsKF3PZ1AA34aO2dt~ zUPo!D#L5>yk(7yg-2-M1>-)kk@gzBX!%72o(ebPczSM8kYJK_M1-VEde9_o1tjT2? zlRcW&uY~~L^YA9VgYx?A9*Te9K|?J}KF}SrV&y1k^7c(bEgHM>f10s6I&Il=f9V;E z(b$?^wpp)T9>sLI05Sh=McyGdd{@U9x!CueF%;(HKlvH-T8AF4VFiGWYd4tLb{I>c z6|2#*Rh5PQiazi*hXE@&haExgKG*p&487EuvDSv5TUeg2D~_U|UspyDZ-I39lmmhm zLm%YPK`s~ruXEe0-6uyL;F@o}Zi`KOHY(efg+v9_W%xuk?DPBMzv~_GCci0#s2gr2 z4~Okd*@VKs-X0`&DgHQFXaITsN&Xve=3TraPJNI2mBW(ARqjWRS40Ar)372fDa|Km zbrP^?VP38lb%MlBB;2U80ht_?{AtpQrzF10!-57uy1o{*gef3VF|#yf31A9e+-w%$LCBN8_j{yF)Rh~9`rnl@!- zXkaLRhixtqr2#Rf)m#}|@ud$_kH-izrkZlYSD7NHGKP%XD2`&ot+uj68AAUhUaY9C zD{qZeGnS2Hi2PONos2@$X$moh<~)Dv5Q??U6p^v_VwHjMJx*I`f@9U$tYs#2bo;K1`QgHQq{(yQnMWxvU?tTR1;i|qx+&S(7SzWO@ z!touuE=^)-RrVL2hlgN1>K#xSvseh;*R>N+p;lQW^*!qmyb6%WxJ0Rwi>v<>K0kom zalDLoy8L|}yX&%L$D>;Qokg@nIQg>3)b#4 ztdBv(@0d~v0!r9QQe>sxV}ki?92fPiu~PNje7f6=sM}MHdZ1CW3f93iEbc^gW0Kt~ zfy>XdKZj7^Knm;KN#mCd>h?8V9W?DZD?2Qx(UBZs4_gs_pkImbGm){+M8{1s-U*KE zYRe6}+pYtcFgUfz#FD7Us*c3HkK&h~bSKmFSEy+tThgQ^_**u-On7S!<{{9ZlwKCdblL8CKqJM_*T{d?M#6gA(kD%)XD|V z2sRWzG`hcrk*^}F+ml`H4=E&K^COMXiA+M&ik4uomZHR*NKI-kMTDjwKa;8~s)w5h z(y@eaDYRP{_Y(1A;%-QV$-8IudGLgrAokw7bO=r{LjJReW6y)BQ{xH0fF;$olj(IN zwutaen!lpIa=<&nz&TWzXFywoKDEV5nCjbv+bFUoCGk)gYmRKwOgh1(zQhaJPL=>4 z74Pg!LT&7Wz5PX{ty@)@J^qPlQg3Kfnsj+J>WQH@-Aj}u1s%lS_!25DQVaLalR^n; z_-9fph!9)+l12>%_@D{IK964`EuXZL^62je!K@5=e_!D?bOh}wt@U#3XH^)nB%%Nd z-Q!u??O;BTrdC?0gmPZNhGEmP!+N0mFiIq$c#=L*CqB6CCL)sMIi0C2Nj}~CuBlYp zcE__&=#J6iRoEl;ekuF`DswK4P{mY2{PpK+=yo`cd=*La{lPuD;>bwTnN7|fn*^7$ zAJlxfZ<@2&h4t`RSz689K~_FT^6p^JUqpEfYI0H_gg7_Y_t&C=mC=2_=JY+*rThcv zI|ki07=HAOkP4zQXh$J3nZu8kX8s=!I$}t|wm5SN<$POtLc}K{Mv%YKah|wEzR=c5 zokUZ*SO_NJR}DgaFTq;>AK%Nwrq7Lt_RORY>0AR%#3P*_hE6>U+>2OZT?T&j?@m5l*wy*|yZ4V{9No5-G1 zR&GvTMJ2?`BaKCkZFjX-QNZQ5gJ8ndVu|b7HB85!L$7>B&|!}Mk9d_8?}|z?m21&z z3^T?X?U0sD$zs$Xp0CcZcs2x>X23grV|iqv0n)etWcFW&*K$XnvWZ}n3(9w+L6{)luLvw4{$j@YsA#JR5!RD-Xn=Ps8SP-HkAz z{g$$M;P%;Er9|`b~uz2C=U1Z zV&7d_KC12F<^M8ZnTZfi90;F&O_L_~tRh`9eAPu2P5y1~e@pOGwxI6Rok05ZJ)Hk; zzTD7Lf)T=G_nyC?Cc7?-Py?KTvZH+?T_T?MG*-IP$oL@x}pAp#~3UsBu> z_&N#F%V3R1;^4mu!7>+?@=0eK)vGZ#3^zQO&dP+m3g@?b+}Zm z0|i{iz{ljMSXa8>wDU8JrvZ{IwP6Uet*WZs(bgOgzHeD=8EgQ{)!C6v7sThgvs32SxD|WV2RWC0kD6`2L(+)V2@DW|{f+gOj9=WX zeVgpMqpx=d?YS$!wSxOC&=BdhkNc*1igk$<9jeYxN#&Kk0>9R4Cnn3pUL-3ZY#*m+ z9O@K{qrZ1Vyna(u0Ani10{U+fz<*xvhS@W!uAJ*eL zB*?=xBLX9yb6UnDXMWM+`^$zEJ81LAKq9(JEiSERyaV6*XpUfd`JAtec>nmS0*6Sw2zyK5^El~&I+7rpz5_-1faoG($5lCJTjjm;=cf->RG z{40gWFVyFh2u`(#7Y}lo9loQKsc1BUxv-|Mj!Ptyv0X(^Fi?sVB7|7#{kr|}+*Q(J zUv(0GYkuSj8~*t5pG3Z4MNR=Jk`Xh~GF3aocu55Vk6ApQ*5k|Y0F}&}NsW82E^Z6H zDWxn0l`coNAn#^KgHt@9ZLO#E^dvzpc0T1RW*N5*ZNW3+)7gJte`xIQ zpxkD99dB4zuIDf2tLy-!1IUcWOpO_wN7758%~<`2m74i0nIVG-N?64kD(gYHUacn} zp=g<8VtU>uT0C3hmt>iV_fr%cQyow~`KB}8DdPt$p+jm&Q#Tcf%ejMaT8Wp&I!9> z>&{&+=mS291{|d*G&tm&@IF|`AO^4Xj#6ne>(0#{_CG2Lk0a-|reRR_3$&5GH{KZ{ zxh>`8aU^tXn#u|;!na_M@n1GQ5mngqAwfh#QEUfGOm0E6w6UtV2arqY5lZP(*Wj{Q zfWA1gSntja$PC^}VE_b}Tv$Fr>AQMl26}9C!P*-$7jlC`#J}Z2d=7d%%b@<-U&UvK zcp%aXGVcl{EvVgx}Wk;iJ>47P6fam-}Ns58h40cU?JfE2Ia80HtU+WD#qo z*k(VE2ECU5P4)Tf(3}O685?~SH6OlTP2rt%d4C(xZ&%o_(!t7DxfQW*>U3UW0cZUr zJoL4HC#s^g#b``}6W#NCyxn)Lez12)Hw6-Hw$aw9lbp#WTRxZ?dHXE{TBAm=&zEWA z>$W=)bo2J=_Kg@{{Kg|%{oM;C_i(e(H{fBdoMGm@prNbCwutf_ZrWh=Q?J0+0AV@y zMco40l^5YEG^sVBG6RATtTnS>#O-4MH*xKWb{+cIljF+y&xjqnt>=zU9!K8+`b+t8 zd??b;A>1C@+rH-l-Ju-=Qafy*$%(3Kry|NI9&3?-eD`+H!IIlPjf${6+uuX5CEyf8 zujuX*(Ia@*w}z-(TNL4?^zBvn?ZxNAl(qX7f1~8%&ja+4!S%01X^yVo9bvkatabmj z)rYme!>cW4)f0uy8)muQQ?SL#;TYqq)N5pNAof6>y}l?ZeSF>>lVaS(yjzgb0f}Z) zb)$4X3xdGvw?)qML*6Y&415xKdQh)Zfw4@Q&e%oyQ^sK_3>zAq{6(!S{8vCE8$S^Z3<31o0^EZo|4%JR|f@OPJjJ78Nt1Q?H(@1k4< zMbxwy;zNLeW;z?%17XnjjrOj#y!F^h3s;(8%A;r`uwjDQyGkiy?j^$O)Ngj6&w7MR@@Yk4Z#UfA9?khF3*f@&H)7P5=)^>)^2aP5DU)%$r z+jtDZ67cDi@8#FRiat-C)PoqbwrDy`I&njsTA*O=lXM%Fopa4d={Krdwb5{aj0V7m z_jez87!#`8a{IDl=V_l$<@j+j@(hLTMBSXzA0PoRjnl|4HP?KeS0U4YAFXo~3kQC8 zY--g{M=jaW1)j5E$RYRnhu2g__35I5bhK#}nY@WnIkyWhu?eYnwmtg{ZB#1MF3)?~ z;B_2-f9c7Qe~fa&=22&Bf5bx4YkDb*=7HgaKJc;bn4b+7(qcm+qVBwG#P)eV>ph$t z`KjAm9!T7;C$7~yyV5D)rVx^@;JEE(a^BPUc!uFU=p)}(zVHu+Keb9MMQpHrlO~(! za@pB^>Lvt|!ugSKQ-tH}^u5U$PYl8aQRUBiLPuuJ6$JgyBWQg_QWC z9adY^oc!Nq$z;Lof6NLL(#f}~k2sPJv?4-Ni$Nr=Ev2{^oOiuFelD|gXgGj&B?JG@ zp&V_!yWwprcHjn!XyY)8|9QXV#bkWutHrf7PlYYbmUJ4YB2c0&Mj^&hNJQ_bIp{8^ zTDfS5LGcKbAq`hFDD9rdttTl$&XeGzBP3xCk{h!2v#bv8b6_YmCc;HA`0i- zfj$K^X=q`W4Y%|kaVvw{#;bM*mp}Na^9@+(8GcHm1g14P8`!TkO`Pi7a|yc3CGF>S zy0}2>fs6UZ9Zw(JdbcEjD*Sm9Vamgw`4dT(a}lgz#*hiSV;>m(ARZi`(`%~)^E`AL zMu*ST!2n03=xo&p-U#8|PTV@Pw1c;aYZ?p_;ZI$Tmba&TSCs2EX^!pv*Ud)OZC4Fv z+a*5ZK)wfSOGwqy4DHY1dqasfGF-4@=;l7~uAIX?7q_OOLSn)vN`L~)!Vt2Jzv*9xPwB$|xQW1dQ z+d-{gK%1}~n3)}L5-<9>T4KY=IIQa@L-^9X9f=46Oi-`+{Er$~mq3?(9GN7oow2lG zSm=z%@p6ZvBGNZBF>Ewl2%s-`WNJQL?_&E7ga8B|qaoLS10XCXfdeeZ&n>#X8xz2K z*gT1Iok$ON)k9tKpbr^J@0;(x@8$5;&t+doDB9oOv#hdU8~x6aOWXOJ7js4#xZow0 zZppWlX3UKYp83PK^ZhZwZD+!M&*RE$ii2t|nv$JDMmEi(C9O@-OyoMN;R7_CNOad< zOv231AS`_Qks!7X)jtG7*R4*{e>;)7)R;4Si?u(9R_eaoP#{G$;pAg5`|h2Hl%<>D zR3u80yT(tB74>qMLoD6&4FWA@eiYXTQ0kkYrX^Gn`|fav>zrLE+h8XNU+*2Du$kL& zrG3mk;9E5SC`I%{;@)}c+oDvoEGgBqH#Vr5Ihl2lYCr0TKuCzS7r|6*jhOI@;4K!H z3ZWX8{E9bmo!tc;tv8Tyl;?$3;N`jAaw`X0t?mU4+#h3Pt2bWbd^SK0u|a^k25zzOCsY`9Z)Xc(mM2R55% zbzKl3&9VR+VNe(e1oN5YTN+kL3d7d+bj4AkM9nL=hqVW;+_!dy3&rA76JtT*mM{O2 z0GQ`A*x?Uo{t(8m`u>Yw*>^MJ;~{^kSELysH4(+!#*`xa2It+4aB?LgUtsm|SC{pM zk=u+}`@}IPOd(^fIIx?JSmr2O)kdU#;8KEIq#i>C84W>>c5f=MupAHB-ZcYNsjwBT zN!gDg4JYz>seDPv@y>KJ?X9px90=CEVswjAPu6*b4j{`7bqY>|_00UNIR-~@upUu& z|2G`V|7k94AtmKkE%AmYah!ruRir8T@W-@Na!d2rt~4A zKd0$}NM~UdSuCnZMaAV{T8YgK`y~|+;FzexFXFQh8uMRphev#fSMXu z11?uQ_WV0dAOCx{7;|Rn?%!;&h(F?Sm)P#Ba0g*DY&(=GVp35nzk&+#uTRf?CD<7b z?Ua03d4Ed}r}q?K{d`yc6hyxm&?|ARaYE?YE@|AmF^z6r}kEvn4YH4B9xjAm5fNc{yK195s5_YPPUJ4lY5a!dewXBd*h+4Tm@V{ z{r?^AF~x+x>!+D_{lC)%ASUgSEb5qvLZ18FVCc$ej`5-w<+D0_a>UyLn^M_%9}-BY z6Nq>_p?nw{FJcIys6JVm%Tn9#R0<#l2q!tLl0kYW65Vswhs6Gt0e%Th5{k<6%dESW zAKEV;S1k^~q>VqrKx57$dI%YGj^Gh561cp;EODk;Uz1I@%9l zm^Ym~)IuJ2cPx*NZJuaxy=)?rvzA=ScI*TZUahMAzeL zsV#nte;H1CbRqVd+0K8sy^TxDOH1Nx?bw71B{F7LYClYyd`R_sG8%ofh|6VVg zf3KG=N=e)}oTO?hGVTGG_SA9KAwMPhh(J;DSL}Rzuwg53SJ3C%$iE`;!th;&!xIHB z_i+4PE>Y6tS@~I`Z$1p3sWRDNBG0e%)FXLM+U#TrEUP3BydnA>J5qLY{pEy{P!^@t z&Aa5h_f`kKU5EWs8H0J9Ako|EyIUyh#yVM)w!oK9OX_}~>#DLB%vq&6v`J$Ule31%f?@<%|B6FkK z+++<*Z&hqQL^hFk^eBvXz;w$|L5%>9><+WHUcV~_tBseM3rbO8RKVj75+p;D?kto3tu z8c*v-ad7SYvF{T+xI_~a_ixTULw?C4w6F=Ocx>2PvN$>}IIJC*x1k6c??G54T54^fe*-c4 z-YmXA@G&edKO%<2jjP2C<+d#l&2)NWaD5TB^|o{GrK;0Tb^&;Ix$FqfZxn|ioy?9D zNZ)?)hcO_cH{kAO_p}v-G#Wwckx{xoAMxYI?Qr&=b6s{3n4;%ivrk6~TwO<&j7ILY z+=_nc`1E_+@8okP%BE;tlg^7M5(rg?5d<(^zs%JpR_D54;27-0w~M+gCWwYh$fOJn zVul6YCm7EO>@bWlCp02OrlYYRe_!!vXh&UV6&Yz_UD&c%U39JUeYiGT@>me^$7I#1 z7H+5hN(R#GQAz*dkY%wwpUMRr)>Rii^KT?KiSx6kDea8Xay<1{G!_R9t$qyRD$Ok) zLg?^ptbJHpcp#BhIVZ^YfI)@n+hhC|7tq|pap2Tl3g@}eC6X!?`)o1O9zH((aIw}j zfbhY{6TWKcM|_CiV>=y_?=Ri2a7I`X&S&5)yhZUskS$18EUP|Er_$jo*AvGT+xkF5 zoextC&Ef~er*LfZZSM3D+S?_z92aT(3{QRRGyuW0pC9_ygrcUdnjIDp5IpsE2NzQ$5+B z)otMlfM?WX)?+-Y?(@29CFqNEUI&upEW#0zvmoZ<=Pz4^q25SgLSou_4;%;FKCb>7 zmCfRaMw5eV{@2`)N-<8zT|)caI*$3Bi5T&4*ph^N@t1{5gVBtD3?D+LLLQMqyl|Vm zlvl#M)OD}*v?%7PBoI_v?(jQ4?s_wyVtb`br>jWk@hyF~E@3J|w{FP_!!H|(0Qbgu zkH_CW$A!9-sm&`H7CNNK((5gCx>oxWqQ@QkU>O}swoH)I!eU|8jbL@WS{viTUL(Tp zVEjdmq$aa|G`k;0!wAA22tCyLJ@QMKE75g)AIHIU`#u*!K%PS)fl)d-QTP=>I7xc<~4TE z2DEuEm!Je?qxYBLoA75T222R=Nu@gYk~yUq=}FVbjLy=`pSW*Wl3%HT?5!@p%<#rn zHOIqv=ETYYr`!{*u-xN>eB69G!s5?4qP8_o%tt&nTXUl83r;(=9ru-@W!k;0G=nS! zCK^U#2W(LFcbW{!_X6ZKUvj}mY_!61f=F8*lpk-rvx#jV+4K^aEp^{YDMTYglm^JIi z!c7kjoL6%eZ5168qvFxLTjM_7-fn;>X-FXKb zCmF?z&tBm+L2~qtZMyP#k34)W68+gJ2HcQ=r!L10>Q$vEw`csx z>C~xldil-AEh%$!zrRh?T3KCf8!S&WJ#3rHQ*UJHl!1blqUymD=INE;ZF-brIIG+8 z+7%;_rYZyYBd{&-1bqoUX$-v59hkbN(}dkOZP(;WCE{LaG300!hMT@=`oq{xar#u~ z>}Ck2>`FU`ko+gWv17OMviq4=NSz~p&TJkwrup^u$okEhtm&9Ez`$q5J;QU-z*bc& zHqUPF2Y7!a zpdNzj&&NJqFTL%?>gD}9@+Y~uXSE?1pFQ1GWQH4({hFKAl*wC&g}YrrJURaMik-_Cda+j;NS6n*I8vy7>hqiV z4r({lCQd#AKvG%oMiZ$H$^-4P4Yf3IbN_qLJ9e|(yq-Yk)Il9a>4cQe z_9MicgDD{BxC`DlwiBtYR{VfX{Yh~kK+u<-G*#I=H&)(Ao|FYgP*9KD;>CGVFSodg zDf2%lsi&9i|H6TGz!$;+tI5FvG%8rs=j29RFR((-G=3vNGT$M%fT!EBob%$zj`0r_ zFYohZeZ%ZR+L$9+LwGzzB28ol2+$6Pt(ymAP3S)smAE|^IpQdMr#;B^bIw!3k(dp9sC`@1`C!?=S;&LPnJ@kp`bT3#+ZB=L(80l< z3X4^VKKgueSQdL;InayEHt19%rMNah5BF?Tx?i-!wIbhXMIgOUIX4Hl>-CkZrElIu zCzB?6(0mBRcN;OlwCm^{_t+CAfo%knP-9~9e-$UrDHj|w(87$zE-VRM=ceA3P! zx|+?Yokw=m4$=@C`%wGk@G1gcki|3!J&W8v;mscs(&8=efJXHD;(yhZuM5L+m^jM)N&E-WcN^ z7+*H`5;;YZb0iOj#g9?r-w2(yxgQKfES9m#?zg>dcYL-jE7!JL7lsOt*^v&a)}{S9 z8a?hGZ}$ezZtv_T4oFg8c?Q1wM`}^`i>#z!+?j>P~YG$ zu$A&-;K*e@(T?V_0XdKLq+S5m!`^h#DC(Y?xh_ z>ek?|pfkrkx6Xq#4|M@wqC0_LyYAn$S(CpP8)FHmlqncT#g)=~=9zV>MdbyJ^*ea% zh>pK=1|fHv(-hwpkI|*)$B8An{lT1ESlF1Xj#2)GT;>|GNgGPw5@KDR01|f4AvAz#F%xQG7bI3ZMEmX`fshWBL7|N!p~-9l~`_9&cir zeFti)Ry=Lt;^SkAAz*JR=VHD>*I3MG3*^Grr;`N@-w5$I<+2zC{=6=Ao3PQ#Ye0H8 z!uD-z=NV~MBY1`AcwS4cGzNP$Oh9stn=3S+%DDgdW{fM?c$X*V>DAYHP0MP5S?CSy z@yB^*mrKQKqIG+j(XY7V^+d~+aNlA6?DvASAi3K|4(anPg{EUsPP&PdR;xaQjEy83 z(DKym`y2)knl5jZi}<`Sq2T~?-2ZHkAivE>U32(k2?#FGWC3k%COi){2g=ZMCa*CC zZ6Zf(xE!y#SU6jw`p$$7FhK&D%etob<VF#;k@tlDG~uJ2g!)VHcN5H^wnsa56=c45=+Qt#t6;hQ(~noC?{ zRFATgO~~uslWa5_c~+wQ{e-i&hR|&$E0e=jt?2btd~5geZfuTdqva5AY>@C~yXj2X zWU959juYv@e^{@_XTu9m1M`EO51ZDw=y*pAKP#(pU{V_i1ss7uQ(2_-0}RSur{Byz zEi-e4yfpT6oLrR<%Jk6{CgV4ITrsXA9IZ8JZFcW(M=usb^x}S@dPSC&UUG-Lki?_# z3a3f@sh85R^KBjdjrzb-2&!JT`!tmJ!uZKtT79aA#IM{4oA@@~|==IHU$O$2@ z=Mm`2*W(9LWb*t(iQWp1P6SU|%&`Yos=xJ(ZM}yU$Oo!}9gP{0*eINCYe{LjKA4q8 zz#t}_tB`=y;6ice)(#gV8T;n_hO{-szQ!;aM5qWX(G- zxgOU!Zm#Ak>pYr`v8wK|&2F#T8W9|~%e7Xp&&%_2r3!mA+%xUfy{6t`iE6~{*RPNo ze%I1d*6q?&LDq60W47P()a!zNHJt&-?_iBvJI4}wAsoNHJegm#E#j!|dmM_mtG50~ z8HLJpYb#Lowc`3b|H33zUo^|sDIAtZGy7v z?ea(NT>%$P-o(PW*%g6xBT}dV8aJd-MQPi1%?sYRDPR)CcXLguA@O-epNbnJh0?J& z@VGE`Q2OQmp~>sTk4f=q;B9%4`>0KHZhxAy?P;vZ%1+k!_(o@F%ap>&L`T350Mxa@ zKU#GXb@zUiS-x|rImlm;z(j3lcF`!Qmz=VECDIk6cT>3Kbea43{n-}`#Ma+Ev0!e< z*Rz`R8Pk*R)%KcZS@Y?gNF7Kd4RG*$8o@b2uE{&|uykG`N5csb5I53zlmo{zPv0(w0&zYYX+RE%-iKpr!sJ@JTd zypQHfW~Lm+x-txm#kB~zQaIdl!q{K>-ky!&Ar5BV;r8X5bzk8@1}IZm}km zBAuV;??2k^UkxdG*3Q+lvp0SuWgiU0=vGvTAlzr&tbrqx*;dl2dC`9zF(u~dYfv$C zN!=x2MSp30|8aN`z`Y_{#oi?8YnTLspGD}{pmDRt)N~C}ivw9eu~uEIQT4d=jrX{3 zPIJ8J891+Poh~SoIY9ZYFy4liK2a^c#3+E|<+w>LATJLf4 z3)9X@ct9GGzCEX=-_)t0X@=*q>6i(h2B9I#Cwr9NeGTbB1p9Y_F&g$`wPOvPF~BJ~ zgPi?~;cEg^c+ZEQ&4UNh3`}+@VC-Q9>jeZ71GBtSI6DDkcOv(DBf{|bd#Tuo<@sjV zD>j56;Xox5t=7`Uf?eJF;=D{!+8Xw)ybzyn)kWrSv<&5qpBJy$GqFS19doW)UfUSL z9?I{O{)Ripb=n{QzV1U}>wQLB0jjA#GEc|6N@B3X_j3=D@{&rz}3 z^VPWx(twk}qrWf=^z%#K))l8fMc%k$*I6C&01uWfIOt3Fqh%pdX_p3N8G>0b@dB2X zKht`+2rC@|d-)8dYT)XECfw9bGCc2Xt=Vw z{uJ6HsIL&`OPjL~59;N$M7p}a%yk6&x55~yT)gvk`IXiU34JXbfsE^!exrb^{TJk* z%Z%&Jhqj-_eqoA+0XZW{#nekt-`mkX%C7c|H@#gu=9+h#X;}X6!@)_gRf@ck%P|iE zu2}lKxW)qPdcA^P)0E&A?aW^zP{Txdiwk(v39+kIJf;0*))9CKBKw91!1VRed-gO= z6TvoT-0tg?LK_IZop<52tudimoH^CiiyW;;sI~V!TBggb_@LeZuKUe5T-R1X)FM+= zX>g`fZzk<=P1`$_;qdXseX3?#+-1OoY#uxDzEP)u`v>Zmk4Ig#j~Y zRTsk_W8L2p!!8w{(oa_T;dNuI@8<0HSrc|xE}*fK$|*$i-@#fm966EFt?H0qP>XhV z1GX#8osA+u4Sn)8-x)+|y<2yIW37lQpP!W|v#3Y$p>WtI>LAL{CFB$$_@6TxI`U;PCFD%)H96n3<>2VO zU}o))k=utvMI9y!31gQ|5(o}7NLW(Pn~xceIF zRl;q~q%v;N!fl3?X^ZQp)RLENU5XH_E2kCXmnrp~;Qwe%KRQCf+T1N`{@(tCaj>mB zqHa-_{kFRUT;1=L_h}uI90%?5p1BkTQ{y%loxT`zFhQu+ZavI<*}h!dlQUyAJ0c<^ zry!wi;?EejQ;5N7F7>L^J?*YaocpAM!sRq85BCWUNjIxywR^z}mq*5}C`|A8YY1pL zmIt(K(KF`wm+c;wzcrr3@p$6a<}@O-AW}DEZ?MK*<@M#PGc>Qm(rUC0()`957Wq$|F@2LsJt}lce3dL|HL9ocsu_ovcw<#Wf%?ST z+W+BcaT&4xUEHiN!O`e_8}|sxUTjLq&2HtIa;9Zp)g?`SpKV(5xemz!$hsA%s8kMa zPk>@@CPOBR09?nOmXP^eSDqYK#Xv)=QHDyRSuVv^dkx*aagLL( zh>*Y6D#dq|{N~$7PzyZMWs8T}Xd^VRkr7$x-eKSg*CCr8S$Ead@G*;tqrb7+#1g{@ zo5tHL0v&CSgTYrELZnd-}Fvuzc_|UnBizOW;r?5K;$XU?tx!t5TIH-&2Q06WG2&mX#9I%#m zdCyg9+a|e6{MN#5QQ7f%HFibKrA!y}d_FgQD9|z1p<-wTZ!e=(S&r;O-?(oYE1zYV zHqnu`Q*hYUQs`I`zsq~&;^2`JvmsUgU?^OqOJ6=S>}Zu{cmjd7AbHe66@3ki#=yfQ zMveGrz`jw_WfTe@Ro{!NtAQJSjQ)nPP$Z^|SUobd{MZpqE%~Tkf-?6`aJ6<$H%_1=6^Ra^`b8r8;Lv zdhK4OYBVZCI1B15hV9Y>f#(!``SA+4rqgsf)+7c{Hl_LTj;cJ6nR-Cd-+seG9{jg3HT@DC^eVwu@W3x* z8}=xS^Rh5fuRip@NIgSo%#K{4exd7pMF|#oGOy4ph&t@B#XF#Vul67wsn>^JfzjYkvrk`EuSPKs&cbU z*D4ahCijfIt{8oNnpJ6e^#2W9nQ1Xj!j$Xq!F*e~@M3a=O^#`xEB1t_ELd`D@ZIJN zw$y1wQ*|P3!IlK&a_7jF&PTqUfa(>!mv_>NZlj_Bd4)Fso^NT&-t3PTHUn!W< zo;8^JM15M$os+)Ef^}-X5)uLMx&Rgrf>hKAhm)n78(eQBg*U%3`OiFUmK!eB7}1Nm zl`6V2B!N}1DR0mBkW;|cY_X$V4hsao{flodhbx346HC~78gxv5K0~yBoDpqA%6qr6 z{-nkf?78`u?S`lT`}40DXkI$A9s2KfE(kJC2znNuFmLLsc~!TqV`yP#l+FM(m8X(| z4jVO94xKRs@&XVvA#lb<_pTW8bKMQhe%(nUZJs<>i!qFlqADTfOkvrH-qDrbSvE7WMnu&R@LDiK%(99M%xZu)~yB z?sr)^NLhlb9}gc-E#hRhtdoQNk<3wwSJSf1z+%&rHOu$k{^(P=Z#7)I8@?k@kQ}Vo zYZc`GPkpGNLEu5|Xxn@CAgytgj7DbL5y#nu)^Zs+&-;t-##d&1xMd9;!x)Yeh0j4J zT*eTU4-+{4D=7rOLsPYoP^(hLbfPhStTg;~jl`e7QA^bSbtrKVs7im{3@tQ=?!R|7 zq+fb|FKQT1)7~&_b4ACfV?MyKxKj%q+Y~(5+TlrfpbQ`im)K}nkC2GxtgpT#IMBxU zXfXa1oEllkI?FRMNl4F+BISo7w#tR@Y0YtWjNB>YQlmysu+%5Ofy;O&VBP_agK5anp;k{*dEe1hk4-AqToypCffRmhq_|+J>Y|Q9;1jX z7idQGjJ_{>fYf|j4(KPwp~?n4h7o3aq_NzppmZoS6{H<6vj{C*8!Ad~uN-NB9;JHt z1vEVH8d7?6irV^e57TlTT^u~3x19D?WB3q1o0qVJ=#LcKttaKqsIgcdCkOWyjQUUf zhOWSx%rM+@ks@Pm9D3&B1eRI2jUn2b<01GhKaA%WXv~B2i%CygTt8fidc-0TwHMHT zlI0bS#rdZnuKeFnQ@*RtfRuyQR4brRvm?8;Q^|GhwUOA{eXjxPzQ{~Lt@3-CXDQf; zsJgV!R_XV>iPdAU^NGE!eki9sKR5LJF@H=dHjsxdB4l6wJ%w%%mo{^R~i@sOh z`C^kN?P!aAC|SP~xi{*ru&mZ9lFk@>bNtEE;eKn<7Xt@-$FXIOa~i}IS5;Ql60>{S z-5uFk*KDeOG|q(e?y|Ml&Dnb%_~bgQ2d5GoH$TC=5JE6*B0kX}(G4pV$Zz$(&H|D6Bbllq=VXfsaN%^b9qsbVK{j z)pVb4vrxlc>#k2EiQHRa?l}$&A73tQ{V-@@+@S{FWY3(`rGXWQP^TqD1$C)_hZGwe zg7abctzYdnHVHVcpiE8~o*T95f~88>ev`M{%7LANm2%AXxQorF&5%OYc7&!ep~UO{ zfFeSS7J9Z$bU*tImR>mq1+2s)SaD-dbT(RoUnpO9nQg%H*RS~s&)5kcdcarHJgwOY zU&DpZ$AX411!^BdVg9mk`G`;L254}Z@$9)?VI|bAsK<=|XqGMu<3K2EnlZJK*kK?{ z>fe}_zi>*`{w`}+VnSX@nuD8rSYXLj4f?iaJoCU5pT{yo7Vi?+jhv#+YtA{!;2_^e zy+n^{lLo@z6td5%3cvCu%$D*b#43=w)2aKz4ovs=IC)DwOoo42t<<9wteFdO?2_5w zg7HDG&?sZa!mo~&>f;@0w&3wPfKy7h(TeZQ@zk!%o>JyS^^LMyZOf)`4wOSKP`+^cEpi^=qNmp_{`upfTI;!Klqk_YcGwavL5YJEkk(-i!HAxUj9D; z&~)?ON3Xe?NjMew&qaRDhKotLz4|8Fx>V@c_+stIdawe@XJ2wB7G6dM3iCyx;zG1Z zC8||!lZkjqW5!B<L!I z*G0}-1EmPKF&}4WY=4=Rc9=z?ES-EJ?;>;xiK~O%?+#zZdA;NeQ$m3AV&Y9qeAX1X zbl*mgdzg3}#Q#s(&Xl+L4V%*CFjf;Bji?|S!b`@a{1ptqum7vS{|~??7K-PiPIexZ znZ@Lrqjj1H3z&EYD{(*FF&e{IxfFCN{OOv6t&g)Iurl6Q; zRq#!nj!{bBu^;*M?wC;crn=|L=!%8p&5nwFkwRO315<1uEpCW;&n{SS zRl!!=lP!8&YuVp_soMAl9n5NG8SLTOZM@$NPA=a;Xzl~J&Ts)*a9Ih?Q3tQTwC!z0y;CnoR zPwv7+*TXpu*hfRL*OP?}a-Iu@ssyu)be%$Ob*T_gm{-YuNxdBzlEjJ3j-Nk5jCNZ| zvSl+q!og@bsa( z$hCAAM~y2YZ)bknR;+w&S__xk}=w^QD1f4rE}f?`P;m^p%)?R z11A8-Wvjh%QLnG9H!k=|8j<9WlIRuu3O`PJwoe2a5??8bG2BH9b8llQhpT3q*i}*X zEA~dJMYV7>Ea-6%J9|6lob!0|IfZ<(gx@yaHd(AGbmsQ46TkAo7U?$Y&*48Ej`g=` ztjYhoi}c_o;a8G*sz7cHzv%e~(C*Q&W9oEky033~^*@=`3RCMGBkV)vzp8R+Ibt^X z5lm8g*4bPfEKY$>Xh@CEZ|1TKuqqu%^vv)~_g4WY|NYRjYn3{|{M|L2o&A;_DOy?~ zO@CR~Qg;mO5bByGU(!YyZAK9#+_s)ld}JRm>fj#6^jc?MpQG!f8PQFCs0`^*HqmxK zt9s(p7hzIZtkilkyfY(}KXo(Q-8MYetkTi{4r_Axmzne+FtCs0Fc0E=Zr5g5wYbIS z*U>0Z;HApa8jTW4&;r|=vgx27aBWL8Ib>il?*Mv(cuS8SG&NQPB)o88PWuM79vB){ z+{P2BN{s)BsS^JaQ_WKBlO6XBUzTpPEJDd>L&*>TA?d~`utuJwDCX)^Im8CW9B*n% zyxNZ|XJ59tDy3pt+L(4-hme<2zJa$#Tu*M>DCsgNIXf!2ks#xqoxSBNhlRGb zb2^M8@-&CgD#Atbs!EuG7p4m(G`C(MC{{Tj2(<@HSs&K_3s(P>sE<1vdcNZYGF(Cu zd~hU&(%Ol=iY*^pE_Qr0 zFEO>5|L7CA2^uawn`q<;Tw?-s-1;WcDd5Uszb;D|v>|HI*SLBd*__k|3;xooRz88Y z14{&b&jtI9(O%~jubTXS_uM>Vi>Y>Jr_>A!C8I2kV!rmns2$ZWq{yItMw)Vo$aL0i z_ZhUhiyer5l{GzT^|2QbpvjQoPxdLb2ZhSpco}17Hr4r>nn+~eq@`%6$oo(K9{!rk z|NG&4LhK6(WbnlC?I}e+hGC0Rbyj!L;+H$7mNHkRtWFDqVq<@Zt}M~Y__ZPS+x3-X1PPa;+%mKP(CY{s{vDwy`R$FfH z<>^}47G;Lz$5+&^*anVrK*wESvn0jpOfWiYz1uewVkl%z?1`z1%3u6^pRvsO9d0Ge z&cP<=n&6zkdH@0aZyH0)xaYu_C1g@a!c(QIrAo6*7eA_L)NmkeYb?hqWVF6yzO1pb zP#c@J*nhbf5ruJZ;&tDE42^>p6>J1Y#q`{7Ort2|Nj+n_j1~!-?VtX+_*$1JY5eis z=(KSEAfG9rt|igIRpb{~1kNuwYCcAm#W{gpmUL5e z!~Y+wpyVTjer>#c$aHZ9*6udn{8=-&3~`R?9baX7Uz!qE?Ik#p?15X|z@-X7H9+GW zG88z3uEDsqM{ykIu1l%Kpw=1YcnPpqbDz*Ctu|+>6W~o8#uT30b9CYQ!Aao4vNe*I zu8HAK#M(I5CbI_A4K^GqLho?>mfc33|4Xp0>av9BEuGeIn-|AXg{ZoljuH5n=3$}; zD(ALWZVg6au}WdYG_P|Ke+-Ww-PTzgNm?{Y>w58nuc}QgGM=;6gZ%kdVajZPb8&K^ z7H97Fp(-U@%*h&y4zE(gUb~LPZdOe^Y;`S zNqv;&)*W-Z`3Jp*9X2*Dm&tR37ezCbTCSxU|Aj<5HPDl~@>n^r*j72m30&dz{Hc^l z!i+*yf(~1^+&4u>4*nv3rX4p);5+ZSjw_&Q=RG}^ueH#KtrHhEK z_v*oa$+Ipw;DT6*!ZvO`kAE|+-+TKh;N)ty+pV1o)2iyzPGxm@9C@V9_&=40nMHM| zlg*GHm%W{dX9BHG%i>_%@J*eIaTodxg}EJu=cby(@pl!uIy(Q!^2Ft5&sJhLYvRYH zNK75l*$fhLoh7D6@O=9ILcmd-&rdt6OWsTsY8~i&BrC~`xQxDr1W8P%Y(=Ul4n>@|=bh<8gh~6|~=C{MWFgSCw;$dE5jMbL2pEjs! zXv5vTecRwRlnVi`Hh!9?phR4F@Tck*2WHhYxsQDxQGOP1MzjdEd$sX;MjqESe}u%qnug%l*xn;V|Lb%$=1WkQH7lp-h(ZXnr_xRsTevm-Z{((y+6m zJT@ln3zk#33!^d{v|xF-F$P*~WQf-)=Y4d8uS3b}*lyfjVX^G4g`BXh>0o1`N)Sty z$X?-NQ1y7u>T}s=w|59;Pk8fSbeKnyPEn)P(?qZ&A@jIGp3O4)8dcB{WUlxKS3&s3 zBoC|RO}iHW}4_S8vW*epbZ7MUjSIZiQA@ZHHqe^Wo#G=WOynnOXU+h%4!` z^-+GAk1?+tTLt8wFHgUZLJZFf>C^w?i67b9DhwPlEPRKsc+p;trYy(|A3 z4;|f|(~)ivq(1FWUEcpK{b_@WmDTE$+5rC#V{aK1SJSQuIcKpJYw?G!-Br8lzAr21u+GxD z!BXB2CMe$|15BwqvTD2Cod!e{!89tVca3*fe&(CGG2i;otB*61+Zpqa-_R(@Zezb? zmamq(C51?<_Nr}u3LDE$^>H7)`qAvA`$d6bKPDw2g*`(pH0|2}*8*kFnH&xDAipH!n|8u-MuyzM-(lL^*{Y-L+JB^O7*(Ouq2kOqxjGo_>@U-h0>_twRddd)_ zar=x{yd_K3+rDj6iO_E#K$LCBUPd@$@8!U^9RYmNlK4k^i==DovP`!4MPjphDkEV% zaxg`>f~ghlLi zu3^kIy0&SeM0)mOU~U@tST#nLhp{%eRb*=wb$xAzA&i|dT=@vQaz`rW*&&gfzR|Tj zAgYBHj|%IQ>jvVpmG%TLpEsCz$LnlW z%~D|D2UvSQk3$F^M&+n#iyKx5HN@}HtO)y;hy&RvP16hqmu%Ad#484V-P*mmPOHTT zkKDf0>k}uFqrz>uS7lWgLzn7Z+ibPTj_v6^dF+Da5P^pex(U7x_eER_wD1 zU?;fw2f&%FBf>#T4N8k#K&&nkF-pIACy!`h|2s$F_o##FbK{!)gV*m}V`wB09`|Kv z6sdE|92I=cFZ3>o44u}acH$euX+CcVrw$&7YBJWf+-If2`o95T-K2ES&Z;5o;O^di(bfvtplD&@%0(PC7)2KQf2bmr&;jn9su z<@elElTt&INCKxB-E6cu`LtmV|L+79Pkml5&bkW$6JE#En+99K6whSWgkMfAT23{l z@lrOD*Bgobo8fejQg>~(MOdk@`@*7vK zkHUG_t)?LzexMP9+bKQbz9n~d=URfR33$SJat`$b&kK2iCpM(>KZ+4Pjf8KEwE*5z za`&%d-zp+g9Gai7)U~<#;Q`%Z+? zTqgjR*D$L4T(uxyzUjafbj{~0872v=USm;Wp4JvAww`y8)}L>ad{QD2bb-f(a&6qg zfvPhT^@np9p#D_)5kW`~_RmtbT}M@U?FfEE>AIun zs`WmmB9A$?nqv<}sNH+$_(1gVR_V_OGt#bR-_<(D15rC0ykG$5!Do5knB2Tty4-NX zR$qEcrv(<^u`%8jU$pG4W@qzat(aH&c2w)x!f+8FZ($wd#l+hoI2IHNZSUo^X6Xap z_Oc)LXi>F6rgJp)EC!A_KkE|%9uuIru0!3OlPebqR8t)JkzV5&b2qQ}DDGu{j$COs z%DdGo?*_u8UwAeDJN!&b;__^~tSx^(g3MwydCFq9)WNcwVFD*(ynv1jKOxtc5RPT62D5ejx7@sI?1omBd{dgfFz zRH@~=Iu;jm=xUim{O`6mmcn|WG*wkDOgC7 zu2R@>o$3cgIM+#7qTx%rf{oeXCw_Oq^ zw-&NV3qLGc-n!3Km9haXgwEUJ;RS4beQP2_OU8|?KA?~1ec5E1QSu}?FI((t64!hg z6=l?m+w$R+3QU7gJyOSa@t9QZr**jDdPNm|Q9`ySMpiba&~Sl$IM+uNP^v%08iC!y z0(&1>hDFhPn@t5y6bR+B$Lbe=F+rf9K=>fhp#>Z?b)r0ZHeOtmV4W8hp9B;xAl6ZE zvgL5|FdyMo&JF!m2!+JIS1!Zls6XtByaZVe2-nx&4@{%ZCCa*hxFp*wWRZt&4Hu;L zB02B)%xh0JI)CXnG2E|C^hT0t1PzVwNpyBOdAUm^7gWaDHtIZ`rE_2>gfjs3$xOm;oM8{2>~ET z2GeNl3rR*9{q;~2Jv&V+?aNfnvvK#zn`Y+CLVLD&PgTk}T<@cF939I3CCT&~164Lv z{@1ejT|uiFxs~`=;U@%61ar6uBj`UG8;7Rf2IFb!E75tG_H9tujhC|8-Dl8 zj)$$j3}9=?C@Rabi~;JElj@`zF`|)IYzlwWikr{_Q5vE(R3n@2(#lG!FUJ(-%XZ9t zQAtCkezI;o|EgGErvnr!Ozh@KwQ6$LM$6lSS(E^5Om(mUaA=VK1KF89cbc2dqvGbF zdw$a29TgG|iY;rtOS~J=Uy0EWb&(G=+TpcL%^l+^J{n^(rAA3yT(^eI<}{Z(tD;7Y zXT}`krFJj#FjaymWSH8Ha{Lwsp7DX|cjYFGL8UMAsTY~sV0WFc6wwEb29rUd&s_~k zui4EU()<+~!spF6SCRk%K};GDt7@!jCU8d8&vs~%7?fR&a~*G_x3<+w36lR{2{XMw_f+umhUTr(1qji8IIiu7U=6LCOo%r|KOnR9i3RV_?I*59(I$ z#gahvhbNjGdpcAOLhxNY^KZA!z+(arI>>^zzJS2>b{gX{_r9xpdTRHxmNWk15OT4{ zCL3Kw89w3bHj9~S?hRsoUiSN0cnH&u7t~0ED;ChNVhsO>O1BCyS$@B*Z=D>7h_;*< zZY~mST4rmSuTGINetvVSfb_}!;zoG^99iY$Nne)M`Wd9-T4SN=m30>TOOJy>RTH)9 z0~g+n9sLL16eWhf!@$%QgjuXkuN@4}=Mnag2lqdO&O2P{9alpAl_ZQ?i`ys6ls5cM zl^U*>61Kh}9K@8v2V;}R6+m)5cqv`c>AgVb2f#o=m;qNauZ~d{QwUa^4*U+wpWS4j z)X-=U^*xNTu?m+g5w}Ze+w@2o)Gu!Owme))g~`=ZdFWf@lY4X#i>$W z-~9^NVnjEovS0hW*C}U?=##=#aT-GBnh|#Mr+Ydn@7In#t7p{;LSL)bwZ}At_(jiY zDy3fx?~+ts47mG1NwKx|K=rqdn;xVHk7n2Dy_u$OCZ}dkUy^xCK}9-?&hbBmAP1b7 zrqyvtE9O?;9_y!=lOmf-Yvpd9*{1JGPuTz{O-tsIw1i4y2B7QE*=@1))t7YsrHw1C zc$Euqx3~ZL?q!Yt`R<3f<*&_=D<^c^O*~Yg9&po!&1~nbet21|@o1|I4$e@_L!ny; z-Fe&|xx8jtKlP}z#>HQGgWKI44{qU?j5OXTd>gn7a%5_o8|OdveOHsc9U3t38eqbQ zM0U4A?0v>yH)^22#poL*m8UbZNID-h9?mg&Y#Ar21vD|XSipA|vJYCGeh~Scey#tjVi@|<9 z>K-P`4yM(4JYDH~k+K}{QSXNX=$yql+s$dvacZ|-xb121#(-18JON^YL0NoaBQMGQ z62A|uf4={Aky*$7Cq|A9M8x8pp_fVSwF70~d!M^e#U)0P!`Wh!93c|vT0MlRD8Bs?x){!1M>kGL zy86S}?rV}1XBhtdvG=9;H+N9J^p32aW4+OX9sB_=EOr)j28lF1RoQH8+x+L7lZ}vxx038- z?1j3n^M@W1;;$KrsrK?pjYdJ5)%-2;UR>0WJZ-I1%Bqne^Q3=ljJwgf*C{3XxAnE^ zUW;%4coHqezvw0TY7A@`tSN?1tJX!+&dASb|Btbzgcw!ds}p80F1Xu#q`TGQO#G?7wiYgyUi7L*nZVy-DDr`nFn8q-enO0fcFWns^59Hq+HVc-Fe&@!$iXG< zK+noS{ZT^H>Q@yM-h>#&OU+dCcKtQYgd^>+#ym_6y7$X|OK;ic8@1EH$DFSFi?W8t z&5~>DGAt*`QFIF@*sF3ECKdETy|R^U0X!qDb$=b6tqnfAFKtlejtmi9E}-QpeShV~ z2HK+i3`l?HwZS`2wQLJ9O?o%>1u;nwEsm9g!B26gLhh^LO{w6w-dtsWV#J79z+MlI znWh0MUym@4s>GmqDXS$vyBISzm{NIY04v$j_fRz)U4tD}8O*nlPe#mnyDD{Nhjq)!P&m8^Xo%+s0TFxX0joUfqKgG|O6SV~)M z&rH}WsgjIQKTcF}zntXmJTTf^CB?RI$7@vCJN3P{kaEeSFTrzv^PnKcaqvOT<>xYz zeJ|4UQ6$JbU5}O(KMjr*$17NW-q4E(@GmVj|NK-K^6$YPid5eb@7D1e!&Gk!?rDbp z-O%@sa@f?tJWg|;=+#0_iYtU4;2sXmCRw?|1FPy#l2YfuWVa%Z6L8!CNXFH(21+3l zn{fluA+$Q(6M051QF%*2X*}GhCWeok;-412(@h>X_DP1hMDZJ4mESEgE9#2xUTy}R zm+6Y0G8tlq2iBU#y(x)aI1}I?cdE1sN(LW1%}fC*?TLl9AEN~2Vq$w2*W5=71yFa` za)nOqqVLU$WwH6_(mPDu(m_URL~Gw-mUc3&#`h-PM{sA0*eWOMIEcb+tgsiN|6D3o zyIS-zZOb;1^BIL4*GK#+UN0`@1>dFRGAmC9$T24hw{!s&}0z(;oJqj&EKi2(xa)=|cWzFY-6d>I3eu z{QEs%QH@eYw27gE4{3Hw4wY7E-|5PNf%8PohyUBhZcHsV!pFwla}#XeLpaFG2;L*4 zQqr4!t*OCbWV_1=Td`_ zAP9SC>WlnCqeGFZ=9RYLp7VvMOYDsjv{TE`LVq(v zaz%GrN!ZA@nFpjia})L6kw#~LXK7Z4br>*A4~a+&9VCRHeUTqdUgJQ@2uwDip4{4xXDBaunWZ%;n^ zX{5Vf_&LOD2_n-Abc6VjTRYlbUrHD6!i8I5X0GSe@BqCYGJ)t)29%;gWi;AYoG}V; z5N8s*uJ;K@V5=R}@WSe)3SH#T-MC(o&!cO-`44C851u2<2v185S}yiOU#pRK>&-iH zY<~;k*Pw3ytvU-fnLGiQZI8zDZ7 zvgdmbWQ&2tMEIf)ToZ#XF$~`lzvr2hM6)z;&to}O`Q2p!j?t!=JKN5EB3-NJk-Eyd zDoVY6xNDu!R|knZ>G8WX@+(T6PLUQvX;dBNg&WALgX;JT<2PigkAK)}XzfVO{fDdjztPR$rnw^MWD%LThsyCWM8gD3?bKP_ZyZ^ERo7orIuspacdm_7L+cVYG$a8c`$k3 zzLZ2^$~)k*|Jc^Y=>YPCedh_UmBaQOpgSk zLcHrg$MPr%NXj-a|9RLuzIS6N`kj4TL)S^e=W6=8g<)AfDj9iB`9yJVT-$!KMWQ2w zcco$fiVLLHVxGKj&!vlK9QnKW9{mZ}{;%u9C9y#>N|XvLqfXQyMyjj#UVDMT75p&E z8h+Stj0ytErLHagB_v2lnxSg`_sTiZ&1UJ#Ug~9bzB2ExrH`vTYMdT2z6Un4ts`1f zG7DS+)NXpypQnm zJhMoRYfnhlZ5A-az$8=FA|M_)aHJ@9eBH318Zz4YTDdl>JQeEnDlrVx0=XFTQFZ4} zup@KTjt!z3KQUTN!?_kpj6gkw(9g31HAk4k`r$ccEC1ct6Psr=hGe+7?~*@iNxLkc zt*%U?;h4z$@{4lxkA=A)L7&C?EeFkp1DA^1oc||G$SIb~Q`S zZ^PYew7S#}Mi2X8)_ns+?0yveFDgCX`p?O|bP(A7>f*$m$hOuiH{`OjszN3A zeTL$kVV1$hC~KwWIRku6$Zi2>2L!fsIkUC4olB0U3n3r!4hdfjwtCFu_ z{CtLXjW6hq*~!3ey_rs@=M+{PWC4gth40T_JVBt{{{eyiPeC}K()>3*HpME_ z@GTCj1oe46M)`;Awu|=7g2jg!dss5a3TrEiy{D)Zn4FuoM{O?i6ON^dg1B;C& zg7zAEJ`U1f??g!0CytX}u8gjnN8K$<ft@cHk0)wdA&6C&$w|b&9NpK-eu;MmTE~BTo{nsT z&+yQ>5{FT|G=wdWdtnr#4`MT(XQg*rr_2VO2j&KM!wEUuZy6d|!DOyGFNRFo==@1^ zSbH@+Lp+Z9Kn)lRC_q+)E@xuR_MuDNt=1%)oZ52PyPtC&QfZ?(6O*-y^AS5wFm!=F zVxWroG51wCSRz#;-*IU_Pn`GF4*jL4aGtq%?PjYV)747flD8=3GtXp1%&VkBhEVNW zo)qF}K!$Ft{&}MTV9OZ(#g>^`Kav0cvDl7m%^Q*8egupqjDAq`Zu8(5WEgF}sl7rB+7D}e z=_%yO-S)<2h`m~xB7Izk^niGK`oFW4|7&T1`1ZQGn(=YZHw_*)WADyKT3}(gqVa8D zoA)>!Oyy!WVH~?Cvv0-~DYlT>FHWXn@4Rnu2d%xTWgn~{+5!O=u@bJa6w}tP3+zP; zof!|7`x_mhYUjzo+Z_*7Gmfrd-`Z=QugrYmR**rhR1v;81i3H>@9h(xl{M_9^V$>J ztU|vG053#t=zrPi13US5O7$Rc^2Q9|h(AL2S=_pEMR@?y5sODN( zKeOEt{UC2^wfD^!9AN>|4F<#boAWp~vIJxGfd^mcv0I?$Yr>rAX7iRO^?6EW$gYbW z;JPhbBl(c;R)EJTp8!IzVx$-Mg+_xZa_~M|IZ6`>1q5>+x5C$B4}SKutk$~=$@CjY z+C14R!v5O>w_8k&FmgqpN!}~;m)^XrwMY=XaY^vt=VgP79d~2ZZqwmfPg!t#K@qLkp?SzLC^7Tm4XrooP19w$jizbJ$DPKkeYxRPc(NQtvj^F7BAjnftZbgxxD zfAiZ56}q!*N@4`k6Uk9<<-#6yoAZxH$K|cRs)&~##{OO`$rSk}c^P=2+HQO46#01% zaNZ?w{@1M?GHZyf37IfKB0k+EGM4>#*MYP&0E<1_4D6IPM5>zbzxLpVTaZfEQFq8) zda#5Fn`bKWMnnvnRD40)`)H*GHY~r}AJ2x(^!?DRvuX9%O3bxeYql9|`w2@nVTJ7e z*{Dn_)@MAJv1OcK-eeu~CFVDjvFgTlO_Heju&(~Bh3T_dr02i}J9NcmCrniQ*m;(G zRs64(lyF}y_GCC`Mzy8PVUMWIJA|#Hu^`OS!NxHi3BLnvA9EL*KB=K^-qfo%`Ok0% z@mX}~IY533!2vwXC66^ccM3R=hYQa$dB5XWtvk2Dr{mgMVGqs>$tvS@76VA(`NlJRFhCR6WBOld zYUKIGCc^!XFiJ76*e;N?j!83OgD zNxQG?;OnS~{Ugje2O0@1v4bQ-F&PMAj{r+YC?;gR>F`#`leTQDVp#Q4SLn5uEA#~a z`fj2d)?gU2830BpL>x1sW-C9c_Y^a8C+wZTaI2R}i?dQ!$Pb_7aZ-d6?>b>3Bz$N8 zavj~MH}Bs+_B9}j3vmwp&4sS*XNF80ow!zC7ATeW2ahFd7!Nybf&uc~9x9hkUAitvC#53e+f;$2qa}GK8Nbl#YjAFutZHiDNnpXQg zT7gKXeZ{sb#!HooKI% z6fgQ>g2Vd~64;_N64)ZU13{>Z{X0ktQ|5V}t1hgATh;dB<5HbM=PuV+IWz!Jd_vcD z*v0OR4Ah_Jj3{95oQo+S^>nldm-f%>1{=Oh?3iI!K2A9T^_`R5xCN-n0>)(>FW)K! zv*jET-QpI=N*RFT%de>TqSEo9)$MF#r$bB9|nk1j3KNW4k8HVoAf573W*Cc}p z{nWs9CU-E~K_Ya9f;`Ptv-8hWofoesrwuR=uG$11NUt}z6ywX-bp9;9mm1Kx-%Dwl zz@$#yGkVlmo4jWfauP2)f}xyk{Qc0v|M{Ss_XbxjPBr|ak=7URpOO%(ORDbdJ);a; z04BWH-7mvWYHR_|TW0U_W@zy)B)O(oGYtGA1L6z%ckzmO7;2UclP$}sMSP19XN-K{ z$A(4EADm7J5)JHD^&8<20DL0|Er93DsJ07;uhTY~*uF`+GXj^&w>0t#010^z&X@^q z@5S*~vT#0)-V0mHn!9@aFSoB7S((Gx|BL`2jt1}GQ`df4zd3|5oLe-L9VG~s5#Smo zGxD2B2*(<{KWXy1?o&do-E$t!5$A`iKfEU9!MQl=JF-E(R%7{mHzj-}FEPVG%!fqM z8KGfc`884IrRQ@``ON#m#OsHnnDDQy5C?2mPzV!~bLQf=y#)*`rh!Xnsw|Uw^@(w0IlLIqH=i@Vs zzpUVMhE-|*4=(MHRZRF7Qy6y>^{vqezwD+wB<^j@J$dd-8%}WvNDlB+XzYB;uv8c<{TE80qx69*2e%0P5p2YTG zHbA)kI8)eZHS*B^n9I0ygP4baFeAe9y!BVo0mZ6{2s*Xpi16g3zgW|F&H`R0vVEtD zyRT*_$3fAW_q}4`oENp;yg-6Ndzpi@HnR7=8p?UFYV0+w(eklsrA%wRMMbLL5mEzY zKN0ySBSJGC!O%Be`*Tom5z!AOgItZ+)YopIAOC5(X?#%!T&m-_7$XRJXTIfR8=zIj zk*~4Q@z78>4>Z+!p6_=kL%nUe(;8bH-(i|K552i4rdCxOEc#rl`M-tH|FRFx{9n{3 zg^AJzjTl40R#uNEy8grLawlxm274|jFxX9r&XxN|F6DzfQk0}fJx}-* zpe)_J`mva670n~nzx*{o%iwusFyk#K9U=t&g%}Wpdqz{9Zv5S4Re|X7L(E-;Be74N z3~AKgT|ZoH2KY9WWR)wRe5@PP!|eyp#oNh~B^!O@Ff~dSHquy7Y&R=#I@vR+>t4@zXQ-t*i$^Raf@&>Tgb1<9_3j)=dsA znL@p=Wo=aTwt9=7siCZAq#x5l$T3RyV0zrjTJ!hJ->*AMs|HbKb8Kxlg@>Xa|5nIu z^+>|sE_k8)+(sUD6Z@S}A0tM1k*!{#x4OR-X*0J&(wi4ma(A^SiJ=(OKo9r*>9IYC zaI5!ANM+M@x+<+`B>n@Kfuu}cC6u@fPy@tgNANCv<#h#A3>#OLAedP)c>m%?Mo>9+ z`0xC@qH=mt*_MUqt3vRH>vgHfFH(RNUFX9B1K>+vGo4eX>H3{}&GXrw^W&Ym#Hi0| z>Z!#ysHEt_DF<5twZOto?BI{80Ya`ipH@B<6d>3~B3V9ynQ^VmL6tH4_N}3rdRktb zSyc&xS+Z54&eL%N!YkeAe3+8&^&Hn3{J;;ptPZ^pg=Yag_h9N;tnhG;E6fm!m*=Xvn#d; z7mpWRTOT8uIAn*Hq@)0Ip^2+m@ztseem#AwytnsYuh}A{=#iLvqW<)ecb#aYV*Z(Mfdz6KJf z>XxeiQHV^6{=2`OvsMfRr@Q4)x~q)rGOx=}Fs6h)h>cFvoS6cpf!*RZsVKpS>_acK z^Pha5X4J}R1Y^rvgg4ij9_|KT>Jv}YScoWlS-9^@eI=Q>Iu2DtKQ%}^6F)pxl>^9} z5sJ@dwq@p=_;;N61^c|e#{6&xjJ>gJIuW>yJGHi)6~Wq85=1gk9~i-1xtcM1c#f4n zsZd1O_qy1+AnEDW#WT7#gon)>Wl-?M{@YAuAUpQyC%LxW-Fc;wg_PY(&&{nJFU}T$ zT*9svN=pRb*$>$jYY%#h$va6@db;<7fe+rO^WKGdx1TXt+IMua-uFm>B6-`i6s8Kh za{!3wE0u;r&C9UDaofO>EKw~_Q>&$Xwv^s_CkCDDYkIpH?XcM_e35F8rm&I}t(iOC zB{p*~oV1%z5+D{`VO(>yFR2o%Dhq76hp$)}yi4*|rNroALI#XHpx}`Mx<9d+06QTMo|)?}ju0gTb_ zrvY-sdcbJ;YeMX{*|Awlj$#5!Q<_%o?RsbADtRJlljo+vWmoLH?Z(SP@=ALoy|J|g zxi<6fU$DVU{@!j%8)}KK9ypEML@}wi1ApZ4`;i^v*R1dubspgiTH4B3NW(bivH%|| zZbQ+v05}i7nPZIlj(EsKhu1qDPt2Qo0oMx%t8OWXz?}4YOuvV*II-z{85r$6-@Iz| zg9)h+o(CZb7DXIFh87I9@*|4*Vm?F+n0x&n@0+}K8l;!sDVVX{IOX?v#fw)B6s5K^ zuHS=*eX?KS`^Qnzhh$fL2Qbf;{~Gf&<;G6NC}li3zkY%h7EcUJG;F7RX%CKq!`HU&_*B>?EXX4>JGZD@0t;G#7G$H=#QyF1w<;q!v_ z)Y&F9x6cr^&UvJ`bO7$~JU~t#la>cD*^QJkbJPI1x&LV%`-kKrul>mOpC}okWHd-< zXy2R(TGa1@E(-MZcNW5BhcR+Cg)wLll;0s78pD>bR=>GLu4lJ#)?6fH`&1GRDX&vZ z8o_1{QW3+o#zHsdlQ>iErq;OH#NkmXYqAfXL#v{;Cy|s$i6EblP!OLye&eJrBSG~& zy59Vw%PkuZ;X@U=kv)~Bfr(<{$SjS>UVSo^;^%AA>&glCj|xOfxz2RWr5EYjtQ-fd zc$;3+0zIkCE$dO!=BbWk_!>x+g&J&q_b-}>%`_!!j_>D#gi7z6nL(a=Hd6X0-_2N8 zkDF?EPl>F3f1z@JnhGIbK`Cg(UA;lQVzg%v0%HjhV<0&!BeiY1bKDNT4nbMlcJhb| zZ=~;}K5W8CwrDFV!ZHe`(~43_j1QfzMaV8%Kw87BMb;Tf^ooHNpTl!n0tfNG4G@sP9MQinHyizXN zP0FDYMc^Qq3G$v0*(B<296$unihz;l`2gJ^;JUsPVj=zS6tWPHvZGP>6}AZ zR^++GKh}V6(~J7@3)21{qD>@)CHF`z!#Ea9kJxDj#)|lommir8cyq*phD>{+^DKiT z67iLmIGp^*{LIlz_j;d^aDqeIS+HI`U?5fD(MSb~Lqf<$$tXW%o*J>38TgPueMK<1 zOGyawc)ZAk@=OmVG$Kr^n@!+?D3uKc50S#ICk4Xg)@{YT4|5(a?*mOU{3@v51$=nm zv|(H{B^7~fK4{l$Q6J!O6tT^YxXG2;j&NJDh1%v>9^Pgw5Vs%o?7bn_w0oQiXU)|> ztDH6+?)=?MH05I<=d4+|R(im$c6w4(HheUBEd38zo zr}Xv^$)xI}`|=o@5X}uBO`<{Qns(n&$Yf+f8_=~z!JFMvlZ7Ioh=mj0+xI9C~ zYOvGv=7X*t=0W{*kwMVhe(SH7^Roctyn)90Q zkQ64{%Ni5-oSXQCXour#9t*w{QD7Y<@Mf(Dbimm`bzkPTNWw+Xwve1uf4|5->J=8} z)>7ZYCjOn?VTF#9IF9~d*xD%fM9447wqW3k!$C7t2cSWD~v z%PY@-T}_rn(qf0PA!EfBlFKGawlqOKsj6E_V>>c=NV^12i5I}}TRcy~T z4n`TSM((&c1%mb{uqMMTAHi)7Ka=-xj`*-^&l^_R=v71Ms8=EWk3b}#bpi=qdb(&X z&g!clJF*!e0>-MMx=+3B5bMddW!+1f?EO$A(><)UVWwW7t55|sKDJf84?lNkE~jJNng7~cSTnZ zMg5GmquHs1&ku`G!rQ~K-SPucie9KL%B&7k@g(xc94MP0Cj058wC-aSjRc0ZIY0kf zt449bXnR?!QOSdA|)YwqPTR+6v7^Afq@^L1LRh2H%Z zx9F|=Vx`KB%lTI_tN^wuxJ~|vH(QwA0=gNZRxE4zukwiDAaz7dbx8}o z-&-^qd+D7>eVN1t8>Q~&K}}Zb4eM+9%cg6jJ6R^iUA+w^=%U+mmWY1lBVwb^>X&iF z?mc~)s1bCnxBP_1j;JI|utz>B>dnV$z|tmG*qzxmXrCHiSxnSJQch=L9wSFXJQ=uJ zPYIUkf(BwDaK{UxC>Sru6oM(Ir8n>D)l<%X?deXIcmJCB8?BrWaU$rM3Q;;{`U<^J zlmQPh`;+!G^OjXilc{@RdL#-jqW10S!z|3)tQ&@i2Ke zm`8Y|>CK;3c9B`8>oCBD$%j2>G-9>XivL#@z|%XM*@HA*&v~izL)tvKI#hT0SbgRL zRC|%mf3>YaP)yffXVIpY>b_J^(R>04Fmilo$6EaZR}G4bLYZPT8)he2y6L>3{4*?0ikTb=be2 zdE(})7LmGQy5dIkkb>U`?-yHOSBOAq$EcKxRyw#2y^&{e@)-s;I^~qZkHANSDJ=?#TH}JEoLdp+odInVpDE&T*KE$3M-_0PgfEG4sXJm;gNrSE zwqAYe>dIHbF0;8HN^BUlbk81}l9fdF?Xn8uM5#2XgB3|{+WSwrixh&E%~kS12KjP$ zeS0iI5+>a!wV=|BZUvoAv^F*9o5VIdVL|C#RNtKW#1-xN{J2ts>p0T&bIninDgE?s z7!W4*B1XX;|B{hqq7tQV-uHaG@_if!VBPx)Nh{Y1OPUnA{vFsO5Bf&{4pVH7qd2ZV z4X>j1ycv&=aItb&%@DHR5FXuEvYYp1LxHa*G*SE>B}2P^c}QA`35G1@DIR~)L$X-q z97+uWN`qbMNOt7dB_NF>1XJX>roAoby_?U{Hna5o2tt!RBf)a26_9u%`!(@`&Mc)H zS4ukWHiB7BXcSIQEDcu0ne>u=CFh>;tM9{#C2sA4$2(8wWh5YCVqIqaLxNdX2Af-v zsU_;1@xyMRG%rwOU)uYos&JJzCl6oHH1%-E<6lt%$5|#bD?(_o4~H|B08-m>?*-K- zF^i4e`uUyY%n&#cZ~b;L4+#$_MA9ikr;)T35`r(2m$`_!{gL!cL39meLD<=2ZVhva zq&%f>`YO;DQw1ZdPXAVtE!iQ^bm#j_XD*PEWA-Ai#Z86ObLlEYCtn9uTh!W@7|^^; z@PwqrT#h0=SSXcMe3y{!QH?)*V+-$;YxP)QdW&v#y*lzWa^Nl+!6Wp-QoPre0)jVF)kkfrA0B?r2C{l!N(zGV6c>sz#3-__(Mn*SdW=? zmGm>O1Uiq}h<*}_;>?>t#Qf5+rB%W9p@{I`suZ_??8GkiX`3G^KntFb*ui5#z*r5IbP`r zd+tlV?%%7pks>o7GOSbiF|Vg?hLHFKH<=-HI|Z;+xD@`Ol0Zm5(ODK2qmVA{PPF#KoxEy#5ht=uGg75Z^YqTnI0=jgN$9K=4H9W+D2R&^#Hh-7lpEkliB&cl3 zOZ@aXjm#S2Rmng1rF-O+sE7s+WtOvz*cz~LDxda`>|T>3+@{T@w!pW|p)Ks4es`Ro zMt_;^!dj* z;U@%HoDUfX-=@tkF!u2v{s=}5o_}eL8&RkuQBR1ivwT+5=>%-cfHVz^iN{AEDkzQC zTxgSDRe9c3J8EDr)BO%Q>i)6gcn!*Df{h>ZId=u=NP~EZgrZ8UzOkT!H+?}Pla=@r z0>-K|yV>H~w@kP%KW^Yw6hvWCHc(~Y_?H@&VU+sh68WV2_*5Zwc@gW?TX%#LVsbcs zXdf1Zos8x~t9-z{9!lREKLbfhPTrTta}gXz*|WrBRCywc(n66LG;%_Sm&U*w+ikbq zjv(fS0g>k~I${rlHnBDLEF+z>WZLLQ9+a#Nu1S}qFJ?f-7Gz9r<#>CYpq)jR>%2Ew zJN*8OWi5|^<|O5wSxD z=3yoNoLp0xb>w5!68_aOTW?m6j;L3I$8BiLmh2!ghB31|1OYe8qynno)}ubGy%DgC zb{Qoq3^%wIyZ~gvY_CEo`_-(6d&l<=0!v?EO?ru5tUp+yR9zRPxo@=_;6o@hqxyoK zUIrXnQMevG3Af&1zHSC;*~(|O45xWTrCQZwN&VZau!dE|llpsVo8Btlhrj8Cf6M&3 zTf{20;`i3Q%GpaSC)q$@DxSQm2rlaPGowdWJ&4huPB7%G$&;iirdc)H2}`djTltmR zghpgj0t-^Pb}>?lO$zwfS%jh#>yq@UvP*~FWzOdGO&RWurcQ(o8;I?3{}jXPd=<18 zY6Zw*BAU~$Dr>FnK0Z8Ww^pPQWO3e5iCExhYQcl7ye<{hR@qm)V$aiY4ki`s@g7X^pN$fZ7 zRK`FkaWuL~y-O6Lv^J18do}XVF_ZRO*E~brj}FFpO3To<9{w_CDZJ$B0bNM(pCbN4 zVn2O7TQsiRCfxKw-Df9ZCkgV*nN|P|0C~@^^vnH+3g5^Hk+cYQR{Li_WgLXE${y<~ zB!=t2cdA}~sL`>;~AyrZj%k54L$xN=Rjz!A;KHqGrJ&`o)5r16#dQk7O(D&f>Xflwh* zAR~QO7ZY!)fMZvT-2r><*D9^37)Mg?1$F%W8!R`4`z}Si971vY4jj*Fj4Ut@cL2w( z?3XWMP)|9$fXI3?DdoD;-Ud@d&yp{}rqq!M;ViMYCj^%_S)HpLC|svH;@BMR zQW=&?0~aXCl}RCFKSQX607*99(s@DzI(JipI<5r{FZEtu1mo-3VJ{}Qw0Pn}mY8fL zX~!<~=9}QjC+efDn0VCdA)Y_Yr@G?UZsf*{fL_V9TUm7WY#@yKud1;iR{JV9$Uq2d zgMP_g@rMpfReLo5m*O(gqp14|qK>p}iKzQ@@pla-Ct>as7t-xGjlc`85;3w5`rHMrP`w*Ac`#D{FTsvU_aoTv&fxB?G|?u;3sRs zKbz&WMS6ZR%ju3}k02}6vQQl9C$JMixMX>u;43XV>Qi30(Mx({IO^Y>#NHu)YduDO@QVX`g?bl)_QT&NLVVZg(cY8OSL{5f{Ug z;YaDi?ex!`Q4}?sRyO;TS`e&aJeKfVEo5w4Exrk@pk(0BpaMUC^E|;u5P~4GM2R8& z0yfRh5N0HL$lW-$8!@OgLHtXpKV8xSua(lYybZb^z61evG&x*18O>NUc9wtN6D>Y) z4Ex4gfi%0v(D*0CS^e_wv)7^YiCnZEJ)tQVWx(*YKE>bf&yN40;abg0+uOdT z-S)P9L17Cps?BE#PQ4)MVsKA zt?mN|AgWqBL*)7TM_ZBH+>x?$z8yRW_s_F;?furx2K3r^yZA`-bd+nDE@o$@xjjwb zB{aTWj<8zeGPV$uAiKM_6(Pm}{+)Y}|pLz@jIFB9Vg+i?6BZ5;doMQ7u_se-kQ(cXaWo2V8$Xz>oQ>KQg ztvjR*biXmI=CvHmLG^0hYXVQP?&GuVzZz&I86_R9*qDp`P4iu-0eqdy;sA`*ru}p2 z?U1oDuzfFc(XY8hYpZbwkNEPCN*Ph<>^1~!P+n+A`x--d+=re7>z z_w!N*_sHL_oSa?}v&5%r@@uAvxVenUH(~P~eg8>sNzKx5(jiIH{%i^DM@m24vpTyq zW9K51`YD7ixuW9vUuYxKy~4@uRgky@5Ex-PQW+h|&j%UNMo<2I!IVsu=@*kJ=$|_1 z80fq}c>Aorw>b4P(G8awouK<3Q0GoNoRw$b%hb8I zU8&c=fAACgAB4SCKpb7LEsO>C1a}zR-7OF#XmFR{?(QB4?l8DZkl-$X;O+y#-Q6Ml zo$uUp@8kc}5A!m6ckQZGYpvS-Mj=KjBXZyl-?7|H6Xz;NG}aPIFieA};j>cZZhrS0 zid_`$1EU0lm%`p(rh~YR5~?M%vxSl>;U`7jFn11x_rSJuEFOEo;B@y}m;{E14u}kd z7am=!mPkOVTJGd2Rg%eDpxj=cgFZ;n5TM{ZZbktT0AeFZDsYk{S6d`PZ^4y?n1lpj zHei&Ev7UG&RyXfOMt9jY(KN@#9+H0LRSLdCXg*V`ygM0m(Jxb^CoTy$(HLm4O569& zJ1@&(UBx8F=ochj2+6asQa-I}rlurC;!IR%%u&|+Bo+qFc=ilVztONBr2Y_j;R&!gk;#? z&NulA_xBm*Wm?#c&E&l`2rSmHZ^}Gj4G@(&P5JpfO9Tt!-vpVt3}UCTmdH#i?F#Dx zKf&tFTeSWpaL2s_@n5n`o{9)eZvOE|IjJ^>xS0fvOoN~Kt2(AyK_H|I#4C<{(TI3$&kCgwN7ma?#kgdG2i z<<2)=vvFzaW3xXE-4WW^0q<5DSbNGIuoQVu{$Vz?ppL%Ih+j4~XX^cBk?ZZG z>Go-O)iT!}u`<&xS4Wj+zeUn1jX3htdQL$VXWka+EyBgq1%|qv2n4V`(NS$7?poV6n5>^fMK!@{dt=-)V;x0p5n+ zW}1Du@Xq&5q{#Os%c=e4Sk@fdVR%R4%pKux`DI2nKWR3rW&&h*nw=|e@P;w^7G>rBfll^ibMdC>z)Sa~*r*}$-7eTyu z-1jJTniXLe+$edCwMv>7^0iG=_R*%?Y#)fzVBviHJPgzLNSKP3M`V+;|%Ms7_hXFg2m`yGWWxzuEPGL?yuZeUWE34mt9J*{mAWf zxGeR9?Qa-@t+cx4816pT*#?}vU`wg2Z4^}xdJfg1Z)^EnOU-JHe>Zcj;*FA^`(}%N z4Vjq$5)`bU7q&$;@9x1L(;uRpzB8Vl1K=Nu?=p-Ch4@T~vfpN0J)f^q z7ay15s*qb^oBxPpi_PThWrNB`0*TD=XPnw8lgj;{b323wqJORrd2W$DM3>|BVmWo` z1T{1IT_iK&z)@0Q?s!ypeHlU-#-Q3&N^S@x9j1}#l5z`>?sDT6d8Q2uGS}rTFxH8J zS6D7R+eqi>^7%nrECl0igzAxdENZpxO)ms_#le*j>Bq)WR9EHy&ieiE1qo4BT+n}9 z%TO5Ieri-2x8m_ZVPyChVzwb;kP}F0hC6Sg^_KQ5zjkEJkW%HYxxsGRyG^H zQ^rUe69T?7j>@4jymg$N;Jk*-IA3W;LjwR@+J%S;(Jge0Il2!LB38T>%w?@L_hG4*ITEE6gAZI6Fc1R*}SMYMjcSQq(7dgf!ZqkvK^ zusd`v&d-qucIVl7WILDUJ&o%Te*Or+1(zuWACYBf_di+FiRN8o_L(2}=@8?)69!vz zh&z<7MHl<~W92gwY+(y)`aFu)_eI|gYSXZ^e0p!b{#|MxKR&#{#^=IOr@~qu&+up& zB}f)M=^eC4e@l%c=zU8@mbF2h2}TFk*({Y#mVx=_gCXD&lk$Y)IwgAI>WQ%VkV)ea zok5RSJz5lG6(w=*eJWG`P{Pd<2p3ekYtj?X$=f@QU4YVedFJYfl znclTy*e@Gm@}3UAvdNt4WAzl;&j)r>Ovkh~B;2=}%Lfk?5f0Pj%;xIeI;_;`3%>@M zlIg2{Ir>YT498o@s=k94uG1;d)4ZX%dde9BV$7jf+ijg6_d?(A0^1sHQ>Ku~>3}kF z=3;hTtJ<69yy>_Jk6|m1j&Bb*rE41XYK@xYU%q%H*wn-k>g3sH3TUr7_DJ4(dD<1# zQ7uM>$LjgaLkQ0QVB}P<&~ORgw16G4N&2@1IN*n~y<04|1Mt|FN#ncXx#Dd;re3Ey zcaf6|Zz&=grYXzN?jjOgKB+1AL_mVfL0G4|4i>yG@6)KZMO2WxHL;m45JSlha?fD) zmT`uW$JM-z4dVGf6&ZQ#QQE}gO6DF2 z=(uxWN8xC3(dW^vGN-JXjrGe6TuxQQ)9CcoEjnD*PX#{CO2n;Rfh6>Vs{4|SoAMX! zw#Iq5gcr8E_Tp}ZV5!p%7H89**jc5U0M@DF9AOvji)bc?cy^R_DYc_Y;c7!6MIJV1 zh&|p7X$lo=Xl7LMIXBlYg=aP`N_w>j#}ec%VYgZP>OR|$(!==yV28Rj!*1DCf9%Jp zPgDu~+>244I3+&BMCeM5YYq}{+w4^l9KQecK9QD&UiPk`wDN8)HO(MybNLNDgfNHV zniE_j@p5ZyWrZDd2U*{yh~es`!D5|_w~6I&S-s7$!MPFIX7B77zI3&9(e&`~=?K=He)&3oh_x$aq$>9-ACeGbG^ry!y_Jq58T= zR)j(OOHSe96+RRq02PBNd+s^QoUtD01}?3r!U3?OppvFQT#DW^=t9L5W9e}$!+FlR z?r`Q}{+(<#?sTcDI@ozfLf4~w)$Ww-+$x!>eHrbW93~t#auAf#tiR%)quWXHNgz-& zTZ)K>Y1$SNsY$ra>RAW(pT4tLVgyO7tY8In$>|VE2I&(>rNV21qtk{~`id!TW`@wO z$2-xMW6y}53Dpn;)ESsL*v8=oE`U04ZNy$=E19ohErdTi4m$oCJ4sHh3nUy&a^t-p z?^nWJ#d%Nn;|L;m$tC7z^nK>8+PoXlcrMG~#uZl@Q2sa{%0e%sp9a3KKxndGT!@$$ zNc@>pYZ^_qIyYX1!MA(zP_2LZJ-~bnV3)Z^&t@Hs9q~%HPn;0SSni^@F@n1&5-wakc!y{DtpEn@lwsUS;d=+s+_epl<~Jm0*{7TU2nPlY zjfHqse=Wpvc1YfREu)xoP((7!gSp~9@Nx7iEhup(ov;44=3n7EO1I?3vg^Ke(69z3 zzB1g>PXgy`wBmu71H|0kwqu-&g_inGh4(hB#Zm%^n=&7S2eV;ae2ei{DX~gbsHFRD zT{(R$kb(orWc}sTd`c!Cs}G-0HT!B1&8tM)kZZ6-aoP#o$uK4nSwZ zj?MmdOUys#vA7*l{NKk~lcCBbA&MJ8VNQ{d{r%}OiqN|Dj*)^fZ|VW{16ZUt(}saN zjDfpudpnMVDM0ln%e8#;*KFiThe8g{TN#bQrFPQ5cG}W^TsHJ7Ze9#I``+J8<>(^n zgvCbi-Re>2s{+1wotxmV%(_l?IoUGe;n#TpSh@#<2wW~9Gv5eubIY156S zkRzgkV0O$>?Z`IW6@AIqRP`QJs&sWY2B?LHX%0vT?3zv8P4&MScRJ00I!xKQeJD&9LsmrN!_}-K9C6%j^@o-y#Tk>}@=XV9j+yfiXIcOMB z?pz{lSe=qKv9q0?*}~+*q=dc4QLzDZM!t*1cKj9%cAqdG|21?Y=csrM5?NjZNU>RL zETRBfNXfnogQ7@%rQ*s0%b?|h?+-{0s^BVUSuZ%;_G6RkU;59{C~#zqCzh(so^$4m zk6F?>DpI9z!y*R|&3m9Z{EKBv9~!3>;`zMt<9CbTy6lMH-1pCYcrgTH2cPVRXEfnZ zAF`Uv5bi#|`d7^jO@nUSyjP3r-|svwMRFKY33S3ldrp$PQvxE3|EM2*Cf%(F7KS{HT8OO!7vV zZI3SGL|P*r;fUwB9=x58&W*vU(9NIoT-~{>Ks;vShgpj`>Pj*fdV}2)ium<|HiZf@kdD6ApumOjxvn$Je!1CJ6@}M!Zld~uf^{U} z@Kq13e?BQ9A+9wCM#11$ zS|2EfABR1bIDsVd1F0Hr=QOTHwYu%jN}V)=_V!bcSJTHt%ptxaqK55a-Bw4E+%}du z1|hOGl*Fxx#T2lfoV#klVu|ENL+%L6n6b7>LlY`#iYI3!o%DR=Rs!C$8cyVaQ47a} z;Q2|>AU7&4%FSq^T}zGRAgT?6>~h?AprggWM7*j&5ULYvU-z(LAS{=O+@;(r!Ku=^ zwN`jMd1mMFjEp?~vs_VDOyPqS+lK=Z3{x>h$c>s2xO?j~MSv@69IhgcKenqn^;SF3 zUTWNxna1qLF%kv|GIBQdoPF;OR4G)MW!F&vH#HEl5symp6|CalK+*2By`AT00ZpAd z94cfF!H1DBbJ8f1V6|a#-hTObPWSo#IL>RLi2cBwv4z`yc|COO?N{Fp5*vm$3llyG zq3dk5e)q9n5b!G_k$B!9BHW{9f{)a*{}u0j!%nOMed4&T@e1>zPS+o=B_x;G$bsFF zqdG_cy}{2tvSGiKuVsFD!ZG5?lHMUuk(|#=-Wn@%#RJ*5j$wMG)_H+Nt9d4$Q{l3l zX&jS>=(uAeD7O!_-t%MFQ1ZikavqgyqJe42kGN}_hHu$wgLEj*9*2DWrLd~ub-Jb zd>ls=(@ER_(JR_&(G?!xA|%BFeZP00OlBVEu{;ub=_Y?oNw|whz?o7lPCAucz0Vg(UaV9%YZk|>49YaL=I44(I@5ciVzt^MiEW>0qJMl9< z;uLJPy`}9y&)5i@0A^P3lF|oO!&`Z4eW&|MSy@2W9c!o}bESI*ox~pa5A)T{#mt-0 zipWay7+aNNVHx>SEPgD&pOj)>+YM9GB>P`kB}?})nyPgIMs@%hQ+;->ASbZu7;oy* z(F*~G@OFrMHyDR4JMQv_d;gTL8|eZ+DQAONy8`WD2^*3-(|yS9041J~UmTBtnc#Nz zvwc`ZNd#&8u55UVr(OhopRGjANc7)OYtY8xSDe~(*ILw6AY_!ZyT{rl+18`3?&bT9NVVUye`r1L zm)EE6n9V`4-~k=!8O)$k@R(uF-Vb}(?~6pqEls6)4`lLcw1&Pb$H0M|%*_80OC%^b zcxE=AQ0f-Ib1oTj;?d4rd$H}!57g!efi)5hpi^nb zn{eT~Up0Lu+-J%~qq>!bi0>L9yQ#{7z1=;sbz;Bdon9=bNALkag#pEhgvw4YHzeUW zi48__Wl@i1Cfq3>)3?q@whc+0RwDyAr}1G8O+bJ6^UK&7W3WZP?@uK5p!MRz-pud`)a?dRU_EXd4sJF`2I|9;~+u&%oI191z zgf+rJqn&S`W6Z6-(Jc9u<(lHa{Udgo5o>kHAq)NZr+@bI@V(elYdX7{q0#HzD+5j( zBpJEWHHOf62s91zKp8A zV_}a}j%(psr*hiXR$bcHhVS5~swufhni(p%I4Md$hU1eanRqJv*K@=FDPMn3L+N82 zCfhN#)p2Guan9g3G&}wEPOr*k)FfZmEinu9-g(kC+iMu7WP$ax^1mi0bZ)#^AE)^H zmE&Ct=VU4=W*%=2+`rWVlE}?_7~AHDLa_T$@))TB6s=a$e52|><=3$uCwjzfdXr`5 z`s7giW!e7RzP7W<{F_&~2e>7~VuwV>&MqRqI9JhE*X_h2tt*Br#>Jz=xP-J~S((EG z+qJ%e{Yq==kK~RRKy`jYh6(N=r!+EO zC^P(pb@rGZB#7Hal>(?s1WV5zZB~n?pd&)!+J({G%=mLwOUA_>`3>Lnm+h0)r4xDK z3X+3wZ8A^f1|?bEfdJ;OaP;F_=&Qj^NJDc>fj@PnSE`FV8K60?ruVu_`}zD1uV}K8 zh4Ink5=eUbGl_K8SibK2glB1zx1=2vgw6`ZQm7MU7(|)uW;&%6d2>V5q-xeh&IJUU z<_bt`0mO&-?uTjNIAzzhXHRrLluU(I^IRfG{>!SEaEZ{+O`5hC{MS! zl|n2<5q@=|c+)i~W1C09Jt7-Lnp*gP+X9Z=Zz<(7b3&*=v>!3d@Nbl(or@^0hXcdr zemWoqDve7Z?RzC(VO+5T%~kZ?ki|#@zqJ!`D600y zC;vvj_RWx3_C-+JH|Pq6-c3tXK~x4d`=GlQKX6keBd={J{41&c_nu@+v6URGqA_1> zzOa$$x4-a>-?d3PdyaD z1q^Jz6o}pwvaTf9`g*c#pwC@|rx1ytX20g=B6E#SN-o7DRQ~ zTyA9FqU!Q3$a=rvRJWHTh3&Ep-g6}rivG?#)@wl5^H6}9$VEC)E=y4|%ZCLJDXvtt z5)r#z%n+&;YQ9RL=l>#%L;`?0rxG2bN4mD6fIwJTV zt3>UrhWLrpY^0Wi8&4h7(k82!s`aitwREl|UBC2IckYz?i7q4;WyA;$bEmm7tu|_U z;J?{h&t%@HsYrHXNmW12&jc1znbg(YSWXaVdGH}o35|c-w!|22WI{U}B%=L<0vwGz za!m3Sppi8F9T7rDhAaoFIbE$;neZ@<3}uJ~Eu7eo#ii`32*%zx9R#xCq`@yKXnY z`_6j$R-Pf~6Q${0Fj$yi(vFX)??Xew+-&#5a7$z+0gY4_^>Gr(WbPmj5b_-oZG!lj$rxKL#mVh3H^#^!VVDyfrQ&QF9r`$0 zZHz2@2I48ak&yo7Bni%x5o(b9w(X|+v8ejJ10=txS>R7Yp`AGqGes*FxdUauRU7gZ z%N1+QjasY{W8ydwzX_xQZyH-@!%0;-Q2lkyV6Js%OoBQ}Js9_#h=7g!_!;C_yGC||&o5V|#`ScT8)X#aVl5I(92LzJ+Uoz^t3iX9M z{7T!f&A+_Wbt+C`?irgW{8pK0=;KQUj*w+n^5HXV#fiNFoUhz<#SyvM=Ci$(><3llA%CFuKlwfS@-&Tq&br_|!7+nS$v=4@;0 zr)FlUFT(dUZZ)+~wUN*sy`gHM5XS&AP+-%Cix_aFO=w=j7^XaIHU>W9H&L z^FP9c91(6=JJBvy&&AY-@7u_=fY%39=iC@<{CL}|2@p?a?MD@>V)1E^z}2u8Hlfr8!2g|8R) z;qjI(>BY8hxjd8L6bs5kV;g3j50+q#sk@(_fo4E@j!yIb%*5(e&^YfUi1&gNF?TPw%U3?2l_9;V#Y%0N+;SiwZ~|g zcwlBZz2cWBIAjPAlqj$(2f0pQ)^&0-)34~etKDGA{(1$D#2cdI^M0n5MVOJw#m|b9rd>4w5+A;{chk|;Ycao z-Z3iP$cbIaZ;crmNe*dI{+ve!^cg|K`*M6FQIu4f67=DPQvr()+9e{=0cr*#Cpa*V z+RPr?QsJzvW>#7qI<9K-rxkUYpGwtA8fYL3l)E(x@ zR^%wwwa5PcEvvCjs;0?1^N<#Tg~8MDwzOVEy3(Z^_s#JTp_G!|M27D>4;oh~TfMQt zQSz_NqYieexqYL=sL!#_txRwq>&`MQ|M4*Z9<%`J9yNjahCKPo%&Ziusx-#R9O2G#(h!!ljDftugf!*yLeWtC{_s9#NQAM zTxmX}vs{(Qkd*LrDf;gwZ-A%|RFBL52vL`*;^nLbktr56VZ)Ge(T?U(^%^EEJqMrx+a$)nCrt}kK?jAWY&6u!}-SSJc zG0+p2f*0a;xAdvh4Ah(b%aBd9Q%GS^XisRg#uO{>Qe_0InO`!Rk<=;3FM8J1sv~08*zD*|GW`$G% z--T))p%ikwv0}uX3Y!?}GS1_Y(UkK@jfx7hBfcYn|C@BzTk4_=r(tQ(avr7)TC!F05u>)T~|l7p5Rqn5F^Zi@YqWM93az&Kjtd>4NNn<#9HSx02}c z*^mjNC=RwH<1cgpHsC_PBL7pUs;BhsZ;@BB&DSTXlG{w%eLK?`T_bgtYvNzF(6k8Rwuz_KFU-} z8-2byneLm1;Is3S(-N6I+*%^z5Qgc|UWQPp;{n!#BFR9w<&V#!yI!b<_?eaH!?}pO zr)bM#M{ggEvpMYTm&ZAJ@BsdJAP{^v9cC?MADfd>v?aLWzLz#7iWvAtym!AMdae@< zaU=aR?Y`U0!e{wd`P}x^EDmu&fv?f2WBmp*TO4;{ zOzen!8D?f)-TozIJ4+IO=w;$iZnwmahcWBKqDB1YPy8!mf50fU-Be+8w(Ux38a;-$ zpSKD$PqaT*e8Y>lQVbanPbIY@r9{1N%-_H(d7g!fjDb|Mdg#5$A5|s#3)jZ`Peb{S zZN0m7m=~_Xzr?-%rwSs%LWanv3oJ`%;aj<)Wd0t{sf!& zVfM^LjzghNVKNcJVI@f9E2}xPbXjYy=|=!2#&2<8H@%*afo@2kuSku&YJFrEDrc_; z{%qyY{PZ=T=EJN_WjY+XsH~y1uAypk!NXYQwHcA?ULi<`7krpgO6-wJEyPf>B#3fb z6x3On3$R)}_f;f`ymRxCum!JuJ!g>ywJV^x?5n z)s*1B^1^ggxmT0}eMp_6g|i-&W2HF1q%&oG#P&*(_Fk3-@AI4nIl{1=wpPU1jso7{ z>Xk0DH)s9Ftitx+!{Tw6Q)tvfY((0LHY~RfRJQtt2JK&7jh_{{T28$ig2M#5@ny1P zM_SLkecgFdL!;-HkIWp6dg}Pz4U6&r-@T860^2Z1*FEl`?(Ad%K`&SvZOW4R=F}4E z2e*rAHSLEO%a5$MD|2-Ga+1Uh2q@Z}t<7I_>84X<$ng(w$eHq`5##rlQvGT)X|6z5 z)HSY=9$zlrE5jguV6AyF0fZl`R69e=a3oD`y0`b-E^*oF4Ofk)#)lzf2~&jR<_xSD zJ8cAV;;HA=fTtbkE~F%mm91Sj@W~hPPc82&r)%+@{| z<%0ajeUL|Dl1EB$R-b-NA|Qh?gwM_3knB|FlV_B!UR}>a(08Vg+_LrWkYteO=xFoE z^~PI}`TTR?nV5;{k;z=(kUJ6i=7bDJMMv%LCo!lObEfc?9H=3P{a5m(xJ264Vp%;r zm^vC@4868qO^UG4SytXN9C8dql+HGkEFy zkbjF=mH(f$~VkS&l#Zhbu;tD&;k2@?*iz2Up9wWV{B{bCbWn1NZ-r#zcE%G z0dykI7%Z@$auzavJwK$V)5+c<#?^*^nVEt}p6tc+zahGJ$e@*HcGV}^7AwPTz)+Q; zd?%=v`Rh!=8#53Go2v%88z~ZE#ilIcZ zpi|>%Wd-ABT)w&?drWAUk`%(}D5uq1b2{!dw09|$z^+q&P_wihjy522glsr{uE8nl zz@kEr*xB!W_XKj|OeVAlGEyy1aMv(fRwCsDxMoZ^6*A~q+S<4cZ@V`Aw~-|*<=Cu( zE_U~5Liq2L&*GUGXCN-nidq>hWDOaxgmjt-KIJGGx0=SeNB247^HnnBqd<>a^w_k0 zZ95G@hZs_^9^o<~^&* z+cM6b7{_H~YfUJhV{*&7jnwk7RNZYK@W&29tYi*xqK~UfP4~+fH*st)8|aTd{21JR zM^IyQHD!GO_&5L6Y^#vKL#ibUuCMss#J=P&`~A~LiZ@u-(y=`<%}QQ?SyqM@!Lys{8|i|4JVNLlh~fkECS(3LPvS{F>sv~Y z@P4LkCCuoRusT5Gur1zjMHN-qV1EThApXwcbGN`yE#8}aFs7AyZ@Hdah!U>%o-p?F zb-Jl9pA>bMuzz%$&wyzo6lrG}35)$yB;c$5x7%?4$PV&8Gtj50$yWCfPd7$$L5n^G z!L%TJIZ$|Gozhn_2{{nx&m8R2;UHLGwIL~R_9Idxj~N1WPxN437O$tu6&_%PP~P|w zw;wDtM5SG@iW$!hu`g@s*Z!Uyuw7n-$7NyO9iwTS`F%)!l<*31EwTRcE5^|%@Wrhj zZni?G*7RT#D(QDAgC29`9z-ysO=ETO_B(_)yX@naPW{h%KUzzOZU|Z-I^_w4IF=aP zNFKeq1;E+;_xEQ$Kx13pVd#kr`LWOneOQn&SZyX7NpL&LC7+Qm-8XWN*pqHqBKvvn z)Mh106e0jRslv2HJA(Q72O1@WFEK{0$-39jv5IrG8qef}D7>NBK#X)JmBncdIJ|cQ zG)!hA*EQz`ng-rYW{{fA7&n(1e{+RT!x-~d^P(7|Sp`_OEqq^vL#gI06V_JNi-vD2 zpYQVXzxw87ICo`ai5QE-@|YS&ol8cGWFZ8e?|&0`Jirq zorC?YchvK`?J7~VV+MgUCq}LVW^}8~#(Q;(j;pcbe5pl2AGSu9b$;$?0c z*ujDL9K=gZ5sD6fC5R#v^PiQ^!1)=0usK$4-XVgWfqoiQBiJEAHQw_Wz?7 zbeBFxcR-{VMOE7BQTP3egf2hN`l0;2>>w^OUUxy4Dmf2z-ub;fSn;Kn?&Izdx(uHz z210;m7Q*A|sofC;fC8{biOAM}>gqOa)k&wuRF^MKZi|@&9o^Bz*K80v+>OC?6?(;$ z27)9%_B?icN=)KHfC0QIFQl`y-R_cZr&}c9+O48BtfEfVnmbVE5O*J1Ik&9Lm`{eV zM5u{2B7C65_Lb%NS6z>?(P=i}R-5J~iKF7Vp)b%puCw8&(>4e8A-pw-oA12;%C1O! zdZ%4opJVXs*NNWzGT{7 zqvFB?l&%T9{+rlvwIG;ymGRoj9=LYCO5VvBMUyr+D|3h;D?WpPhyeb;h@M55XL5K6 z|D{s)5|sAv&-a)S>9&5X$q z48c+JS$Bt{Z!g}3Bk6mMP`kJ*T@oex9Ny#5NV!Fx@L~)}NmjxT%X#GVZIo!e#lF5R z75U#z0`{;a+hy_}o$UyjRB?%fs`6ka+26a=&1YQ9X+bN+n6TCL9JWjpQ_zIizWbj; z3>PFqF7|bl;LC;WJi56r)KktVZH#(p5BvE0i|JZ;5kf*E39uionQR@z<8rVoI>>%m z^rcAK&&iTB`9iuvyLJ-IX@2Y@)>89tZq(W>ss~wv!tQNZy&Jm4DbxG+METdTp;Mhpi$;{znM_ zv$xW4Lm`k!M*>QMn|#Ng{!__FN8`VwPWe^Zt)x3#tf)(z zQGyt%Dkr~hE&iN0nG*Wao(_g>w3}QxnaiJ`TxrfXNjkfb?;^=TG*Wh@4UW7i&Y4ZiCga5pJ8>l0)nBq@^@*mwSY@!`I}G zB`c#nKlBKHS#f&*uCTKH(!JE8{(4cUG|%XBe>iPXstw8sM?)o}Vc?o6nwbU&4Q6HQ z7bGb(nOa~NXBzJ6N`zNRDF1&eoOm7rBL1&Fvnag~1sh5aG2R1&_k%~f(Rww&Rwr}5 zei-Jt=Qok#J#{UU+8tWATPZ(U5p*S!^_uu|m%s8`7Jh(0Auw(!Eq|ms$Tz%Wfcp!j zPjsNyo_E$44p#~`6R&w;z9$Jy`K2}Cd7S$Fh~IaHk~JcgXsspn_zc|k5!dN&%N_c> zbffNzRxVPumiS|uB1B}^7T5=`mS!SJf|9SEBa*i+m?tI?yAohQBZW|ot6Hr8+cSiO z1s>Kft#f~Dwk&g>6;bhtbp?7*vi*~K&#|rqzYU(77=Q!^7=NNo$0~#S7~=S^5YS2| zGAvsn$qvO1qroPIxX5-2%cZbn&m-zBTje)FEX(2bM-tztYWD%Rvtwwz=ER8jV`n*iOTBxn&x3XD}^GZrF+x2zx#rSAi zsLW?q34Lt0@Y|WM#pJ(|04wAFDbC)JP|t$V%SS=vEGEPlD@$qeh%Hv4u!_?hg)LV5 z93z?L>D6d$R)7XnDa*gwP!D^?4msOTu)V^F?()*XrU*c~NH>6CfaqaiCY~b8O;Ta} zn>%N6s}$lRHWV-_dHq@hF^~FwCw7SG2RZY0!x_2_=nb9TThNHNk$Z2R zXrl@m=DwG$&Nx5bmg+-$`ry{Pl;Euvh=w~$KNv3mf4CNaGx`R2^4yJjzKs$d-d<)7 z16xY%Ive|$i1##{?PtI?D*`B6(S9w+WB3!9!-6P-EJqF~>$TFwh+*sYep`me3*ixg zA$wl~;R^R3CTez#JQjwrAMx&FWgd4?)c^Z=HR9Sl&B9$P0O5m){sumV%R@>dZLaT0 zG@e13hisgjTKEVes+=^PLid9u)WiHZ^uFfCX%zg!I%GYM=w0E-fi3x%3^0>*oj1pa zVfc(1Tp)iZcch>kn|rRBz1`Mlp-! zagqh&V!JYTL6vNoow%XTlAhT@J~xQ!sK3yU8MKqxXQ=78&E&dqh#vugP%7b`Y_LjLw$9@oz**ia$5Au7?|{sbDiz7iL0J4Z~-&q2J!PlMwHtC?EUjO(_+xP?924bSx)M&Kc&r2QE zHV%tBnaCuH?urv*WU^ePo#^9}A&Wl}jywTFjSO$;!cYxD-L#`#;t3#9|I-|R$?#w( zB%Wslpy%%vM4_4;L}ye6v}$Ymp*tF;(w#D%s^oP4BwYsSy+T&A?$0P7b)UO(&V;m* zH>qD1a0vBFx#*JJ1Um$@Z7}`Tx<|noqne$YK>l|U?}>pLDBf^oEwY3hm5PbsN%d_0 zT8>I2qAa?nj{KvAz=N%a{)g1Xqm3J%5WOf^cLXk6pZ zLWAGZ2BTHH2r`zHp?oAHN?Inn8UTzj^uWq%r@Ht97=%K|rWM7Lnx*62g;_0Ii$D_H z|3Y4k@fPy3^|6L>NNEUwpwuEFi0S*3BE_=wtC)1vdVahOUIFvWGfsgKF9SzWGc^JQ z-kmlpaTZGs-Pylk1k@t@LmoL?N}-NFT7HdVnW_m@*@gwcST>t@f6XbX48HSt2WVXF zW+@KdOwv5{>F#0rsQNHz9>%k6OU46m;F`jW!@2wqZU0}d`5yYt?S#lm;k&61a%zAc zyp2wC-@qN%=pUhj-oGMe$BX1z? zzez1+@X>z@$gr0ZVyWNzG2|8#OZMYhf0P(Mt92-qwj&wo4~EQ_rupl1{HPdV7Ea~E zwk=j9sEG0tS!ud&I9(PAy%Uy+I((mEGn^V|)Qu2^%3-2|%8nLDpNj@N%rkuJ=$oC3 z9wwz2htrGZO%(bYn0x=+^nW7k@EcT{QsYNNB&<@&l_XJNxm@YlAQn|t_q=vHy1$c0QCzTHZDxOfV?W}_*(DnGyOT$i8W z($!m6I_^JWErloX@o>j6`~~)Es9(I~OZhboS2Q z^$6+xGN)g_;EH=-d+2hoI%)m)oSKSD8IPEOLDnhz>%40JSrL2aF3Cjmfx;P5GvT!AsP%qVtd^C;cC>?%!h- zYYIe2!k!jAC9ueHAUvJdGpii?Qr4=iOEK(BwQ``v|1clkfTUb46fi-28`FL+>~eE( zIoT0YoGVT01)?o5LFX<_#qISzCp1mp$Q)M5HEnwCcHOvUb#PKh8}b(ydr&`rYq|El^5hA6kLZ5T%wM5P6!yIZ8YySrlu>6C_{q!C2AQ>D90 zrMp28kZuHFkpAxR9G~-k^8@zmy*jS7ZXf$K-_}D1?@_cVomKByx zgUpX6%(e-ZihTax5oo<*xn|m&h2HeWlGBO7CoE&c87#j9E|k0-U1(Wm9ZZIHfN7xp z-K1aq*32iyoXjHX>A!A_uiHnQPSeVDE)&+S6M58|9c@p9Q*^wIPkwl;VJpG2sHcg$ z@9OHGXv^-I#ahOHi}SA-noX7^9ul9Is8-oZ94=0N=JA=M@8fmAZP$!NUwo(a;kSD2 zGRan}qXP!UsW({mzOTGS2rOm_?1VaOyl1YDcCi;tApPkFcjB`}hQh1$ctzW5Bn4(( z<%RP`kJ)6C&)33FC;wpZJXPF8YRag(-)HO=MIK+gwVkrMar_#ACGugh{=uCmX^cck&2Pa}Tzk4gywY{iz zlv7uEH3oL7n9h*Vm+6OY-@4ats7y%8MX=BnFjt>?qYt?UZKK_6t6kK9`e@(V(ssJp z4Z1qUGOrH@n4YqeZw@<+?hcRnyBTX#ijum$22C14QRA{U)3F~v%iHFt8FiCl97~(k zYGf@_^2wz&nj5ir*UJ*Z)%oiKQ~Py0`(@hPs?DC$;G_OjKiXuz7F6pfhQ|FiqZ-;c zj=b$hvT1G{LJih5b;A{Q7}JB(Z#Fahtx0=P_psk$+#DxD`9lWQ6)_4^Z;Y&;_P-9S zXi--}GiuW4V0u|)z4x5HlH()UB~r)MQ;&KnW$)x=nkf&~BafLHJ2t}xh0#I}5vNuAV#i`?D!XM5jgs*;<4eVM zB_Sp6QCCI1CH^6fnuMc*1PqojZ+puo=nf-~^O6wo9X&!6>-5FyNG|AUYBzC43#$2p zd%faFp3MyRckzA2PY(o|X?;Z+EV#I)U-!j-M$+yu7O#1vAP3Kn9|?QU7Uy$4v0bC+ zdPMxS-W2RNh>ZFHt3X;yOyJemCYNm z57$l{(bbf`%oB{zrkFO>nvJ!8Ukh-4m8+~_69fGtruH4IEByLNJvPkCR1vE z4A8{4`!8Iu4UDn%_eICpGAo*S1nKg;k}FTiQ&o!^N}mv{V{iH~av{XF%>?SA~X zA+8Cs5Ab$H>l`RN%V3*ueuX2nIKwL|)Y87$IKzNZK)Bj1#m8aWQR&04)74=XT}5>1 z>b@AlQd0k$%0Q-U@9|g%2L~lRz4DAC{kAVP&mKw-s!H4Z?yOaQ{(k10aTpHq^Ymwe zy&ub+@U*mVN2!eRuK{?K$<^fxbw7F#kaVP&wXJ23{~om%(zm0KV>jVBpN z&{{8?osIc?U?2D(8FtCmwt5q9><^7)q^A!e-4K4*Th6$Z^ReT6^ZtX7*L453LuV(Z zvQ@8ny(jXwAQv0v-@O>!46c@96g7U2uY?<31|Fdo7Em#l}74xvPSNnv=R{hWN` zt6e3nTRoP0OVCx+j;+C%DX0etmmDh2*n{nWoocR;=1O#z8#e_mmplWRhAJbfeMRSJ zw_>-I{?u~0*8rVsJ+`<`^9*?OaPV8Wy_MSdwq{valh0%?#iug+maN$^q*m#FuB2@Z zf6?5oUf@|AKNGcu-AmJcn`McQ{581$-f^wKL8)1p568NJc7(Xqfnhz|+GbfdPGX41 zdw90+*Ze_7IoD|OvEu7qx14U4tdp&D@u}~p?4wMnH+cpnZ3gL)T&|-QwN4F=oV26% z5>O?t9W7Fjx<8tL6Or5!~XZe#{3s0e1<#w(!Vz}o# ze-3{QlQSLqZ2Hpj{Y)sSE1T!*g37WnjaLS>W7g$&7cFceH~n800s{~6aBT~mMsW4(?dd7iPDfvh?96=GT&oVXbgzE(RF8~w z3{yyi+Z!64Pl|N<-0{6J&1zswb4zUb-MV5BaDZ0JCdptGiFxcV5Re(iX)$t(jZoEC1*viP}<+KOSPLG}~J zOK~Xu{_v3Cs9$lJtN|67zUL0`aYkCQ_2%Ow9$QD3QP)&Q%TzjbVXE!nvDrpqpDS8}zl&ONSaleE?=h-iM6( z27AQ9G4}C=EpzCJm7QhgDIW{L^eEcKIXBO74$vi35jkyrLza01H*oq8g2AVGdwEh4 zAZRvOxYF3r5H~#9P_4PK5YMtNm`wN#0Zf{k=WwS9jeP2kx?c;*9)%-PQzTLw{+u%5r zj@xw3*|s1NZ=@xWyULVK?t47Eo@AEE=V|<4J*!4K{KU9>BR-j}j``hK($LLgWj;W* z^LF_|WJonSYK>#u0E~V-uJvS-NAhvZ>)DyF8N9Vn!3)C7i*IS9vQKVaXESN`dfUTC zlAA|57Sdfus=L41*==M((>RA``%@k2yxuuh3=t0@#z{~}k*CHa1>AP79X2F>y&2?N za4|W?0(Q@m_MF{e)V}`OM5bR zQDjYCUP~Es-zhh%M5&LfO*&sM^JRI@d%FOeIH-UZWQR!Dt0aXB2bLa{XtEY|3B!Nh zB&Q%^TQ{1mq198lP5Mw#21+HNtf8Sn(ctOGT1U|eVUF6WOQwa+v+vll3gd^rbf7uv$8Vz*w+?Wa zl$0@i)lgNp!)>34h= z5~Vn&9r~H|+{b*A?j{5DaBsoh%0*$z^VU%>uJxgt?w8%>E7AT#s4Dxq_O?)KDD+T_ z*g^YPiZ>Au}X!b|_Q(rAx+bv{o_&eTi!3`ITOH1eN;--oRxF>oS2Lf6#{%HBd<5+cy)hJMb|`3$ zy$UxN_V-~>^Jv*j#Dar=a`RwO5elTe*t^$Q0r;k%W=u-~qQ!%}@tph=t1?Vm2V&ti zE0OXrZ>?_QLce;aX6vs{v4G}vUGGgA*!@da%cD)_ zT+(+QoiGwW-TYHkPb7m8K;TT?9k(UD=36|M6SAXBn2A}r7fM9wjp1z(dnCJhQZ`s) zx*b=XOOk4?Kf!W{p}ap$LD+^^kX@A>ll+m?*paQrW4N5Tr*4X%PPPv>USd|)#S4^_ zkM23YRVUO++4dr562KjnAJhMVECFS-2FN%>e8hmJGt$OkSpB#WYMmSa|A_y(h(%S- z^@n&3&W*GKJsZi2*@E3qDT4zet^MlQ*uB#B>ViX~;==#9(bmo+zQw|atXoYg0-41e{BL2jagXY?R6SkM;(6Y*&o#vj2*CiDT#Yv^$Q z6ibpG{C5@FA7AX!P@Vi`0Pr|nNY#qfJxQ2prwS=KSfKjIsP0?8tmlhRYDC9@ zG6#CgGd0HbhsCFBcuoiB$qBgs5EVgN0(&q>`l%~Zwd4lxSc~XDHH*Nyre4Tlr0Qo{ zfaUWB>&6_n;#c9!eUn%!zXGQDlkq1>L?ZuSFA$u;Muw%Lj541`9XeIKh1E(nSqN2z zygkUac)7zD>{*wCJR!VL%c-j${IN(wlJ7GP#ecO9l!vffwUEv{GLh+8P_hrQXVn;{ zz4=;uCZ8$bA!mSJEK5&35vzfwMoWE?esI;|22vc*Ovv)v!RxUdt=+f zj}NxPOffdHBJZ04c_y1eW=PPM6b5Ws_{deCpo(m5U|L6toiYo{X;Q-fN_xN%m|*w7 zm;k=SdUp8!!Ag(*e9%&2fKVhe{;rPKb?492$8${eJ)=Lf)Z5%=P;X7B_(J{)7C9u8 zQdP)lltTN>y;DREX1Y~ePj{vmC3k~(jmD~Q??$lJNB+e(Z0k@hEe7WS*S~-W#BjK( z(aVyrH8(1GpQ!|y96nxK0m}vmBM4hO&^`tQG2_d@TKn;95T?cfy7K&otF&&ztselU z(V^;>P5kfwzhWW7Ta(8+v88+f3alLTg_-tY{RY#^%X)&ALcO%|np{UP%sP(gvNsnP zi6pNv>hWlJyar|eS7IiIe0>N@uSwirI5xNEU#t~9TWq<9k0> zCx?D&`SD1$9alnyYOMD=qT>u(8(75E5)qi}e`Qffa`jNJw&qbEo?wtYzI1;S&HOAL z7h~yBEIE~&TucxTn!;Li;wL68MAD(1@blqVR8XISw%zOkm$6r#aBvLu|Iv;Z$aJ>f zsk1=qp(yF*3!1T)+b=t*-7|GkD_yRcZstnXX#c_>e1Do8-jKP#-qt)!xZ<#)?M*Ne@Ts_MW76@&_5YM}vLVx& zNjE{abXGCH+Ct~=Ysjnu4*=`%@eM+P+I{!DrQpF23HOOSm!fyej(s2Pg7=pRP0Cs( z^$TwjHmSA#Qb|??yyMagADL{eqW)DQ3keFkf8}$RI#foZz)v|Kr@u!5bSlq}Y^Rw% zv9dLTAcWBmbo zG?UI$VMXW*Nl{tP=gaQKt|PlN)-X$vD3c00ux{UBE!XI?A*yVM++VC|Iz=u^gf59n z{=MMmUz02|V51M{QOV^2gBsG<>ZE`WFa$=ovpFO1aQ=h$%+$R;@_MtZeo^;*!-F${ zzy6PwlRv@K)iHLB8mqZ0v-i~u9r}Kx({ahgq|=TC2-GAJOm^tuDJ8Abf6dE1=H zK2yE6wceq%63~`0k~eHJl+3|;n{Xg3k$Us6N(^X^ap3EmzMnxUfG7ak$ zM&k~j$dZ_UMx_uBlZz1rhIT<`VHjfF)m{6X5%qHH1kK{u;zR> zsSN%{@_(uHwh96s`6di|q#UOyr;B;%aXWjPLIyP74XF9#Z~*r`#D$3mSi=Pq9pU}J zeBKJ%p6PEzz<#I;85Y&QVGI}s7SAniJ8K=R35*P2U98o2XJ2n*W6-bOy2$1PM1&U; z=nf)Dc(!8!GUw9-T~IOC^i6NXjX`2O|M~W2_E<&l)JNmbKmHrupb)`g(c=aj9Q*I^ z`p0>wW*!#I5!o;D`&vO~S`#*?l`x$t$*C73!w+O6P1is6#n=z@2jGgANa1SknF!|` zGfkzG`>;i2QUl{w{{)gD!ChBV+W=Vk`ablX441;Sp|Z^WLyhb}|BGBrP381DNJQ2_ zVY|lz6PK6lTpdj?6L3Sai~XeRGf1UuCL5R9`LiA$LaY)g*c+L%Y3Wg#;%DCd6-9O26F4KFA65R`|b< zy&{6eD+`&48-yQYz0J{}%o+OB=x}Y7Aq) z`6J-G0wn5z1S~J7qotd+ve|2v#SZcsYITt-Ty`{9#7nUt<9<^MwY2r3Yqa?{{mdv``x$Db=|Buclg$arX{ ziZvB(I3t6#Nv(#tLjkI@DL)c7><}s{-o{zpNrP6SRhbS z#{M7Vg#haVVsvh>CLkU;XHr%#H+AKxc6tQBQ7@ab0CO60+LK_|GNRwg^Fc2A14J|B zeZau!uG-eH-)1@Rm}nywA#n6ZMf?qS#v?^N7AC$h z+x53cG*xj;+V$0t+K$;=cOz`Un1bZmLzowEKLJvDc|t6#53D)v9UY;-;j1E!}dc`wC;$Ca@8Rm)0Ezu<0OQi2%NwNNWzdNr_e_ z44qAQD@B%yHp=dEm{|2fr*iyV3>t0t{s%1xhLq(5PVxr&qDogD{wfGDFV{Z2tBicA z>Ru>G8Gz{akR_|DV_XV}K`$^27Ms=|F+XWWyOnmF zMh|`S0|-}`zPedx7E0!lA-dY}tMU^QSRaJ^N97t0`#yw5S?$Fae6uqPycXbMDqW5> z2BM6A&jW$>2Nf{h2<@+!pZRD{@55|`hfa+I5fhRHgHb``<6{8%j;w=4*{`H6{q!o1n#uNG@f(Onx(?s!&CJO+X?kQy6XnK* zK5BXMdV8b(J3Dk>KlL^VSotFis0r#lP@@TkXpRZ|fXw{x#l+bBp(a+X`NWcyRlmbW zOH&W)Una{kf?qMpZ{>c@++>c*Tf48~x>;c`zB(27Bctr5z%5DHKWs`?lUIGJd+{sC z&jSyaio*xX|2xTnlgQ4HI(mtaQ9WJqsg(3TaTmD@*1WToZCe@1o34pfiWTEpIqUi;M2_aC2+V}S|QZnV2S*bQr zkKY&mF^K?X#0IC*X{2LZy(7oMyT6HGuSwoR7aw29q(@H)*bgQSxDo*hr)bSVAO}&q za+senj!`?bS*FY*__%z+;Bk4jS!(j~5{jt$0knKO4=Jmp=j*PV7P8rnkI>ifanPLo zytb)@A^jc?S zbfPR|6G#+W#n`VP3w&&h%$g<`$Hw|&UQr;h+?-H54p)C}Wj^X}wqg}LOp6!oYlB*2 zlJDNTi;*V)-Z)G4wZtAa9&E3s;EkR(tF-VzGzIbPWqp|v`}J$1iPKLq6fs6<{|ZhB z4C%RPX>tA;sII5c{QG$!!Cn_L#=PegcMsF@JW)tn1ORCX6Ln8aRGoQ02mv93Ukz>% zm-yY8^wx!A=v9RuKe=Je0013Nx>f(-v#BW%2?%f7?944iuS{@c5%RwrJZ`PC>G`B% zk%&(rMve^7Fe`{h>Ju&zsC`_zkwro1n9FpztVdCzc z0A%@OfH8~=j4ur5pj0*TTzfpUr~CMX%VNmfKgNT4Zd6z8oYxqvS4l!-^Y3NpK!%b= z2HHNPy=6`kwfr+0Rlc968nUXZym}PszZY!EV#+!Kf=zdE`|evSxvIv?Ye}fy1^KQU z>wuZs4({^9o{l`Pp1mToNNkFKUXO_b1k5zQL@S&?)x3hmkL7Q19=%P$FW3F09&<`b z3!%mXl=K~`~B`(^2sDlvf;~sOlT*I<*4tOXbvVyjVKZ)jBI7Cxtm~W=$ zmghl4fjQ`lx&X^!O8}7etmrBvyMIlR2?fKb5y=Yn@{5nI2M~MS+PbRtk$a67J1a63 zW%I83m5f6j9YBt*RXq@$^v6r>z7Yk`*u+o+@;SH*hp|5qig3XDw9QtRZ&O;n=EGFx zKm zRZhBrrS?Zn)E~jVP07dA`0oK9rM9g$CSVTti%9#*6vbGY9nBrzj|Tb}jL3wW-?@@- zP$BG>n`_4vN(B5|Xgr`KY|i?&;d}}K+V1t^#KJ|WGWozi;q6$B(^tjSn|ZdLD+2KK zmI~1{6=D~x#TZ>;u;?Db9eW%V8OmW`{Wk5USXhWH=(J$im*Hsny-8^AP8_HuypoRq zaP0hGkV1UY1DMbg%k^2#lt^0Xggn6B&Yi42c{udVlIeQ8+*=^fw?7d>|J{~(Q9e*A zCC6M;n8Kh-rhEtk))6V=+<5P`l8=di!$|Qp+L<@spw@QMF9s@{M5Wy0X^>)J|G73A z<_A`RAEwaDfX5ikCcQiLY{iUUG^hZWkYDUxfXgW!jyYu#3Si5XLo%3C2#EekW$oF1 z2M$^rJ*W|4Y%tsjW-i4B`@5XfeELN2DzcYh~YX z_BGzRyH9U&cd+|I|Omn!1B05V|!|8q@ z0my0kcjxA%d92*n!?>5%@*7RWS!3g=Zu)yT%rwo7RJFG}lb2s((jka-B}kNA-Jvn? z;pi&r4bD0z)UST86b?fUgb4L!jm-D?t)?Y`&lmzmshG!@RlpN`cU<;P8rtA#7#p{9 zuQxavFI1w1AvA!R%WyfghNqR&o2^3qC)n5_1AiKlr!z?QXV4BYjDg#Fm05SUG@&q) zUsl2oVg!V*r=wfVN9bIGMfnl(n|0h_=o^puAH*+vX<)Qt=+9 zue65m&yl#XfXousL?O!MN2m*Wqu20D%_e5q>3lcCq3CY$l~ZfI&8FSrx;6y^U{xLf zR;8p{DOn+g4Ma4k(Dzywjxnxh6pU|U886>hhR;FSp-aql2I0m6=B;4Hl%lAH1h8Vx zbvYph&{rQG6x2JqFAYZ-+Isc%bly^kmJA3GhGaT1%pDrg@kmxiPR`6Ji{*Cu;jR#4 zulliuZGVH?y-B1cU|17}L;+UaSd>bN#(d#?x5x=h>DY&m-DO@>sU> z{_J$ce@g&pW&|E!SgR>%T!Vv9L5Nxnx01l9dO$sXawr!GqPTK*D;iXOsndC4Q3Eerx&I}Rxhs0#>5 z<-+SG*B!_d@AaHTh)Z!$-LMtfpW*~{4vfs-r_N_^=k_BWN=YGleWyh7aNA<~in5Nc zPcrd|LWl;-hWG5L!Rt}4ioN$$?g%(JOc*E!wW|D-vl8nf0qlx3{$haQjQ%Xoc;se_ zf_Lvx*zA4I!DFI=MYhg$1{#E81xoOL0<6+6Ef8(-4iIMeIRv7>Z59isN>=&KZLyHP zN4kk;hM>sbd?;9S4D0tr+vG%Az_qVx;*GFt{Gkilx9-?;qR#JUB|s@V$PWSSg}F=R zwjAu32{Yl_l{>`glTDx#H%O$N=o|{~&*kXp*Vp1-(nJG%IYR&(Na{&Z91IIUs8{I8}C|oG>{!S;_&+=Dq;oE)=<}t5K_BBfGmNje1m{ z?~q@jV%hdF8C33sN-F<&)5`K9_40b^I^Kxs%8OJKM6G%b?$E0}II%bSJ=bL27Iyvh zb3Qur4hU#1kvzYjuHmUhy?9LB6A6|zWkyrAjHhwfk^(&hyHZs$TpN6KxPN*dSd0EL z)>VzuX5kLA?~dvPl>F{&)WW|&@NH`X5}GCj%;_bY-SBdSQ(x_Gwh7l;{(!|FYtanA zY~KCDwW~^E+XgQm!C-JOwP*qfwNkMZhz7IDiOK)yw?gGw{KV4@xdP(ddHTyzSz}bk3fYFPZD&31kxI&t#gI#*X4ZQj1;byzyS1lpq^e zbbj(f5=6~@1g5X)yTz?SqIQlCB5E=6KJSE?uoz>NCi@yh>AR9pi(pO&RG3`ebtS%| zLyh837kCHUKB+6RCwiaE_cUQzLA z$Cqv!R#8*`d;rQKGUtFA?l*i3eey(Pp7)AvZen-WzvwJ3+Sd1Hk)*lv*&A0CY;ssG zggB~a&q@j`k&(f&bJK>kE39q)?G{oIpkn*jv|b>Ap5qCrG;nzo#O0AVYsq1*kPUuO z?;j@J4MX*g_&k!XX1$c{5hs2aEK7?y4)*fSnvLSThP1F%w5AEu&b|G}ydEE>wK5(G zYG!>`*8Dpb?S`QPB!@{Qi z?o3zgr6Dykb}ABZX}T^sb1XSiUSysm*KdkOImgVjN^Wv!7gi=gmkjIh?!tZNo?wZ{mt^uN{DghA6ZpX5Q zky=$YcRzN%AYgVOL~ri%6P*z3n?`h;VYb(%&l>gDcvx2yCrN+aBfBU$U^>6_oytj{ zrKQCOt$h}K_E43}y2KkZc6qV=?N#n&$k*4%g#*;GLEukGNK&5fvk9I>11h#kEPvJh zlQsr(*YK&acnNc;NeW+NOQd#tNX2`iUDEW{YiH{;hEr#30v*scPnmtSbY~3eRDDy0 zN%y%>`2lG24bMHevz6pYz?KkyNRm<#+S{&B3ef~1Gi5w+WAK4+EGe0jb08v6qZsN^NvBi}&t@h{gh zlblXR_2M0g1erJJ$$(;kfkL73#+I`7*|FW&#HpKeeuaIad zkhBc`^IDd?r|Egp@3z}WH?gD)+qXz)(@$Zmj=c3>Lb)lBv5{aEx@BhUlx$HRB%Q8V zo#gtUoN#f)x|*CtMgLbvePk;TL3~=>De!2eKDW{52j$WOV~m(rYW+_!!mt}DPg?43 z>-Z9O*p~XC*yPCA^nfWfk_{Q676nmZ^WJ_Nv>KD`cu$l?H$?2>`qgio(+il{WzfYe zCb!`)JJGR#@?K*$HI9FRAx>G3Zw&y^5J|a=r)y6C?jO)JC;je}dpakLO_TRa^_ z-FoD6*{UzBoPr}u$q1q=d z!VzVRZ#A;{^U}-p--H+n1(b`&;Q?KZMtSe`p{R+!o)Wj~C$y!~qXDwG^ae=^?w{+7 z2atFZ4L3YhG=@dW5HLJm&kmznPMby8QLVYX2v0gwCMNg%+I`3OzC~w1g*LO ze}A%;=lyP=z_MPkdFe@u-IJQDq`y!9)fHzsrsH8E5)cZ>Ns4JgBr_|7 zSG=5Kt9E|UmSeg`I~P2U{(a=C%FEP9s@uBxwJDhQjq0&jGsWbluOzjlDn8E@6vYSA zjm?62OW%58FdeHg#>3oySV=mgg*lnaUqf8)x1Z$uAh?z@vElbW{0b1!!>wnV@r&Aoykd@wH4#Q!1D$v0quMyOsWX@ z^=IHwdoS|~8^0bWQHgp&w}L8zbu^9w-sWWfhp zBgQ%{ualiv_}3Qlo_b-rxjBEa3y;Xqd+4%4h1Ut6ZS}G(Wrf{m#TCEi&(z;Nm2AnN znuxA(D>3oKvyKIxN-F@|aUkGf)g3ka0Ho>J&t0x!kpdxf1wP&W0TL#@VG5KC*t(pb zmlN@=2l8!MMG!$5b`~^$V@n^M8K>yFcZLXmo>IssbqQf?JRv{h3q40^?8HITm`KRm zKAiI(tj&WmXjN9r%x>sE6XavAEA6PJ9#ca~o>+z*Gd^-5>{;?)E{kPw@}Gi%IBiv_b}J}Wh~OjEac(8g^lX`F;-(tAJ7J{LJ*~ZZnk195b|OJ=F?}f z&bdMORoyF2%nj!b8*yu)2sd6Y`pB*ks^g;s{B`GROuw-w7^J~oV@|EDtc@z8$kgEN z_`F;4jeQC~qZm$NS+FC`?8phsEfS~KG0Syd6UBUf#BJX+KGh&)MaGP5;Bz(26YB(H zcgn+_5Q+G3=UM4FTU>KB1Z6E8Mcq0KnIVoky3USlm F{|7cAV&4D& literal 0 HcmV?d00001 diff --git a/service/src/main/resources/static/js/index.js b/service/src/main/resources/static/js/index.js new file mode 100644 index 00000000..d9e72b01 --- /dev/null +++ b/service/src/main/resources/static/js/index.js @@ -0,0 +1,17 @@ + $(document).foundation() + +// This code expects the following templated values to be set in index.html +// var buildUserEmail = "[[${gitBuildUserEmail}]]"; +// var branch = "[[${gitBranch}]]"; +// var commitId = "[[${gitCommitId}]]"; +// var buildTime = "[[${gitBuildTime}]]"; + +// Set additional content when ready +$(function() { + var gitInfoHtml = 'Built By: ' + buildUserEmail + '
' + + 'Branch: ' + branch + '
' + + 'Commit: ' + commitId + '
' + + 'Date/Time: ' + buildTime; + + $("#gitInfo").html(gitInfoHtml); +}); diff --git a/service/src/main/resources/templates/index.html b/service/src/main/resources/templates/index.html new file mode 100644 index 00000000..1c3c2d0a --- /dev/null +++ b/service/src/main/resources/templates/index.html @@ -0,0 +1,104 @@ + + + + + + + DataWave Docs + + + + +

+
+
    +
    • +
+
+
+ +
+
+
+

Welcome to DataWave

+
+ +
+
+ +
API Documents
+
+
+ +
Query Help
+
+
+ +
Release Notes
+
+
+ + + + + + +
+
+
+
+
+ + + diff --git a/service/src/main/resources/templates/query_help.html b/service/src/main/resources/templates/query_help.html new file mode 100644 index 00000000..36466b0d --- /dev/null +++ b/service/src/main/resources/templates/query_help.html @@ -0,0 +1,241 @@ + + + + + DataWave Web Service Query Documentation - Version ###version### + + + +

Query Overview

+

The web service allows users to query the warehouse in several different ways. The Query + api presents a single set of methods for starting, using, and closing a query. The web service hosts different types of + query logic each with their own response type. Each query has the same lifecycle: +

    +
  1. User defines the query be calling either the Query/create or Query/define method. This returns a query id.
    2. +
  2. User iterates over the results by repetitive calls to Query/{queryId}/next. Each call to next will return a page of results. The page size was + set in step 1, or can be changed with a call to the update operation. The type of response is dependent on the query logic, and is listed on the + query logic page.
    3. +
  3. When the user receives a HTTP 204 response code (No Content), then there are no more results. The user should release the resources associated + with their query by calling Query/{queryId}/close
    4. +
+

JEXL Query Syntax

+

Query logic classes will use different types of syntax for the query string. Most of the queries that retrieve raw events will accept JEXL (default) + or our modified LUCENE syntax. We support a subset of the language elements in the Apache Commons JEXL syntax and we have implemented some of our own + custom functions. We support the following JEXL operators: +

    +
  • ==
    • +
  • !=
    • +
  • <
    • +
  • ≤
    • +
  • >
    • +
  • ≥
    • +
  • =~ (regex)
    • +
  • !~ (negative regex)
    • +
+

We have created the following functions, see the examples section below to see their usage: +

    +
  • Content functions: phrase(), adjacent(), and within()
    • +
  • Geo Functions: within_bounding_box() and within_circle()
    • +
  • GeoWave Functions: intersects_bounding_box(), intersects_radius_km(), contains(), covers(), coveredBy(), crosses(), intersects(), overlaps(), and within() +
  • Utility Functions: between() and length()
    • +
+

Since JEXL is an expression language and not a text query language, we also added support for unfielded queries. For most of the query + logic types that query raw events, we have provided the capability to search on any indexed field. To achieve this, the user should + submit their query with the field name: _ANYFIELD_. See the examples below for its usage. +

LUCENE Query Syntax

+

We have modified the LUCENE syntax such that the NOT operator is not unary, AND's are not fuzzy, and the implicit operator is the AND not the OR. + Our LUCENE syntax has the following form: + 

+		ModClause ::= DisjQuery [NOT DisjQuery]*
+		DisjQuery ::= ConjQuery [ OR ConjQuery ]*
+		ConjQuery ::= Query [ AND Query ]*
+		Query ::= Clause [ Clause ]*
+		Clause ::= Term | [ ModClause ]
+		Term ::=
+			field:selector |
+			field:selec* |
+			field:selec*or |
+			field:*lector |
+			field:selec?or |
+			selector | (can use wildcards)
+			field:[begin TO end] |
+			field:{begin TO end} |
+			"quick brown dog" |
+			"quick brown dog"~20 |
+			#FUNCTION(ARG1, ARG2)
+
+

Note that to search for punctuation characters within a term, you need to escape it with a backslash. +

We have also modified the LUCENE syntax to provide support for the JEXL syntax that LUCENE does not support. The table below maps the + JEXL operators to the supported LUCENE syntax: +
+ + + + + + + + + + + + + + + + +
JEXL OperatorLUCENE Operator
filter:includeRegex(regex)#INCLUDE(regex)
filter:includeRegex(field, regex)#INCLUDE(field, regex)
filter:includeRegex(field1, regex1) <op> filter:includeRegex(field2, regex2) ...#INCLUDE(op, field1, regex1, field2, regex2 ...) where op is 'or' or 'and'
filter:excludeRegex(regex)#EXCLUDE(regex)
filter:excludeRegex(field, regex)#EXCLUDE(field, regex)
filter:excludeRegex(field1, regex1) <op> filter:excludeRegex(field2, regex2) ...#EXCLUDE(op, field1, regex1, field2, regex2 ...) where op is 'or' or 'and'
filter:isNull(field)#ISNULL(field)
not(filter:isNull(field))#ISNOTNULL(field)
filter:occurrence(field,operator,count))#OCCURRENCE(field,operator,count)
filter:timeFunction(field1,field2,operator,equivalence,goal)#TIME_FUNCTION(field1,field2,operator,equivalence,goal)
filter:text(field)#TEXT(field)
filter:text(field, value)#TEXT(field, value)
filter:text(field1, value1) <op> filter:text(field2, value2) ...#TEXT(op, field1, value1, field2, value2 ...) where op is 'or' or 'and'
filter:compare(field1, op, mode, field2)#COMPARE(field1, op, mode, field2) where op is '>', '>=', '<', '<=', '==', '=', or '!=' and mode is 'ALL' or 'ANY'
+

Notes: +

    +
  1. None of these filter functions can be applied against index-only fields except for filter:text (#TEXT).
    2. +
  2. No indicies will be used for the INCLUDE and EXCLUDE functions, however they will be used for the TEXT function. All of these will be matched against the original unnormalized value. +
  3. The occurrence function is used to count the number of instances of a field in the event. Valid operators are '==' (or '='),'>','>=','<','<=', and '!='.
    4. +
+

Some geo functions are supplied as well that may prove useful although the within_bounding_box function may be done with a simple range comparison (i.e. LAT_LON_USER <= <lat1>_<lon1> and LAT_LON_USER >= <lat2>_<lon2>.

+ + + + + +
JEXL OperatorLUCENE Operator
geo:within_bounding_box(latLonField, lowerLeft, upperRight)#GEO(bounding_box, latLonField, 'lowerLeft', 'upperRight')
geo:within_bounding_box(lonField, latField, minLon, minLat, maxLon, maxLat)#GEO(bounding_box, lonField, latField, minLon, minLat, maxLon, maxLat)
geo:within_circle(latLonField, center, radius)#GEO(circle, latLonField, center, radius)
+

Notes: +

    +
  1. All lat and lon values are in decimal.
    2. +
  2. The lowerLeft, upperRight, and center are of the form <lat>_<lon> and must be surrounded by single quotes.
    3. +
  3. The radius is in decimal degrees as well.
    4. +
+

New GeoWave geo functions are available as follows:

+ + + + + + + + + + + + + + + + + + + + + + + + + +
JEXL OperatorLUCENE Operator
geowave:intersects_bounding_box(geometryField, westLon, eastLon, southLat, northLat)#INTERSECTS_BOUNDING_BOX(geometryField, westLon, eastLon, southLat, northLat)
geowave:intersects_radius_km(geometryField, centerLon, centerLat, radiusKm)#INTERSECTS_RADIUS_KM(geometryField, centerLon, centerLat, radiusKm)
geowave:contains(geometryField, Well-Known Text)#CONTAINS(geometryField, centerLon, centerLat, radiusDegrees)
geowave:covers(geometryField, Well-Known Text)#COVERS(geometryField, Well-Known Text)
geowave:coveredBy(geometryField, Well-Known Text)#COVERED_BY(geometryField, Well-Known Text)
geowave:crosses(geometryField, Well-Known Text)#CROSSES(geometryField, Well-Known Text)
geowave:intersects(geometryField, Well-Known Text)#INTERSECTS(geometryField, Well-Known Text)
geowave:overlaps(geometryField, Well-Known Text)#OVERLAPS(geometryField, Well-Known Text)
geowave:within(geometryField, Well-Known Text)#WITHIN(geometryField, Well-Known Text)
+

Notes: +

    +
  1. All lat and lon values are in decimal degrees.
    2. +
  2. The lowerLeft, upperRight, and center are of the form <lat>_<lon> and must be surrounded by single quotes.
    3. +
  3. Geometry is represented according to the Open Geospatial Consortium standard for Well-Known Text. It is in decimal degrees longitude for x, amd latitude for y. For example, a point at New York can be represented as 'POINT (-74.01 40.71) ' and a box at New York can be repesented as 'POLYGON(( -74.1 40.75, -74.1 40.69, -73.9 40.69, -73.9 40.75, -74.1 40.75))'
    4. +
+ +

An #EVALUATION_ONLY function has recently been introduced which is intended to be used to defer execution of part of a query to the evaluation phase. This could be used to defer a range to the evaluation phase, or possibly to defer GeoWave function evaluation to the evaluation phase, avoiding the need to generate geo indices when additional indexed fields are present (i.e. indexed fields other than the geo field). For example:

+ + + + + + + + + + +
JEXL Example
INDEXED_FIELD == 'some value' && ((ASTEvaluationOnly = true) && geowave:intersects(geometryField, Well-Known Text))
LUCENE Example
INDEXED_FIELD: 'some value' AND #EVALUATION_ONLY('#INTERSECTS(geometryField, Well-Known Text)')
+ + +

There are some additional functions that are supplied to handle dates more smoothly. It is intended that the need for these functions + may go away in future versions (bolded parameters are literal, other parameters are substituted with appropriate values): + + + + + + + + + + + + + + + + + + +
JEXL OperatorLUCENE Operator
filter:betweenDates(field, start date, end date)#DATE(field, start date, end date) or #DATE(field, between, start date, end date)
filter:betweenDates(field, start date, end date, start/end date format)#DATE(field, start date, end date, start/end date format) or #DATE(field, between, start date, end date, start/end date format)
filter:betweenDates(field, field date format, start date, end date, start/end date format)#DATE(field, field date format, start date, end date, start/end date format) or #DATE(field, between, field date format, start date, end date, start/end date format)
filter:afterDate(field, date)#DATE(field, after, date)
filter:afterDate(field, date, date format)#DATE(field, after, date, date format)
filter:afterDate(field, field date format, date, date format)#DATE(field, after, field date format, date, date format)
filter:beforeDate(field, date)#DATE(field, before, date)
filter:beforeDate(field, date, date format)#DATE(field, before, date, date format)
filter:beforeDate(field, field date format, date, date format)#DATE(field, before, field date format, date, date format)
filter:betweenLoadDates(LOAD_DATE, start date, end date)#LOADED(start date, end date) or #LOADED(between, start date, end date)
filter:betweenLoadDates(LOAD_DATE, start date, end date, start/end date format)#LOADED(start date, end date, start/end date format) or #LOADED(between, start date, end date, start/end date format)
filter:afterLoadDate(LOAD_DATE, date)#LOADED(after, date)
filter:afterLoadDate(LOAD_DATE, date, date format)#LOADED(after, date, date format)
filter:beforeLoadDate(LOAD_DATE, date)#LOADED(before, date)
filter:beforeLoadDate(LOAD_DATE, date, date format)#LOADED(before, date, date format)
filter:timeFunction(DOWNTIME, UPTIME, '-', '>', 2522880000000L)#TIME_FUNCTION(DOWNTIME, UPTIME, '-', '>', '2522880000000L')
+

Notes: +

    +
  1. None of these filter functions can be applied against index-only fields.
    2. +
  2. Between functions are inclusive, and the other functions are exclusive of the entered dates.
    3. +
  3. Date formats must be entered in the Java SimpleDateFormat object format.
    4. +
  4. If the entered date format is not specified, then the following list of date formats will be tried:
    5. +
      +
    • yyyyMMdd:HH:mm:ss:SSSZ
      • +
    • yyyyMMdd:HH:mm:ss:SSS
      • +
    • EEE MMM dd HH:mm:ss zzz yyyy
      • +
    • d MMM yyyy HH:mm:ss 'GMT'
      • +
    • yyyy-MM-dd HH:mm:ss.SSS Z
      • +
    • yyyy-MM-dd HH:mm:ss.SSS
      • +
    • yyyy-MM-dd HH:mm:ss.S Z
      • +
    • yyyy-MM-dd HH:mm:ss.S
      • +
    • yyyy-MM-dd HH:mm:ss Z
      • +
    • yyyy-MM-dd HH:mm:ssz
      • +
    • yyyy-MM-dd HH:mm:ss
      • +
    • yyyyMMdd HHmmss
      • +
    • yyyy-MM-dd'T'HH'|'mm
      • +
    • yyyy-MM-dd'T'HH':'mm':'ss'.'SSS'Z'
      • +
    • yyyy-MM-dd'T'HH':'mm':'ss'Z'
      • +
    • MM'/'dd'/'yyyy HH':'mm':'ss
      • +
    • E MMM d HH:mm:ss z yyyy
      • +
    • E MMM d HH:mm:ss Z yyyy
      • +
    • yyyyMMdd_HHmmss
      • +
    • yyyy-MM-dd
      • +
    • MM/dd/yyyy
      • +
    • yyyy-MMMM
      • +
    • yyyy-MMM
      • +
    • yyyyMMddHHmmss
      • +
    • yyyyMMddHHmm
      • +
    • yyyyMMddHH
      • +
    • yyyyMMdd
      • +
    +
  5. A special date format of 'e' can be supplied to mean milliseconds since epoch.
    6. +
+

Unique Functions

+

In queries, unique functions are available to find unique results for fields that are + specified, to the granularity of DAY, HOUR, or MINUTE. The field passed to the function needs to be a + dateTime field if using a time granularity, otherwise the original field will be returned as a unique result. Multiple granularity levels can + be provided to the same field in multiple ways, and granularities will be merged across multiple functions if necessary. + Potential function options that the user may use are listed below.

+
    +
  • #unique(field): Find unique results for every value of field. No restriction on the type of field.
    • +
  • #unique(datetimeField[DAY]): Find every unique result of datetimeField to the day.
    • +
  • #unique_by_day(datetimeField): Same as above, just a different means of formatting.
    • +
  • #unique(datetimeField[HOUR]): Find every unique result of datetimeField to the hour.
    • +
  • #unique_by_hour(datetimeField): Same result as above.
    • +
  • #unique(datetimeField[MINUTE]): Find every unique result of datetimeField to the minute.
    • +
  • #unique_by_minute(datetimeField): Same result as above.
    • +
  • #unique(field,datetimeField1[DAY],datetimeField2[HOUR],datetimeField3[MINUTE]): Find every unique result for field, field1 to the day, field2 to the hour, field3 to the minute.
    • +
  • #unique(field,datetimeField[DAY,HOUR]): Consecutive granularities for the same field are supported. Find each unique result for field, and datetimeField to the day, as well as to the hour.
    • +
  • #unique(field1,field2[DAY]) AND #unique_by_minute(field1,field2,field3) AND #unique_by_minute(field4): Find unique results for field1 with no restrictions as well as with minute restrictions, field2 to the day and the minute, field3 by the minute, and field4 by the minute.
    • +
+

As referenced above, multiple granularity levels can be specified for the same field in different ways. The following queries are equivalent, and will find + each unique result for field, to the hour and to the minute specified:

+
    +
  • #unique(field[HOUR,MINUTE])
    • +
  • #unique(field[HOUR],field[MINUTE])
    • +
  • #unique(field[HOUR]) AND #unique(field[MINUTE])
    • +
  • #unique(field[HOUR]) AND #unique_by_minute(field)
    • +
  • #unique_by_hour(field) AND #unique_by_minute(field)
    • +
+
+ From 27236fedf52d1928bc2de64cfb264beb9f4ba4f9 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 10 Jun 2022 16:20:55 -0400 Subject: [PATCH 153/218] Updated query help to point to query swagger ui --- service/src/main/resources/templates/query_help.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/src/main/resources/templates/query_help.html b/service/src/main/resources/templates/query_help.html index 36466b0d..566b1c00 100644 --- a/service/src/main/resources/templates/query_help.html +++ b/service/src/main/resources/templates/query_help.html @@ -7,7 +7,7 @@

Query Overview

-

The web service allows users to query the warehouse in several different ways. The Query +

The web service allows users to query the warehouse in several different ways. The Query api presents a single set of methods for starting, using, and closing a query. The web service hosts different types of query logic each with their own response type. Each query has the same lifecycle:

    From 5dc76aec050f4375c7823e5565983ee273233ca6 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 16 Jun 2022 15:29:35 -0400 Subject: [PATCH 154/218] Added swagger documentation for the query controller --- .../microservice/query/QueryController.java | 2261 ++++++++++++++++- .../query/QueryManagementService.java | 18 +- .../microservice/query/WebController.java | 1 - .../query/config/QueryServiceConfig.java | 2 - .../src/main/resources/config/application.yml | 2 +- .../src/main/resources/templates/index.html | 2 +- 6 files changed, 2214 insertions(+), 72 deletions(-) diff --git a/service/src/main/java/datawave/microservice/query/QueryController.java b/service/src/main/java/datawave/microservice/query/QueryController.java index 25a65b4c..bd453049 100644 --- a/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/service/src/main/java/datawave/microservice/query/QueryController.java @@ -18,6 +18,17 @@ import datawave.webservice.result.QueryImplListResponse; import datawave.webservice.result.QueryLogicResponse; import datawave.webservice.result.VoidResponse; +import io.swagger.v3.oas.annotations.ExternalDocumentation; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.Parameters; +import io.swagger.v3.oas.annotations.enums.ParameterIn; +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; @@ -35,10 +46,29 @@ import java.util.List; +import static datawave.microservice.query.QueryParameters.QUERY_AUTHORIZATIONS; +import static datawave.microservice.query.QueryParameters.QUERY_BEGIN; +import static datawave.microservice.query.QueryParameters.QUERY_END; +import static datawave.microservice.query.QueryParameters.QUERY_MAX_CONCURRENT_TASKS; +import static datawave.microservice.query.QueryParameters.QUERY_MAX_RESULTS_OVERRIDE; +import static datawave.microservice.query.QueryParameters.QUERY_NAME; +import static datawave.microservice.query.QueryParameters.QUERY_PAGESIZE; +import static datawave.microservice.query.QueryParameters.QUERY_PAGETIMEOUT; +import static datawave.microservice.query.QueryParameters.QUERY_PARAMS; +import static datawave.microservice.query.QueryParameters.QUERY_PLAN_EXPAND_FIELDS; +import static datawave.microservice.query.QueryParameters.QUERY_PLAN_EXPAND_VALUES; +import static datawave.microservice.query.QueryParameters.QUERY_POOL; +import static datawave.microservice.query.QueryParameters.QUERY_STRING; +import static datawave.microservice.query.QueryParameters.QUERY_VISIBILITY; import static datawave.microservice.query.lookup.LookupService.LOOKUP_STREAMING; import static datawave.microservice.query.lookup.LookupService.LOOKUP_UUID_PAIRS; +import static datawave.microservice.query.translateid.TranslateIdService.TRANSLATE_ID; +import static datawave.query.QueryParameters.QUERY_SYNTAX; import static datawave.services.query.logic.lookup.LookupQueryLogic.LOOKUP_KEY_VALUE_DELIMITER; +@Tag(name = "Query Controller /v1", description = "DataWave Query Management", + externalDocs = @ExternalDocumentation(description = "Query Service Documentation", + url = "https://github.com/NationalSecurityAgency/datawave-query-service")) @RestController @RequestMapping(path = "/v1", produces = MediaType.APPLICATION_JSON_VALUE) public class QueryController { @@ -67,59 +97,549 @@ public QueryController(QueryManagementService queryManagementService, LookupServ this.queryMetricsEnrichmentContext = queryMetricsEnrichmentContext; } - /** - * @see QueryManagementService#listQueryLogic(ProxiedUserDetails) - */ - @Timed(name = "dw.query.listQueryLogic", absolute = true) - @RequestMapping(path = "listQueryLogic", method = {RequestMethod.GET}, - produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "text/html"}) - public QueryLogicResponse listQueryLogic(@AuthenticationPrincipal ProxiedUserDetails currentUser) { - return queryManagementService.listQueryLogic(currentUser); - } - /** * @see QueryManagementService#define(String, MultiValueMap, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Defines a query using the given query logic and parameters.", + description = "Defined queries cannot be started and run.
    " + + "Auditing is not performed when defining a query.
    " + + "Updates can be made to any parameter using update.
    " + + "Create a runnable query from a defined query using duplicate or reset.
    " + + "Delete a defined query using remove.
    " + + "Aside from a limited set of admin actions, only the query owner can act on a defined query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a generic response containing the query id", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class))), + @ApiResponse( + description = "if parameter validation fails
    " + + "if query logic parameter validation fails
    " + + "if security marking validation fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_STRING, + in = ParameterIn.QUERY, + description = "The query string", + required = true, + schema = @Schema(implementation = String.class), + example = "GENRES:[Action to Western]"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_SYNTAX, + in = ParameterIn.QUERY, + description = "The syntax used in the query", + schema = @Schema(implementation = String.class), + example = "LUCENE"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on @Timed(name = "dw.query.defineQuery", absolute = true) @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE) @RequestMapping(path = "{queryLogic}/define", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public GenericResponse define(@PathVariable String queryLogic, @RequestParam MultiValueMap parameters, - @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public GenericResponse define(@Parameter(description = "The query logic", example = "EventQuery") @PathVariable String queryLogic, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) + throws QueryException { return queryManagementService.define(queryLogic, parameters, currentUser); } + /** + * @see QueryManagementService#listQueryLogic(ProxiedUserDetails) + */ + @Operation(summary = "Gets a list of descriptions for the configured query logics, sorted by query logic name.", + description = "The descriptions include things like the audit type, optional and required parameters, required roles, and response class.") + @Timed(name = "dw.query.listQueryLogic", absolute = true) + @RequestMapping(path = "listQueryLogic", method = {RequestMethod.GET}, + produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "text/html"}) + public QueryLogicResponse listQueryLogic(@AuthenticationPrincipal ProxiedUserDetails currentUser) { + return queryManagementService.listQueryLogic(currentUser); + } + /** * @see QueryManagementService#create(String, MultiValueMap, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Creates a query using the given query logic and parameters.", + description = "Created queries will start running immediately.
    " + + "Auditing is performed before the query is started.
    " + + "Query results can be retrieved using next.
    " + + "Updates can be made to any parameter which doesn't affect the scope of the query using update.
    " + + "Stop a running query gracefully using close or forcefully using cancel.
    " + + "Stop, and restart a running query using reset.
    " + + "Create a copy of a running query using duplicate.
    " + + "Aside from a limited set of admin actions, only the query owner can act on a running query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a generic response containing the query id", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class))), + @ApiResponse( + description = "if parameter validation fails
    " + + "if query logic parameter validation fails
    " + + "if security marking validation fails
    " + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_STRING, + in = ParameterIn.QUERY, + description = "The query string", + required = true, + schema = @Schema(implementation = String.class), + example = "GENRES:[Action to Western]"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_SYNTAX, + in = ParameterIn.QUERY, + description = "The syntax used in the query", + schema = @Schema(implementation = String.class), + example = "LUCENE"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on @Timed(name = "dw.query.createQuery", absolute = true) @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE) @RequestMapping(path = "{queryLogic}/create", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public GenericResponse create(@PathVariable String queryLogic, @RequestParam MultiValueMap parameters, - @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public GenericResponse create(@Parameter(description = "The query logic", example = "EventQuery") @PathVariable String queryLogic, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) + throws QueryException { return queryManagementService.create(queryLogic, parameters, currentUser); } /** * @see QueryManagementService#plan(String, MultiValueMap, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Generates a query plan using the given query logic and parameters.", + description = "Created queries will begin planning immediately.
    " + + "Auditing is performed if we are expanding indices.
    " + + "Query plan will be returned in the response.
    " + + "Updates can be made to any parameter which doesn't affect the scope of the query using update.
    " + + "Stop a running query gracefully using close or forcefully using cancel.
    " + + "Stop, and restart a running query using reset.
    " + + "Create a copy of a running query using duplicate.
    " + + "Aside from a limited set of admin actions, only the query owner can act on a running query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a generic response containing the query plan", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class))), + @ApiResponse( + description = "if parameter validation fails
    " + + "if query logic parameter validation fails
    " + + "if security marking validation fails
    " + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_STRING, + in = ParameterIn.QUERY, + description = "The query string", + required = true, + schema = @Schema(implementation = String.class), + example = "GENRES:[Action to Western]"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_PLAN_EXPAND_FIELDS, + in = ParameterIn.QUERY, + description = "Whether to expand unfielded terms", + schema = @Schema(implementation = Boolean.class), + example = "true" + ), + @Parameter( + name = QUERY_PLAN_EXPAND_VALUES, + in = ParameterIn.QUERY, + description = "Whether to expand regex and/or ranges into discrete values
    " + + "If 'true', auditing will be performed", + schema = @Schema(implementation = Boolean.class), + example = "true" + ), + @Parameter( + name = QUERY_SYNTAX, + in = ParameterIn.QUERY, + description = "The syntax used in the query", + schema = @Schema(implementation = String.class), + example = "LUCENE"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on @Timed(name = "dw.query.planQuery", absolute = true) @RequestMapping(path = "{queryLogic}/plan", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public GenericResponse plan(@PathVariable String queryLogic, @RequestParam MultiValueMap parameters, - @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public GenericResponse plan(@Parameter(description = "The query logic", example = "EventQuery") @PathVariable String queryLogic, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) + throws QueryException { return queryManagementService.plan(queryLogic, parameters, currentUser); } /** * @see QueryManagementService#predict(String, MultiValueMap, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Generates a query prediction using the given query logic and parameters.", + description = "Created queries will begin predicting immediately.
    " + + "Auditing is not performed.
    " + + "Query prediction will be returned in the response.
    " + + "Updates can be made to any parameter which doesn't affect the scope of the query using update.
    ") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a generic response containing the query plan", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class))), + @ApiResponse( + description = "if parameter validation fails
    " + + "if query logic parameter validation fails
    " + + "if security marking validation fails
    " + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_STRING, + in = ParameterIn.QUERY, + description = "The query string", + required = true, + schema = @Schema(implementation = String.class), + example = "GENRES:[Action to Western]"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_SYNTAX, + in = ParameterIn.QUERY, + description = "The syntax used in the query", + schema = @Schema(implementation = String.class), + example = "LUCENE"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on @Timed(name = "dw.query.predictQuery", absolute = true) @RequestMapping(path = "{queryLogic}/predict", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public GenericResponse predict(@PathVariable String queryLogic, @RequestParam MultiValueMap parameters, - @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public GenericResponse predict(@Parameter(description = "The query logic", example = "EventQuery") @PathVariable String queryLogic, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) + throws QueryException { return queryManagementService.predict(queryLogic, parameters, currentUser); } @@ -127,11 +647,126 @@ public GenericResponse predict(@PathVariable String queryLogic, @Request * @see LookupService#lookupUUID(MultiValueMap, ProxiedUserDetails) * @see LookupService#lookupUUID(MultiValueMap, ProxiedUserDetails, StreamingResponseListener) */ + // @formatter:off + @Operation( + summary = "Creates an event lookup query using the query logic associated with the given uuid type(s) and parameters, and returns the first page of results.", + description = "Lookup queries will start running immediately.
    " + + "Auditing is performed before the query is started.
    " + + "Each of the uuid pairs must map to the same query logic.
    " + + "After the first page is returned, the query will be closed.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a base query response containing the first page of results", + responseCode = "200", + content = @Content(schema = @Schema(implementation = BaseQueryResponse.class))), + @ApiResponse( + description = "if no query results are found", + responseCode = "204", + content = @Content(schema = @Schema(hidden = true))), + @ApiResponse( + description = "if parameter validation fails
    " + + "if query logic parameter validation fails
    " + + "if security marking validation fails
    " + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
    " + + "if the next call times out
    " + + "if the next task is rejected by the executor
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = LOOKUP_STREAMING, + in = ParameterIn.QUERY, + description = "if true, streams all results back", + schema = @Schema(implementation = Boolean.class), + example = "true"), + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on @Timed(name = "dw.query.lookupUUID", absolute = true) @RequestMapping(path = "lookupUUID/{uuidType}/{uuid}", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public Object lookupUUID(@PathVariable(required = false) String uuidType, @PathVariable(required = false) String uuid, - @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser, + public Object lookupUUID(@Parameter(description = "The UUID type", example = "PAGE_TITLE") @PathVariable(required = false) String uuidType, + @Parameter(description = "The UUID", example = "anarchism") @PathVariable(required = false) String uuid, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser, @RequestHeader HttpHeaders headers) throws QueryException { parameters.add(LOOKUP_UUID_PAIRS, String.join(LOOKUP_KEY_VALUE_DELIMITER, uuidType, uuid)); @@ -149,11 +784,133 @@ public Object lookupUUID(@PathVariable(required = false) String uuidType, @PathV * @see LookupService#lookupUUID(MultiValueMap, ProxiedUserDetails) * @see LookupService#lookupUUID(MultiValueMap, ProxiedUserDetails, StreamingResponseListener) */ + // @formatter:off + @Operation( + summary = "Creates an event lookup query using the query logic associated with the given uuid type(s) and parameters, and returns the first page of results.", + description = "Lookup queries will start running immediately.
    " + + "Auditing is performed before the query is started.
    " + + "Each of the uuid pairs must map to the same query logic.
    " + + "After the first page is returned, the query will be closed.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a base query response containing the first page of results", + responseCode = "200", + content = @Content(schema = @Schema(implementation = BaseQueryResponse.class))), + @ApiResponse( + description = "if no query results are found", + responseCode = "204", + content = @Content(schema = @Schema(hidden = true))), + @ApiResponse( + description = "if parameter validation fails
    " + + "if query logic parameter validation fails
    " + + "if security marking validation fails
    " + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
    " + + "if the next call times out
    " + + "if the next task is rejected by the executor
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = LOOKUP_STREAMING, + in = ParameterIn.QUERY, + description = "if true, streams all results back", + schema = @Schema(implementation = Boolean.class), + example = "true"), + @Parameter( + name = LOOKUP_UUID_PAIRS, + in = ParameterIn.QUERY, + description = "The lookup UUID pairs
    " + + "To lookup multiple UUID pairs, submit multiples of this parameter", + required = true, + example = "PAGE_TITLE:anarchism", + array = @ArraySchema(schema = @Schema(implementation = String.class))), + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on @Timed(name = "dw.query.lookupUUIDBatch", absolute = true) @RequestMapping(path = "lookupUUID", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public Object lookupUUIDBatch(@RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser, - @RequestHeader HttpHeaders headers) throws QueryException { + public Object lookupUUIDBatch(@Parameter(hidden = true) @RequestParam MultiValueMap parameters, + @AuthenticationPrincipal ProxiedUserDetails currentUser, @RequestHeader HttpHeaders headers) throws QueryException { if (Boolean.parseBoolean(parameters.getFirst(LOOKUP_STREAMING))) { MediaType contentType = determineContentType(headers.getAccept(), MediaType.parseMediaType(streamingProperties.getDefaultContentType())); CountingResponseBodyEmitter emitter = baseMethodStatsContext.createCountingResponseBodyEmitter(streamingProperties.getCallTimeoutMillis()); @@ -168,11 +925,126 @@ public Object lookupUUIDBatch(@RequestParam MultiValueMap paramet * @see LookupService#lookupContentUUID(MultiValueMap, ProxiedUserDetails) * @see LookupService#lookupContentUUID(MultiValueMap, ProxiedUserDetails, StreamingResponseListener) */ + // @formatter:off + @Operation( + summary = "Creates a content lookup query using the query logic associated with the given uuid type(s) and parameters, and returns the first page of results.", + description = "Lookup queries will start running immediately.
    " + + "Auditing is performed before the query is started.
    " + + "Each of the uuid pairs must map to the same query logic.
    " + + "After the first page is returned, the query will be closed.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a base query response containing the first page of results", + responseCode = "200", + content = @Content(schema = @Schema(implementation = BaseQueryResponse.class))), + @ApiResponse( + description = "if no query results are found", + responseCode = "204", + content = @Content(schema = @Schema(hidden = true))), + @ApiResponse( + description = "if parameter validation fails
    " + + "if query logic parameter validation fails
    " + + "if security marking validation fails
    " + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
    " + + "if the next call times out
    " + + "if the next task is rejected by the executor
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = LOOKUP_STREAMING, + in = ParameterIn.QUERY, + description = "if true, streams all results back", + schema = @Schema(implementation = Boolean.class), + example = "true"), + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on @Timed(name = "dw.query.lookupContentUUID", absolute = true) @RequestMapping(path = "lookupContentUUID/{uuidType}/{uuid}", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public Object lookupContentUUID(@PathVariable(required = false) String uuidType, @PathVariable(required = false) String uuid, - @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser, + public Object lookupContentUUID(@Parameter(description = "The UUID type", example = "PAGE_TITLE") @PathVariable(required = false) String uuidType, + @Parameter(description = "The UUID", example = "anarchism") @PathVariable(required = false) String uuid, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser, @RequestHeader HttpHeaders headers) throws QueryException { parameters.add(LOOKUP_UUID_PAIRS, String.join(LOOKUP_KEY_VALUE_DELIMITER, uuidType, uuid)); @@ -190,11 +1062,133 @@ public Object lookupContentUUID(@PathVariable(required = false) String uuidType, * @see LookupService#lookupContentUUID(MultiValueMap, ProxiedUserDetails) * @see LookupService#lookupContentUUID(MultiValueMap, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Creates a batch content lookup query using the query logic associated with the given uuid type(s) and parameters, and returns the first page of results.", + description = "Lookup queries will start running immediately.
    " + + "Auditing is performed before the query is started.
    " + + "Each of the uuid pairs must map to the same query logic.
    " + + "After the first page is returned, the query will be closed.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a base query response containing the first page of results", + responseCode = "200", + content = @Content(schema = @Schema(implementation = BaseQueryResponse.class))), + @ApiResponse( + description = "if no query results are found", + responseCode = "204", + content = @Content(schema = @Schema(hidden = true))), + @ApiResponse( + description = "if parameter validation fails
    " + + "if query logic parameter validation fails
    " + + "if security marking validation fails
    " + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
    " + + "if the next call times out
    " + + "if the next task is rejected by the executor
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = LOOKUP_STREAMING, + in = ParameterIn.QUERY, + description = "if true, streams all results back", + schema = @Schema(implementation = Boolean.class), + example = "true"), + @Parameter( + name = LOOKUP_UUID_PAIRS, + in = ParameterIn.QUERY, + description = "The lookup UUID pairs
    " + + "To lookup multiple UUID pairs, submit multiples of this parameter", + required = true, + example = "PAGE_TITLE:anarchism", + array = @ArraySchema(schema = @Schema(implementation = String.class))), + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on @Timed(name = "dw.query.lookupContentUUIDBatch", absolute = true) @RequestMapping(path = "lookupContentUUID", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public Object lookupContentUUIDBatch(@RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser, - @RequestHeader HttpHeaders headers) throws QueryException { + public Object lookupContentUUIDBatch(@Parameter(hidden = true) @RequestParam MultiValueMap parameters, + @AuthenticationPrincipal ProxiedUserDetails currentUser, @RequestHeader HttpHeaders headers) throws QueryException { if (Boolean.parseBoolean(parameters.getFirst(LOOKUP_STREAMING))) { MediaType contentType = determineContentType(headers.getAccept(), MediaType.parseMediaType(streamingProperties.getDefaultContentType())); CountingResponseBodyEmitter emitter = baseMethodStatsContext.createCountingResponseBodyEmitter(streamingProperties.getCallTimeoutMillis()); @@ -208,10 +1202,117 @@ public Object lookupContentUUIDBatch(@RequestParam MultiValueMap /** * @see TranslateIdService#translateId(String, MultiValueMap, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Get one or more ID(s), if any, that correspond to the given ID.", + description = "This method only returns the first page, so set pagesize appropriately.
    " + + "Since the underlying query is automatically closed, callers are NOT expected to request additional pages or close the query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a base query response containing the first page of results", + responseCode = "200", + content = @Content(schema = @Schema(implementation = BaseQueryResponse.class))), + @ApiResponse( + description = "if no query results are found", + responseCode = "204", + content = @Content(schema = @Schema(hidden = true))), + @ApiResponse( + description = "if parameter validation fails
    " + + "if query logic parameter validation fails
    " + + "if security marking validation fails
    " + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
    " + + "if the next call times out
    " + + "if the next task is rejected by the executor
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on @RequestMapping(path = "translateId/{id}", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public BaseQueryResponse translateId(@PathVariable String id, @RequestParam MultiValueMap parameters, - @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public BaseQueryResponse translateId(@Parameter(description = "The ID to translate") @PathVariable String id, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) + throws QueryException { return translateIdService.translateId(id, parameters, currentUser); } @@ -219,138 +1320,884 @@ public BaseQueryResponse translateId(@PathVariable String id, @RequestParam Mult * @see TranslateIdService#translateIds(MultiValueMap, ProxiedUserDetails) */ // TODO: Shouldn't the case for this path be the same as the singular call? + // @formatter:off + @Operation( + summary = "Get the ID(s), if any, associated with the specified IDs.", + description = "Because the query created by this call may return multiple pages, callers are expected to request additional pages and eventually close the query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a base query response containing the first page of results", + responseCode = "200", + content = @Content(schema = @Schema(implementation = BaseQueryResponse.class))), + @ApiResponse( + description = "if no query results are found", + responseCode = "204", + content = @Content(schema = @Schema(hidden = true))), + @ApiResponse( + description = "if parameter validation fails
    " + + "if query logic parameter validation fails
    " + + "if security marking validation fails
    " + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
    " + + "if the next call times out
    " + + "if the next task is rejected by the executor
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = TRANSLATE_ID, + in = ParameterIn.QUERY, + description = "The IDs to translate", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on @RequestMapping(path = "translateIDs", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public BaseQueryResponse translateIDs(@RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) - throws QueryException { + public BaseQueryResponse translateIDs(@Parameter(hidden = true) @RequestParam MultiValueMap parameters, + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return translateIdService.translateIds(parameters, currentUser); } /** * @see QueryManagementService#createAndNext(String, MultiValueMap, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Creates a query using the given query logic and parameters, and returns the first page of results.", + description = "Created queries will start running immediately.
    " + + "Auditing is performed before the query is started.
    " + + "Subsequent query results can be retrieved using next.
    " + + "Updates can be made to any parameter which doesn't affect the scope of the query using update.
    " + + "Stop a running query gracefully using close or forcefully using cancel.
    " + + "Stop, and restart a running query using reset.
    " + + "Create a copy of a running query using duplicate.
    " + + "Aside from a limited set of admin actions, only the query owner can act on a running query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a base query response containing the first page of results", + responseCode = "200", + content = @Content(schema = @Schema(implementation = BaseQueryResponse.class))), + @ApiResponse( + description = "if no query results are found", + responseCode = "204", + content = @Content(schema = @Schema(hidden = true))), + @ApiResponse( + description = "if parameter validation fails
    " + + "if query logic parameter validation fails
    " + + "if security marking validation fails
    " + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
    " + + "if the next call times out
    " + + "if the next task is rejected by the executor
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_STRING, + in = ParameterIn.QUERY, + description = "The query string", + required = true, + schema = @Schema(implementation = String.class), + example = "GENRES:[Action to Western]"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_SYNTAX, + in = ParameterIn.QUERY, + description = "The syntax used in the query", + schema = @Schema(implementation = String.class), + example = "LUCENE"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on @Timed(name = "dw.query.createAndNext", absolute = true) @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.CREATE_AND_NEXT) @RequestMapping(path = "{queryLogic}/createAndNext", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public BaseQueryResponse createAndNext(@PathVariable String queryLogic, @RequestParam MultiValueMap parameters, - @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public BaseQueryResponse createAndNext(@Parameter(description = "The query logic", example = "EventQuery") @PathVariable String queryLogic, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) + throws QueryException { return queryManagementService.createAndNext(queryLogic, parameters, currentUser); } /** * @see QueryManagementService#next(String, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Gets the next page of results for the specified query.", + description = "Next can only be called on a running query.
    " + + "If configuration allows, multiple next calls may be run concurrently for a query.
    " + + "Only the query owner can call next on the specified query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a base query response containing the next page of results", + responseCode = "200", + content = @Content(schema = @Schema(implementation = BaseQueryResponse.class))), + @ApiResponse( + description = "if no query results are found", + responseCode = "204", + content = @Content(schema = @Schema(hidden = true))), + @ApiResponse( + description = "if the query is not running", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
    " + + "if the next call is interrupted
    " + + "if the query times out
    " + + "if the next task is rejected by the executor
    " + + "if next call execution fails
    " + + "if query logic creation fails
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on @Timed(name = "dw.query.next", absolute = true) @EnrichQueryMetrics(methodType = EnrichQueryMetrics.MethodType.NEXT) @RequestMapping(path = "{queryId}/next", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public BaseQueryResponse next(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public BaseQueryResponse next(@Parameter(description = "The query ID") @PathVariable String queryId, + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.next(queryId, currentUser); } /** * @see QueryManagementService#cancel(String, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Cancels the specified query.", + description = "Cancel can only be called on a running query, or a query that is in the process of closing.
    " + + "Outstanding next calls will be stopped immediately, but will return partial results if applicable.
    " + + "Aside from admins, only the query owner can cancel the specified query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response indicating that the query was canceled", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query is not running", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
    " + + "if the cancel call is interrupted
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on @Timed(name = "dw.query.cancel", absolute = true) @RequestMapping(path = "{queryId}/cancel", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public VoidResponse cancel(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public VoidResponse cancel(@Parameter(description = "The query ID") @PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) + throws QueryException { return queryManagementService.cancel(queryId, currentUser); } /** * @see QueryManagementService#adminCancel(String, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Cancels the specified query using admin privileges.", + description = "Cancel can only be called on a running query, or a query that is in the process of closing.
    " + + "Outstanding next calls will be stopped immediately, but will return partial results if applicable.
    " + + "Only admin users should be allowed to call this method.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response indicating that the query was canceled", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query is not running", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
    " + + "if the cancel call is interrupted
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on @Timed(name = "dw.query.adminCancel", absolute = true) @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "{queryId}/adminCancel", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public VoidResponse adminCancel(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public VoidResponse adminCancel(@Parameter(description = "The query ID") @PathVariable String queryId, + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.adminCancel(queryId, currentUser); } /** * @see QueryManagementService#close(String, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Closes the specified query.", + description = "Close can only be called on a running query.
    " + + "Outstanding next calls will be allowed to run until they can return a full page, or they timeout.
    " + + "Aside from admins, only the query owner can close the specified query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response indicating that the query was closed", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query is not running", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
    " + + "if the close call is interrupted
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on @Timed(name = "dw.query.close", absolute = true) @RequestMapping(path = "{queryId}/close", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public VoidResponse close(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public VoidResponse close(@Parameter(description = "The query ID") @PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) + throws QueryException { return queryManagementService.close(queryId, currentUser); } /** * @see QueryManagementService#adminClose(String, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Closes the specified query using admin privileges.", + description = "Close can only be called on a running query.
    " + + "Outstanding next calls will be allowed to run until they can return a full page, or they timeout.
    " + + "Only admin users should be allowed to call this method.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response indicating that the query was closed", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query is not running", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
    " + + "if the close call is interrupted
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on @Timed(name = "dw.query.adminClose", absolute = true) @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "{queryId}/adminClose", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public VoidResponse adminClose(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public VoidResponse adminClose(@Parameter(description = "The query ID") @PathVariable String queryId, + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.adminClose(queryId, currentUser); } /** * @see QueryManagementService#reset(String, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Stops, and restarts the specified query.", + description = "Reset can be called on any query, whether it's running or not.
    " + + "If the specified query is still running, it will be canceled. See cancel.
    " + + "Reset creates a new, identical query, with a new query id.
    " + + "Reset queries will start running immediately.
    " + + "Auditing is performed before the new query is started.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a generic response containing the new query id", + responseCode = "200", + content = @Content(schema = @Schema(implementation = BaseQueryResponse.class))), + @ApiResponse( + description = "if parameter validation fails
    " + + "if query logic parameter validation fails
    " + + "if security marking validation fails
    " + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query
    " + + "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
    " + + "if the cancel call is interrupted
    " + + "if query storage fails
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on @Timed(name = "dw.query.reset", absolute = true) @RequestMapping(path = "{queryId}/reset", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public GenericResponse reset(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public GenericResponse reset(@Parameter(description = "The query ID") @PathVariable String queryId, + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.reset(queryId, currentUser); } /** * @see QueryManagementService#remove(String, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Removes the specified query from query storage.", + description = "Remove can only be called on a query that is not running.
    " + + "Aside from admins, only the query owner can remove the specified query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response indicating that the query was removed", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query is running", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on @Timed(name = "dw.query.remove", absolute = true) @RequestMapping(path = "{queryId}/remove", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public VoidResponse remove(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public VoidResponse remove(@Parameter(description = "The query ID") @PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) + throws QueryException { return queryManagementService.remove(queryId, currentUser); } /** * @see QueryManagementService#adminRemove(String, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Removes the specified query from query storage using admin privileges.", + description = "Remove can only be called on a query that is not running.
    " + + "Only admin users should be allowed to call this method.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response indicating that the query was removed", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query is running", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on @Timed(name = "dw.query.adminRemove", absolute = true) @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "{queryId}/adminRemove", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public VoidResponse adminRemove(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public VoidResponse adminRemove(@Parameter(description = "The query ID") @PathVariable String queryId, + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.adminRemove(queryId, currentUser); } /** * @see QueryManagementService#update(String, MultiValueMap, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Updates the specified query.", + description = "Update can only be called on a defined, or running query.
    " + + "Auditing is not performed when updating a defined query.
    " + + "No auditable parameters should be updated when updating a running query.
    " + + "Any query parameter can be updated for a defined query.
    " + + "Query parameters which don't affect the scope of the query can be updated for a running query.
    " + + "The list of parameters that can be updated for a running query is configurable.
    " + + "Auditable parameters should never be added to the updatable parameters configuration.
    " + + "Query string, date range, query logic, and auths should never be updated for a running query.
    " + + "Only the query owner can call update on the specified query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a generic response containing the query id", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class))), + @ApiResponse( + description = "if parameter validation fails
    " + + "if query logic parameter validation fails
    " + + "if security marking validation fails
    " + + "if the query is not defined, or running
    " + + " if no parameters are specified", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query
    " + + "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
    " + + "if the update call is interrupted
    " + + "if query storage fails
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_STRING, + in = ParameterIn.QUERY, + description = "The query string", + schema = @Schema(implementation = String.class), + example = "GENRES:[Action to Western]"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_SYNTAX, + in = ParameterIn.QUERY, + description = "The syntax used in the query", + schema = @Schema(implementation = String.class), + example = "LUCENE"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on @Timed(name = "dw.query.update", absolute = true) @RequestMapping(path = "{queryId}/update", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public GenericResponse update(@PathVariable String queryId, @RequestParam MultiValueMap parameters, - @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public GenericResponse update(@Parameter(description = "The query ID") @PathVariable String queryId, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) + throws QueryException { return queryManagementService.update(queryId, parameters, currentUser); } /** * @see QueryManagementService#duplicate(String, MultiValueMap, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Creates a copy of the specified query.", + description = "Duplicate can be called on any query, whether it's running or not.
    " + + "Duplicate creates a new, identical query, with a new query id.
    " + + "Provided parameter updates will be applied to the new query.
    " + + "Duplicated queries will start running immediately.
    " + + "Auditing is performed before the new query is started.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a generic response containing the query id", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class))), + @ApiResponse( + description = "if parameter validation fails
    " + + "if query logic parameter validation fails
    " + + "if security marking validation fails
    " + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query
    " + + "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
    " + + "if query storage fails
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_STRING, + in = ParameterIn.QUERY, + description = "The query string", + schema = @Schema(implementation = String.class), + example = "GENRES:[Action to Western]"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_SYNTAX, + in = ParameterIn.QUERY, + description = "The syntax used in the query", + schema = @Schema(implementation = String.class), + example = "LUCENE"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on @Timed(name = "dw.query.duplicate", absolute = true) @RequestMapping(path = "{queryId}/duplicate", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public GenericResponse duplicate(@PathVariable String queryId, @RequestParam MultiValueMap parameters, - @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public GenericResponse duplicate(@Parameter(description = "The query ID") @PathVariable String queryId, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser) + throws QueryException { return queryManagementService.duplicate(queryId, parameters, currentUser); } /** * @see QueryManagementService#list(String, String, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Gets a list of queries for the calling user.", + description = "Returns all matching queries owned by the calling user, filtering by query id and query name.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a list response containing the matching queries", + responseCode = "200", + content = @Content(schema = @Schema(implementation = QueryImplListResponse.class))), + @ApiResponse( + description = "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on @Timed(name = "dw.query.list", absolute = true) @RequestMapping(path = "list", method = {RequestMethod.GET}, produces = {"text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public QueryImplListResponse list(@RequestParam(required = false) String queryId, @RequestParam(required = false) String queryName, + public QueryImplListResponse list(@Parameter(description = "The query ID") @RequestParam(required = false) String queryId, + @Parameter(description = "The query name") @RequestParam(required = false) String queryName, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.list(queryId, queryName, currentUser); } @@ -358,48 +2205,151 @@ public QueryImplListResponse list(@RequestParam(required = false) String queryId /** * @see QueryManagementService#adminList(String, String, String, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Gets a list of queries for the specified user using admin privileges.", + description = "Returns all matching queries owned by any user, filtered by user ID, query ID, and query name.
    " + + "Only admin users should be allowed to call this method.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a list response containing the matching queries", + responseCode = "200", + content = @Content(schema = @Schema(implementation = QueryImplListResponse.class))), + @ApiResponse( + description = "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on @Timed(name = "dw.query.adminList", absolute = true) @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "adminList", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public QueryImplListResponse adminList(@RequestParam(required = false) String queryId, @RequestParam(required = false) String user, - @RequestParam(required = false) String queryName, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public QueryImplListResponse adminList(@Parameter(description = "The query ID") @RequestParam(required = false) String queryId, + @Parameter(description = "The user id") @RequestParam(required = false) String user, + @Parameter(description = "The query name") @RequestParam(required = false) String queryName, + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.adminList(queryId, queryName, user, currentUser); } /** * @see QueryManagementService#list(String, String, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Gets the matching query for the calling user.", + description = "Returns all matching queries owned by the calling user, filtering by query id.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a list response containing the matching query", + responseCode = "200", + content = @Content(schema = @Schema(implementation = QueryImplListResponse.class))), + @ApiResponse( + description = "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on @Timed(name = "dw.query.get", absolute = true) @RequestMapping(path = "{queryId}", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public QueryImplListResponse get(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public QueryImplListResponse get(@Parameter(description = "The query ID") @PathVariable String queryId, + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.list(queryId, null, currentUser); } /** * @see QueryManagementService#plan(String, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Gets the plan for the given query for the calling user.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns the query plan for the matching query", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
    " + + "if query storage fails
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on @Timed(name = "dw.query.plan", absolute = true) @RequestMapping(path = "{queryId}/plan", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public GenericResponse plan(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public GenericResponse plan(@Parameter(description = "The query ID") @PathVariable String queryId, + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.plan(queryId, currentUser); } /** * @see QueryManagementService#predictions(String, ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Gets the predictions for the given query for the calling user.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns the query predictions for the matching query", + responseCode = "200", + content = @Content(schema = @Schema(implementation = GenericResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
    " + + "if query storage fails
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on @Timed(name = "dw.query.predictions", absolute = true) @RequestMapping(path = "{queryId}/predictions", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public GenericResponse predictions(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { + public GenericResponse predictions(@Parameter(description = "The query ID") @PathVariable String queryId, + @AuthenticationPrincipal ProxiedUserDetails currentUser) throws QueryException { return queryManagementService.predictions(queryId, currentUser); } /** * @see QueryManagementService#adminCancelAll(ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Cancels all queries using admin privileges.", + description = "Cancel can only be called on a running query, or a query that is in the process of closing.
    " + + "Queries that are not running will be ignored by this method.
    " + + "Outstanding next calls will be stopped immediately, but will return partial results if applicable.
    " + + "Only admin users should be allowed to call this method.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response specifying which queries were canceled", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if a query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
    " + + "if the cancel call is interrupted
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on @Timed(name = "dw.query.adminCancelAll", absolute = true) @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "adminCancelAll", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", @@ -411,6 +2361,29 @@ public VoidResponse adminCancelAll(@AuthenticationPrincipal ProxiedUserDetails c /** * @see QueryManagementService#adminCloseAll(ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Closes all queries using admin privileges.", + description = "Close can only be called on a running query.
    " + + "Queries that are not running will be ignored by this method.
    " + + "Outstanding next calls will be allowed to run until they can return a full page, or they timeout.
    " + + "Only admin users should be allowed to call this method.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response specifying which queries were closed", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if a query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
    " + + "if the close call is interrupted
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on @Timed(name = "dw.query.adminCloseAll", absolute = true) @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "adminCloseAll", method = {RequestMethod.PUT, RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", @@ -422,6 +2395,26 @@ public VoidResponse adminCloseAll(@AuthenticationPrincipal ProxiedUserDetails cu /** * @see QueryManagementService#adminRemoveAll(ProxiedUserDetails) */ + // @formatter:off + @Operation( + summary = "Removes all queries from query storage using admin privileges.", + description = "Remove can only be called on a query that is not running.
    " + + "Queries that are running will be ignored by this method.
    " + + "Only admin users should be allowed to call this method.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns a void response specifying which queries were removed", + responseCode = "200", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if a query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on @Timed(name = "dw.query.adminRemoveAll", absolute = true) @Secured({"Administrator", "JBossAdministrator"}) @RequestMapping(path = "adminRemoveAll", method = {RequestMethod.DELETE}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", @@ -433,11 +2426,134 @@ public VoidResponse adminRemoveAll(@AuthenticationPrincipal ProxiedUserDetails c /** * @see StreamingService#createAndExecute(String, MultiValueMap, ProxiedUserDetails, StreamingResponseListener) */ + // @formatter:off + @Operation( + summary = "Creates a query using the given query logic and parameters, and streams back all pages of results.", + description = "Created queries will start running immediately.
    " + + "Auditing is performed before the query is started.
    " + + "Stop a running query gracefully using close or forcefully using cancel.
    " + + "Stop, and restart a running query using reset.
    " + + "Create a copy of a running query using duplicate.
    " + + "Aside from a limited set of admin actions, only the query owner can act on a running query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns multiple base query responses containing pages of results", + responseCode = "200", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = BaseQueryResponse.class)))), + @ApiResponse( + description = "if no query results are found", + responseCode = "204", + content = @Content(schema = @Schema(hidden = true))), + @ApiResponse( + description = "if parameter validation fails
    " + + "if query logic parameter validation fails
    " + + "if security marking validation fails
    " + + "if auditing fails", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't have access to the requested query logic", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query storage fails
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + @Parameters({ + @Parameter( + name = QUERY_BEGIN, + in = ParameterIn.QUERY, + description = "The query begin date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"19660908 000000.000\""), + @Parameter( + name = QUERY_END, + in = ParameterIn.QUERY, + description = "The query end date", + required = true, + schema = @Schema(implementation = String.class), + example = "\"20161002 235959.999\""), + @Parameter( + name = QUERY_NAME, + in = ParameterIn.QUERY, + description = "The query name", + required = true, + schema = @Schema(implementation = String.class), + example = "Developer Test Query"), + @Parameter( + name = QUERY_STRING, + in = ParameterIn.QUERY, + description = "The query string", + required = true, + schema = @Schema(implementation = String.class), + example = "GENRES:[Action to Western]"), + @Parameter( + name = QUERY_AUTHORIZATIONS, + in = ParameterIn.QUERY, + description = "The query auths", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC,PRIVATE,BAR,FOO"), + @Parameter( + name = QUERY_VISIBILITY, + in = ParameterIn.QUERY, + description = "The visibility to use when storing metrics for this query", + required = true, + schema = @Schema(implementation = String.class), + example = "PUBLIC"), + @Parameter( + name = QUERY_SYNTAX, + in = ParameterIn.QUERY, + description = "The syntax used in the query", + schema = @Schema(implementation = String.class), + example = "LUCENE"), + @Parameter( + name = QUERY_MAX_CONCURRENT_TASKS, + in = ParameterIn.QUERY, + description = "The max number of concurrent tasks to run for this query", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_POOL, + in = ParameterIn.QUERY, + description = "The executor pool to run against", + schema = @Schema(implementation = String.class), + example = "pool1"), + @Parameter( + name = QUERY_PAGESIZE, + in = ParameterIn.QUERY, + description = "The requested page size", + schema = @Schema(implementation = Integer.class), + example = "10"), + @Parameter( + name = QUERY_PAGETIMEOUT, + in = ParameterIn.QUERY, + description = "The call timeout when requesting a page, in minutes", + schema = @Schema(implementation = Integer.class), + example = "60"), + @Parameter( + name = QUERY_MAX_RESULTS_OVERRIDE, + in = ParameterIn.QUERY, + description = "The max results override value", + schema = @Schema(implementation = Integer.class), + example = "5000"), + @Parameter( + name = QUERY_PARAMS, + in = ParameterIn.QUERY, + description = "Additional query parameters", + schema = @Schema(implementation = String.class), + example = "KEY_1:VALUE_1;KEY_2:VALUE_2") + }) + // @formatter:on @Timed(name = "dw.query.createAndExecuteQuery", absolute = true) @RequestMapping(path = "{queryLogic}/createAndExecute", method = {RequestMethod.POST}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public ResponseEntity createAndExecute(@PathVariable String queryLogic, @RequestParam MultiValueMap parameters, - @AuthenticationPrincipal ProxiedUserDetails currentUser, @RequestHeader HttpHeaders headers) throws QueryException { + public ResponseEntity createAndExecute( + @Parameter(description = "The query logic", example = "EventQuery") @PathVariable String queryLogic, + @Parameter(hidden = true) @RequestParam MultiValueMap parameters, @AuthenticationPrincipal ProxiedUserDetails currentUser, + @RequestHeader HttpHeaders headers) throws QueryException { MediaType contentType = determineContentType(headers.getAccept(), MediaType.parseMediaType(streamingProperties.getDefaultContentType())); CountingResponseBodyEmitter emitter = baseMethodStatsContext.createCountingResponseBodyEmitter(streamingProperties.getCallTimeoutMillis()); String queryId = streamingService.createAndExecute(queryLogic, parameters, currentUser, new CountingResponseBodyEmitterListener(emitter, contentType)); @@ -452,11 +2568,48 @@ public ResponseEntity createAndExecute(@PathVariable String /** * @see StreamingService#execute(String, ProxiedUserDetails, StreamingResponseListener) */ + // @formatter:off + @Operation( + summary = "Gets all pages of results for the given query and streams them back.", + description = "Execute can only be called on a running query.
    " + + "Only the query owner can call execute on the specified query.") + @ApiResponses({ + @ApiResponse( + description = "if successful, returns multiple base query responses containing pages of results", + responseCode = "200", + content = @Content(array = @ArraySchema(schema = @Schema(implementation = BaseQueryResponse.class)))), + @ApiResponse( + description = "if no query results are found", + responseCode = "204", + content = @Content(schema = @Schema(hidden = true))), + @ApiResponse( + description = "if the query is not running", + responseCode = "400", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the user doesn't own the query", + responseCode = "401", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if the query cannot be found", + responseCode = "404", + content = @Content(schema = @Schema(implementation = VoidResponse.class))), + @ApiResponse( + description = "if query lock acquisition fails
    " + + "if the execute call is interrupted
    " + + "if the query times out
    " + + "if the next task is rejected by the executor
    " + + "if next call execution fails
    " + + "if query logic creation fails
    " + + "if there is an unknown error", + responseCode = "500", + content = @Content(schema = @Schema(implementation = VoidResponse.class)))}) + // @formatter:on @Timed(name = "dw.query.executeQuery", absolute = true) @RequestMapping(path = "{queryId}/execute", method = {RequestMethod.GET}, produces = {"application/xml", "text/xml", "application/json", "text/yaml", "text/x-yaml", "application/x-yaml", "application/x-protobuf", "application/x-protostuff"}) - public ResponseEntity execute(@PathVariable String queryId, @AuthenticationPrincipal ProxiedUserDetails currentUser, - @RequestHeader HttpHeaders headers) { + public ResponseEntity execute(@Parameter(description = "The query ID") @PathVariable String queryId, + @AuthenticationPrincipal ProxiedUserDetails currentUser, @RequestHeader HttpHeaders headers) { MediaType contentType = determineContentType(headers.getAccept(), MediaType.parseMediaType(streamingProperties.getDefaultContentType())); CountingResponseBodyEmitter emitter = baseMethodStatsContext.createCountingResponseBodyEmitter(streamingProperties.getCallTimeoutMillis()); streamingService.execute(queryId, currentUser, new CountingResponseBodyEmitterListener(emitter, contentType)); diff --git a/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/service/src/main/java/datawave/microservice/query/QueryManagementService.java index fb438ec8..0d1e3146 100644 --- a/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -344,16 +344,12 @@ public GenericResponse create(String queryLogicName, MultiValueMap * Created queries will begin planning immediately.
    * Auditing is performed if we are expanding indices.
    * Query plan will be returned in the response.
    * Updates can be made to any parameter which doesn't affect the scope of the query using {@link #update}.
    - * Stop a running query gracefully using {@link #close} or forcefully using {@link #cancel}.
    - * Stop, and restart a running query using {@link #reset}.
    - * Create a copy of a running query using {@link #duplicate}.
    - * Aside from a limited set of admin actions, only the query owner can act on a running query. * * @param queryLogicName * the requested query logic, not null @@ -402,16 +398,12 @@ public GenericResponse plan(String queryLogicName, MultiValueMap * Created queries will begin predicting immediately.
    * Auditing is not performed.
    * Query prediction will be returned in the response.
    * Updates can be made to any parameter which doesn't affect the scope of the query using {@link #update}.
    - * Stop a running query gracefully using {@link #close} or forcefully using {@link #cancel}.
    - * Stop, and restart a running query using {@link #reset}.
    - * Create a copy of a running query using {@link #duplicate}.
    - * Aside from a limited set of admin actions, only the query owner can act on a running query. * * @param queryLogicName * the requested query logic, not null @@ -1199,7 +1191,7 @@ public VoidResponse adminClose(String queryId, ProxiedUserDetails currentUser) t * @throws QueryException * if query lock acquisition fails * @throws QueryException - * if the cancel call is interrupted + * if the close call is interrupted * @throws QueryException * if there is an unknown error */ @@ -1900,7 +1892,7 @@ private QueryImplListResponse list(String queryId, String queryName, String user * Gets the plan for the given query for the calling user. * * @param queryId - * the query id, may be null + * the query id, not null * @param currentUser * the user who called this method, not null * @return the query plan for the matching query @@ -1940,7 +1932,7 @@ public GenericResponse plan(String queryId, ProxiedUserDetails currentUs * Gets the predictions for the given query for the calling user. * * @param queryId - * the query id, may be null + * the query id, not null * @param currentUser * the user who called this method, not null * @return the query predictions for the matching query diff --git a/service/src/main/java/datawave/microservice/query/WebController.java b/service/src/main/java/datawave/microservice/query/WebController.java index f12a73a3..004aa248 100644 --- a/service/src/main/java/datawave/microservice/query/WebController.java +++ b/service/src/main/java/datawave/microservice/query/WebController.java @@ -1,7 +1,6 @@ package datawave.microservice.query; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.info.GitProperties; import org.springframework.stereotype.Controller; import org.springframework.ui.Model; diff --git a/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java b/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java index 0717cc5b..f54141f2 100644 --- a/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java +++ b/service/src/main/java/datawave/microservice/query/config/QueryServiceConfig.java @@ -14,8 +14,6 @@ import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.web.context.annotation.RequestScope; -import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; @Configuration public class QueryServiceConfig { diff --git a/service/src/main/resources/config/application.yml b/service/src/main/resources/config/application.yml index 7e2dc968..813c6cb5 100644 --- a/service/src/main/resources/config/application.yml +++ b/service/src/main/resources/config/application.yml @@ -7,4 +7,4 @@ server: mime-types: "application/javascript, application/json, application/xml, application/x-yaml, application/x-protobuf, application/x-protostuff, text/css, text/html, text/javascript, text/plain, text/xml, text/x-yaml, text/yaml" # response size at which compression kicks in - min-response-size: 4KB + min-response-size: 4KB \ No newline at end of file diff --git a/service/src/main/resources/templates/index.html b/service/src/main/resources/templates/index.html index 1c3c2d0a..8307b865 100644 --- a/service/src/main/resources/templates/index.html +++ b/service/src/main/resources/templates/index.html @@ -19,7 +19,7 @@
    • - API Documents + REST Docs
      • Cache
      • Authorization
        • From d24d46dc3274c6d19fd43e6044d5d562694d68fc Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 17 Jun 2022 12:48:10 -0400 Subject: [PATCH 155/218] Added accumulo docs to index.html --- service/src/main/resources/templates/index.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/service/src/main/resources/templates/index.html b/service/src/main/resources/templates/index.html index 8307b865..ef606363 100644 --- a/service/src/main/resources/templates/index.html +++ b/service/src/main/resources/templates/index.html @@ -21,6 +21,7 @@
      • REST Docs
          +
        • Accumulo
        • Cache
        • Authorization
        • Audit
          • @@ -35,6 +36,7 @@
        • Javadoc
            +
          • Accumulo
          • Cache
          • Authorization
          • Audit
            • From 95d2e31a97004db37386bc2ef94f8265f06b7b19 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 7 Jul 2022 12:21:23 -0400 Subject: [PATCH 156/218] Added scm info for the poms --- api/pom.xml | 7 +++++++ pom.xml | 6 ++++++ service/pom.xml | 7 +++++++ 3 files changed, 20 insertions(+) diff --git a/api/pom.xml b/api/pom.xml index 8626f6c7..fc661d53 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -9,6 +9,13 @@ query-api 1.0-SNAPSHOT + https://github.com/NationalSecurityAgency/datawave-query-service + + scm:git:https://github.com/NationalSecurityAgency/datawave-query-service.git + scm:git:git@github.com:NationalSecurityAgency/datawave-query-service.git + HEAD + https://github.com/NationalSecurityAgency/datawave-query-service + 1.3-SNAPSHOT 2.0.2 diff --git a/pom.xml b/pom.xml index 86f56704..fcac180a 100644 --- a/pom.xml +++ b/pom.xml @@ -10,10 +10,16 @@ query-service-parent 1.0-SNAPSHOT pom + https://github.com/NationalSecurityAgency/datawave-query-service api service + + scm:git:https://github.com/NationalSecurityAgency/datawave-query-service.git + scm:git:git@github.com:NationalSecurityAgency/datawave-query-service.git + https://github.com/NationalSecurityAgency/datawave-query-service + true false - datawave-github-mvn-repo - https://raw.githubusercontent.com/NationalSecurityAgency/datawave/mvn-repo + github-datawave + https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/pom.xml b/pom.xml index 21bffe29..e1b60905 100644 --- a/pom.xml +++ b/pom.xml @@ -21,18 +21,14 @@ - true false - datawave-github-mvn-repo - https://raw.githubusercontent.com/NationalSecurityAgency/datawave/mvn-repo + github-datawave + https://maven.pkg.github.com/NationalSecurityAgency/datawave diff --git a/service/pom.xml b/service/pom.xml index 6270dedc..17e72318 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -228,18 +228,14 @@ - true false - datawave-github-mvn-repo - https://raw.githubusercontent.com/NationalSecurityAgency/datawave/mvn-repo + github-datawave + https://maven.pkg.github.com/NationalSecurityAgency/datawave From 985fa4892ed5ff7fd57bf2b7ff97f99e029bd2fb Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Fri, 26 Apr 2024 21:04:11 +0000 Subject: [PATCH 209/218] PR feedback --- .../java/datawave/microservice/query/lookup/LookupService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/src/main/java/datawave/microservice/query/lookup/LookupService.java b/service/src/main/java/datawave/microservice/query/lookup/LookupService.java index f85da119..4c29a2e2 100644 --- a/service/src/main/java/datawave/microservice/query/lookup/LookupService.java +++ b/service/src/main/java/datawave/microservice/query/lookup/LookupService.java @@ -514,7 +514,7 @@ public String getAuths(MultiValueMap queryParameters, QueryLogic< queryLogic.preInitialize(query, AuthorizationsUtil.buildAuthorizations(currentUser.getAuthorizations())); } - // the query principal is our local principal unless the query logic has a ddifferent user operations + // the query principal is our local principal unless the query logic has a different user operations ProxiedUserDetails queryPrincipal = ((queryLogic.getUserOperations() == null) ? currentUser : queryLogic.getUserOperations().getRemoteUser(currentUser)); From 196a122232f6f2d9c9f71046958f6b65e17fcc06 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Sat, 27 Apr 2024 16:54:59 +0000 Subject: [PATCH 210/218] PR feedback --- .../MapReduceQueryControllerConfig.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/service/src/main/java/datawave/microservice/query/mapreduce/config/MapReduceQueryControllerConfig.java b/service/src/main/java/datawave/microservice/query/mapreduce/config/MapReduceQueryControllerConfig.java index 155c1a9c..ec814f18 100644 --- a/service/src/main/java/datawave/microservice/query/mapreduce/config/MapReduceQueryControllerConfig.java +++ b/service/src/main/java/datawave/microservice/query/mapreduce/config/MapReduceQueryControllerConfig.java @@ -1,15 +1,42 @@ package datawave.microservice.query.mapreduce.config; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.security.config.annotation.web.configuration.WebSecurityCustomizer; +import org.springframework.web.servlet.config.annotation.AsyncSupportConfigurer; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; @Configuration @ConditionalOnProperty(name = MapReduceQueryProperties.PREFIX + ".enabled", havingValue = "true", matchIfMissing = true) public class MapReduceQueryControllerConfig { + @Autowired + MapReduceQueryProperties mapReduceQueryProperties; + @Bean public WebSecurityCustomizer ignoreMapReduceUpdateState() { return (web) -> web.ignoring().antMatchers("/v1/mapreduce/updateState"); } + + @Bean + public ThreadPoolTaskExecutor mvcTaskExecutor() { + ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor(); + taskExecutor.setCorePoolSize(mapReduceQueryProperties.getExecutor().getCorePoolSize()); + taskExecutor.setMaxPoolSize(mapReduceQueryProperties.getExecutor().getMaxPoolSize()); + taskExecutor.setQueueCapacity(mapReduceQueryProperties.getExecutor().getQueueCapacity()); + taskExecutor.setThreadNamePrefix(mapReduceQueryProperties.getExecutor().getThreadNamePrefix()); + return taskExecutor; + } + + @Bean + public WebMvcConfigurer taskExecutorConfiguration() { + return new WebMvcConfigurer() { + @Override + public void configureAsyncSupport(AsyncSupportConfigurer configurer) { + configurer.setTaskExecutor(mvcTaskExecutor()); + } + }; + } } From f805cac94b96ab361c1866d29394d34b61ca51f2 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Sat, 27 Apr 2024 16:58:38 +0000 Subject: [PATCH 211/218] PR Feedback --- .../query/mapreduce/MapReduceQueryController.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/service/src/main/java/datawave/microservice/query/mapreduce/MapReduceQueryController.java b/service/src/main/java/datawave/microservice/query/mapreduce/MapReduceQueryController.java index 0d60c8f5..569f4a0c 100644 --- a/service/src/main/java/datawave/microservice/query/mapreduce/MapReduceQueryController.java +++ b/service/src/main/java/datawave/microservice/query/mapreduce/MapReduceQueryController.java @@ -409,14 +409,6 @@ public ResponseEntity getFile(@Parameter(description = "T // @formatter:on } - // TODO: This endpoint triggers the following warning, which we should resolve - // !!! - // An Executor is required to handle java.util.concurrent.Callable return values. - // Please, configure a TaskExecutor in the MVC config under "async support". - // The SimpleAsyncTaskExecutor currently in use is not suitable under load. - // ------------------------------ - // Request URI: '/query/v1/mapreduce//getAllFiles' - // !!! /** * @see MapReduceQueryManagementService#getAllFiles(String, DatawaveUserDetails) */ From 71a748bae398f50e1740c6481db0e7571556cb5d Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Mon, 29 Apr 2024 16:05:43 +0000 Subject: [PATCH 212/218] PR feedback --- .../config/QueryExpirationProperties.java | 74 +++++++++---------- .../monitor/config/MonitorProperties.java | 24 +++--- .../microservice/query/runner/NextCall.java | 2 +- 3 files changed, 50 insertions(+), 50 deletions(-) diff --git a/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java b/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java index ffd6854e..8e0f4c35 100644 --- a/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java +++ b/api/src/main/java/datawave/microservice/query/config/QueryExpirationProperties.java @@ -12,15 +12,15 @@ public class QueryExpirationProperties { @Positive private long idleTimeout = 15; @NotNull - private TimeUnit idleTimeUnit = TimeUnit.MINUTES; + private TimeUnit idleTimeoutUnit = TimeUnit.MINUTES; @Positive private long progressTimeout = 5; @NotNull - private TimeUnit progressTimeUnit = TimeUnit.MINUTES; + private TimeUnit progressTimeoutUnit = TimeUnit.MINUTES; @Positive private long callTimeout = 60; @NotNull - private TimeUnit callTimeUnit = TimeUnit.MINUTES; + private TimeUnit callTimeoutUnit = TimeUnit.MINUTES; @Positive private long callTimeoutInterval = 1; @NotNull @@ -28,11 +28,11 @@ public class QueryExpirationProperties { @Positive private long pageMinTimeout = 1; @NotNull - private TimeUnit pageMinTimeUnit = TimeUnit.MINUTES; + private TimeUnit pageMinTimeoutUnit = TimeUnit.MINUTES; @Positive private long pageMaxTimeout = 60; @NotNull - private TimeUnit pageMaxTimeUnit = TimeUnit.MINUTES; + private TimeUnit pageMaxTimeoutUnit = TimeUnit.MINUTES; @Positive private long shortCircuitCheckTime = callTimeout / 2; @NotNull @@ -40,14 +40,14 @@ public class QueryExpirationProperties { @Positive private long shortCircuitTimeout = Math.round(0.97 * callTimeout); @NotNull - private TimeUnit shortCircuitTimeUnit = TimeUnit.MINUTES; + private TimeUnit shortCircuitTimeoutUnit = TimeUnit.MINUTES; @Positive @Deprecated // to be replaced by the long running query timeout private int maxLongRunningTimeoutRetries = 3; @Positive - private long longRunningQueryTimeout = (maxLongRunningTimeoutRetries + 1) * callTimeUnit.toMinutes(callTimeout); + private long longRunningQueryTimeout = (maxLongRunningTimeoutRetries + 1) * callTimeoutUnit.toMinutes(callTimeout); @NotNull private TimeUnit longRunningQueryTimeoutUnit = TimeUnit.MINUTES; @@ -57,19 +57,19 @@ public long getIdleTimeout() { } public long getIdleTimeoutMillis() { - return idleTimeUnit.toMillis(idleTimeout); + return idleTimeoutUnit.toMillis(idleTimeout); } public void setIdleTimeout(long idleTimeout) { this.idleTimeout = idleTimeout; } - public TimeUnit getIdleTimeUnit() { - return idleTimeUnit; + public TimeUnit getIdleTimeoutUnit() { + return idleTimeoutUnit; } - public void setIdleTimeUnit(TimeUnit idleTimeUnit) { - this.idleTimeUnit = idleTimeUnit; + public void setIdleTimeoutUnit(TimeUnit idleTimeoutUnit) { + this.idleTimeoutUnit = idleTimeoutUnit; } public long getProgressTimeout() { @@ -77,19 +77,19 @@ public long getProgressTimeout() { } public long getProgressTimeoutMillis() { - return progressTimeUnit.toMillis(progressTimeout); + return progressTimeoutUnit.toMillis(progressTimeout); } public void setProgressTimeout(long progressTimeout) { this.progressTimeout = progressTimeout; } - public TimeUnit getProgressTimeUnit() { - return progressTimeUnit; + public TimeUnit getProgressTimeoutUnit() { + return progressTimeoutUnit; } - public void setProgressTimeUnit(TimeUnit progressTimeUnit) { - this.progressTimeUnit = progressTimeUnit; + public void setProgressTimeoutUnit(TimeUnit progressTimeoutUnit) { + this.progressTimeoutUnit = progressTimeoutUnit; } public long getCallTimeout() { @@ -97,19 +97,19 @@ public long getCallTimeout() { } public long getCallTimeoutMillis() { - return callTimeUnit.toMillis(callTimeout); + return callTimeoutUnit.toMillis(callTimeout); } public void setCallTimeout(long callTimeout) { this.callTimeout = callTimeout; } - public TimeUnit getCallTimeUnit() { - return callTimeUnit; + public TimeUnit getCallTimeoutUnit() { + return callTimeoutUnit; } - public void setCallTimeUnit(TimeUnit callTimeUnit) { - this.callTimeUnit = callTimeUnit; + public void setCallTimeoutUnit(TimeUnit callTimeoutUnit) { + this.callTimeoutUnit = callTimeoutUnit; } public long getCallTimeoutInterval() { @@ -137,19 +137,19 @@ public long getPageMinTimeout() { } public long getPageMinTimeoutMillis() { - return pageMinTimeUnit.toMillis(pageMinTimeout); + return pageMinTimeoutUnit.toMillis(pageMinTimeout); } public void setPageMinTimeout(long pageMinTimeout) { this.pageMinTimeout = pageMinTimeout; } - public TimeUnit getPageMinTimeUnit() { - return pageMinTimeUnit; + public TimeUnit getPageMinTimeoutUnit() { + return pageMinTimeoutUnit; } - public void setPageMinTimeUnit(TimeUnit pageMinTimeUnit) { - this.pageMinTimeUnit = pageMinTimeUnit; + public void setPageMinTimeoutUnit(TimeUnit pageMinTimeoutUnit) { + this.pageMinTimeoutUnit = pageMinTimeoutUnit; } public long getPageMaxTimeout() { @@ -157,19 +157,19 @@ public long getPageMaxTimeout() { } public long getPageMaxTimeoutMillis() { - return pageMaxTimeUnit.toMillis(pageMaxTimeout); + return pageMaxTimeoutUnit.toMillis(pageMaxTimeout); } public void setPageMaxTimeout(long pageMaxTimeout) { this.pageMaxTimeout = pageMaxTimeout; } - public TimeUnit getPageMaxTimeUnit() { - return pageMaxTimeUnit; + public TimeUnit getPageMaxTimeoutUnit() { + return pageMaxTimeoutUnit; } - public void setPageMaxTimeUnit(TimeUnit pageMaxTimeUnit) { - this.pageMaxTimeUnit = pageMaxTimeUnit; + public void setPageMaxTimeoutUnit(TimeUnit pageMaxTimeoutUnit) { + this.pageMaxTimeoutUnit = pageMaxTimeoutUnit; } public long getShortCircuitCheckTime() { @@ -197,19 +197,19 @@ public long getShortCircuitTimeout() { } public long getShortCircuitTimeoutMillis() { - return shortCircuitTimeUnit.toMillis(shortCircuitTimeout); + return shortCircuitTimeoutUnit.toMillis(shortCircuitTimeout); } public void setShortCircuitTimeout(long shortCircuitTimeout) { this.shortCircuitTimeout = shortCircuitTimeout; } - public TimeUnit getShortCircuitTimeUnit() { - return shortCircuitTimeUnit; + public TimeUnit getShortCircuitTimeoutUnit() { + return shortCircuitTimeoutUnit; } - public void setShortCircuitTimeUnit(TimeUnit shortCircuitTimeUnit) { - this.shortCircuitTimeUnit = shortCircuitTimeUnit; + public void setShortCircuitTimeoutUnit(TimeUnit shortCircuitTimeoutUnit) { + this.shortCircuitTimeoutUnit = shortCircuitTimeoutUnit; } public int getMaxLongRunningTimeoutRetries() { diff --git a/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java b/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java index adaa20c5..4fad524e 100644 --- a/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java +++ b/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java @@ -16,7 +16,7 @@ public class MonitorProperties { @PositiveOrZero private long monitorInterval = TimeUnit.MILLISECONDS.toMillis(30); @NotNull - private TimeUnit monitorIntervalTimeUnit = TimeUnit.MILLISECONDS; + private TimeUnit monitorIntervalUnit = TimeUnit.MILLISECONDS; // The amount of time to wait for the monitor lock to be acquired @PositiveOrZero private long lockWaitTime = 0; @@ -31,7 +31,7 @@ public class MonitorProperties { @PositiveOrZero private long inactiveQueryTimeToLive = 1; @NotNull - private TimeUnit inactiveQueryTimeUnit = TimeUnit.DAYS; + private TimeUnit inactiveQueryTimeToLiveUnit = TimeUnit.DAYS; public String getSchedulerCrontab() { return schedulerCrontab; @@ -46,19 +46,19 @@ public long getMonitorInterval() { } public long getMonitorIntervalMillis() { - return monitorIntervalTimeUnit.toMillis(monitorInterval); + return monitorIntervalUnit.toMillis(monitorInterval); } public void setMonitorInterval(long monitorInterval) { this.monitorInterval = monitorInterval; } - public TimeUnit getMonitorIntervalTimeUnit() { - return monitorIntervalTimeUnit; + public TimeUnit getMonitorIntervalUnit() { + return monitorIntervalUnit; } - public void setMonitorIntervalTimeUnit(TimeUnit monitorIntervalTimeUnit) { - this.monitorIntervalTimeUnit = monitorIntervalTimeUnit; + public void setMonitorIntervalUnit(TimeUnit monitorIntervalUnit) { + this.monitorIntervalUnit = monitorIntervalUnit; } public long getLockWaitTime() { @@ -106,18 +106,18 @@ public long getInactiveQueryTimeToLive() { } public long getInactiveQueryTimeToLiveMillis() { - return inactiveQueryTimeUnit.toMillis(inactiveQueryTimeToLive); + return inactiveQueryTimeToLiveUnit.toMillis(inactiveQueryTimeToLive); } public void setInactiveQueryTimeToLive(long inactiveQueryTimeToLive) { this.inactiveQueryTimeToLive = inactiveQueryTimeToLive; } - public TimeUnit getInactiveQueryTimeUnit() { - return inactiveQueryTimeUnit; + public TimeUnit getInactiveQueryTimeToLiveUnit() { + return inactiveQueryTimeToLiveUnit; } - public void setInactiveQueryTimeUnit(TimeUnit inactiveQueryTimeUnit) { - this.inactiveQueryTimeUnit = inactiveQueryTimeUnit; + public void setInactiveQueryTimeToLiveUnit(TimeUnit inactiveQueryTimeToLiveUnit) { + this.inactiveQueryTimeToLiveUnit = inactiveQueryTimeToLiveUnit; } } diff --git a/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/service/src/main/java/datawave/microservice/query/runner/NextCall.java index 56c32cbe..f1c3a8df 100644 --- a/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -127,7 +127,7 @@ public ResultsPage call() throws Exception { // keep waiting for results until we're finished // Note: isFinished should be checked once per result while (!isFinished(queryId)) { - Result result = resultListener.receive(nextCallProperties.getResultPollInterval(), nextCallProperties.getResultPollTimeUnit()); + Result result = resultListener.receive(nextCallProperties.getResultPollInterval(), nextCallProperties.getResultPollIntervalUnit()); if (result != null) { result.acknowledge(ACK); From 3d66b35298bf189cb7bc9e429e6746fa4138a4be Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Mon, 29 Apr 2024 18:22:46 +0000 Subject: [PATCH 213/218] pr feedback --- .../microservice/query/QueryManagementService.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 3bd4f172..5509cb21 100644 --- a/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -32,6 +32,8 @@ import java.util.stream.Collectors; import org.apache.accumulo.core.security.Authorizations; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.apache.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -453,7 +455,7 @@ public GenericResponse predict(String queryLogicName, MultiValueMap predictions = status.getPredictions(); - if (predictions != null && !predictions.isEmpty()) { + if (CollectionUtils.isNotEmpty(predictions)) { queryPrediction = predictions.toString(); } } @@ -1814,7 +1816,7 @@ private boolean updateParameters(Collection parameterNames, MultiValueMa throws BadRequestQueryException { boolean paramsUpdated = false; for (String paramName : parameterNames) { - if (newParameters.get(paramName) != null && !newParameters.get(paramName).isEmpty()) { + if (CollectionUtils.isNotEmpty(newParameters.get(paramName))) { if (!newParameters.get(paramName).get(0).equals(currentParams.getFirst(paramName))) { // if the new value differs from the old value, update the old value currentParams.put(paramName, newParameters.remove(paramName)); @@ -1890,7 +1892,7 @@ public QueryImplListResponse adminList(String queryId, String queryName, String private QueryImplListResponse list(String queryId, String queryName, String userId) throws QueryException { try { List queries; - if (queryId != null && !queryId.isEmpty()) { + if (StringUtils.isNotBlank(queryId)) { // get the query for the given id queries = new ArrayList<>(); @@ -1982,7 +1984,7 @@ public GenericResponse predictions(String queryId, DatawaveUserDetails c QueryStatus queryStatus = validateRequest(queryId, currentUser); GenericResponse response = new GenericResponse<>(); - if (queryStatus.getPredictions() != null && !queryStatus.getPredictions().isEmpty()) { + if (CollectionUtils.isNotEmpty(queryStatus.getPredictions())) { response.setResult(queryStatus.getPredictions().toString()); response.setHasResults(true); } @@ -2149,7 +2151,7 @@ public void audit(String auditId, Auditor.AuditType auditType, String logicName, if (auditType != Auditor.AuditType.NONE) { // audit the query before execution try { - if (selectors != null && !selectors.isEmpty()) { + if (CollectionUtils.isNotEmpty(selectors)) { parameters.put(PrivateAuditConstants.SELECTORS, selectors); } From c7e32eeb19212b6696d4965f52ce780db41b3f51 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 1 May 2024 21:06:30 +0000 Subject: [PATCH 214/218] pr feedback --- service/pom.xml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/service/pom.xml b/service/pom.xml index 17e72318..98b7c465 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -24,6 +24,9 @@ 4.0.0-SNAPSHOT 1.0.0-SNAPSHOT 1.0.0-SNAPSHOT + 6.4.3-1 + 3.3.1-1 + 0.30 @@ -197,17 +200,17 @@ org.webjars foundation - 6.4.3-1 + ${version.webjars.foundation} org.webjars jquery - 3.3.1-1 + ${version.webjars.jquery} org.webjars webjars-locator-core - 0.30 + ${version.webjars.locator-core} gov.nsa.datawave.microservice From eed8243fe7c8de7501ed390ef523f0ca8b7635a7 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Thu, 2 May 2024 19:42:32 +0000 Subject: [PATCH 215/218] pr feedback --- .../microservice/query/QueryController.java | 1 - .../query/QueryManagementService.java | 44 +++++---- .../MapReduceQueryManagementService.java | 4 +- .../microservice/query/runner/NextCall.java | 2 - .../microservice/query/util/QueryUtil.java | 89 ------------------- .../query/web/BaseQueryResponseAdvice.java | 1 - .../query/QueryServiceNextTest.java | 4 +- 7 files changed, 30 insertions(+), 115 deletions(-) delete mode 100644 service/src/main/java/datawave/microservice/query/util/QueryUtil.java diff --git a/service/src/main/java/datawave/microservice/query/QueryController.java b/service/src/main/java/datawave/microservice/query/QueryController.java index aaeafbd9..822f1d23 100644 --- a/service/src/main/java/datawave/microservice/query/QueryController.java +++ b/service/src/main/java/datawave/microservice/query/QueryController.java @@ -1398,7 +1398,6 @@ public BaseQueryResponse translateId(@Parameter(description = "The ID to transla /** * @see TranslateIdService#translateIds(MultiValueMap, String, DatawaveUserDetails) */ - // TODO: Shouldn't the case for this path be the same as the singular call? // @formatter:off @Operation( summary = "Get the ID(s), if any, associated with the specified IDs.", diff --git a/service/src/main/java/datawave/microservice/query/QueryManagementService.java b/service/src/main/java/datawave/microservice/query/QueryManagementService.java index 5509cb21..07b9167f 100644 --- a/service/src/main/java/datawave/microservice/query/QueryManagementService.java +++ b/service/src/main/java/datawave/microservice/query/QueryManagementService.java @@ -53,6 +53,7 @@ import datawave.core.query.cache.ResultsPage; import datawave.core.query.logic.QueryLogic; import datawave.core.query.logic.QueryLogicFactory; +import datawave.core.query.util.QueryUtil; import datawave.marking.SecurityMarking; import datawave.microservice.audit.AuditClient; import datawave.microservice.authorization.federation.FederatedAuthorizationService; @@ -67,7 +68,6 @@ import datawave.microservice.query.storage.QueryStorageCache; import datawave.microservice.query.storage.TaskKey; import datawave.microservice.query.util.QueryStatusUpdateUtil; -import datawave.microservice.query.util.QueryUtil; import datawave.microservice.querymetric.BaseQueryMetric; import datawave.microservice.querymetric.QueryMetricClient; import datawave.microservice.querymetric.QueryMetricType; @@ -514,6 +514,8 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p */ private TaskKey storeQuery(String queryLogicName, MultiValueMap parameters, String pool, DatawaveUserDetails currentUser, QueryStatus.QUERY_STATE queryType, String queryId) throws BadRequestQueryException, QueryException { + long callStartTimeMillis = System.currentTimeMillis(); + // validate query and get a query logic QueryLogic queryLogic = validateQuery(queryLogicName, parameters, currentUser); @@ -573,9 +575,13 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p currentUser, downgradedAuthorizations, getMaxConcurrentTasks(queryLogic)); + + sendRequestAwaitResponse( + QueryRequest.create(taskKey.getQueryId()), + computedPool, + queryProperties.isAwaitExecutorCreateResponse(), + callStartTimeMillis); // @formatter:on - - sendRequestAwaitResponse(QueryRequest.create(taskKey.getQueryId()), computedPool, queryProperties.isAwaitExecutorCreateResponse()); } else if (queryType == PLAN) { // @formatter:off taskKey = queryStorageCache.planQuery( @@ -583,9 +589,13 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p query, currentUser, downgradedAuthorizations); + + sendRequestAwaitResponse( + QueryRequest.plan(taskKey.getQueryId()), + computedPool, + true, + callStartTimeMillis); // @formatter:on - - sendRequestAwaitResponse(QueryRequest.plan(taskKey.getQueryId()), computedPool, true); } else if (queryType == PREDICT) { // @formatter:off taskKey = queryStorageCache.predictQuery( @@ -593,9 +603,13 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p query, currentUser, downgradedAuthorizations); + + sendRequestAwaitResponse( + QueryRequest.predict(taskKey.getQueryId()), + computedPool, + true, + callStartTimeMillis); // @formatter:on - - sendRequestAwaitResponse(QueryRequest.predict(taskKey.getQueryId()), computedPool, true); } if (taskKey == null) { @@ -619,7 +633,7 @@ private TaskKey storeQuery(String queryLogicName, MultiValueMap p } } - private void sendRequestAwaitResponse(QueryRequest request, String computedPool, boolean isAwaitResponse) throws QueryException { + private void sendRequestAwaitResponse(QueryRequest request, String computedPool, boolean isAwaitResponse, long startTimeMillis) throws QueryException { if (isAwaitResponse) { // before publishing the message, create a latch based on the query ID queryLatchMap.put(request.getQueryId(), new CountDownLatch(1)); @@ -629,9 +643,6 @@ private void sendRequestAwaitResponse(QueryRequest request, String computedPool, publishExecutorEvent(request, computedPool); if (isAwaitResponse) { - // TODO: Should we incorporate the call start time into this check? - long startTimeMillis = System.currentTimeMillis(); - log.info("Waiting on query {} response from the executor.", request.getMethod().name()); try { @@ -718,8 +729,6 @@ public BaseQueryResponse createAndNext(String queryLogicName, MultiValueMap QueryUtil.parseParameters(params).stream()) + .forEach(x -> parameters.add(x.getParameterName(), x.getParameterValue())); + // @formatter:on } parameters.remove(AuditParameters.QUERY_SECURITY_MARKING_COLVIZ); diff --git a/service/src/main/java/datawave/microservice/query/mapreduce/MapReduceQueryManagementService.java b/service/src/main/java/datawave/microservice/query/mapreduce/MapReduceQueryManagementService.java index c0772a99..d76ab038 100644 --- a/service/src/main/java/datawave/microservice/query/mapreduce/MapReduceQueryManagementService.java +++ b/service/src/main/java/datawave/microservice/query/mapreduce/MapReduceQueryManagementService.java @@ -256,7 +256,7 @@ protected String submitOozieWorkflow(String workflow, MultiValueMap restart(String id, DatawaveUserDetails currentUser) throws QueryException { log.info("Request: restart from {} for {}", ProxiedEntityUtils.getShortName(currentUser.getPrimaryUser().getName()), id); @@ -584,7 +583,6 @@ public MapReduceQueryStatus validateRequest(String id, DatawaveUserDetails curre if (!query.getOwner().equals(userId)) { throw new UnauthorizedQueryException(DatawaveErrorCode.QUERY_OWNER_MISMATCH, MessageFormat.format("{0} != {1}", userId, query.getOwner())); } - // TODO: JWO: Should we update lastUsedMillis since the user interacted with this query? I think yes... } return mapReduceQueryStatus; diff --git a/service/src/main/java/datawave/microservice/query/runner/NextCall.java b/service/src/main/java/datawave/microservice/query/runner/NextCall.java index f1c3a8df..1e3b2899 100644 --- a/service/src/main/java/datawave/microservice/query/runner/NextCall.java +++ b/service/src/main/java/datawave/microservice/query/runner/NextCall.java @@ -238,8 +238,6 @@ private boolean isFinished(String queryId) throws QueryException { // check to see if the number of results consumed is >= to the number of results generated if (queryResultsRemaining < 0) { - // TODO: Consider this as a potential backend-agnostic exit condition, but be mindful of the possibility of a next call failing before the - // consumed results can be returned log.warn("Query [{}]: The number of results consumed [{}] exceeds the number of results generated [{}]", queryId, queryStatus.getNumResultsConsumed(), queryStatus.getNumResultsGenerated()); } diff --git a/service/src/main/java/datawave/microservice/query/util/QueryUtil.java b/service/src/main/java/datawave/microservice/query/util/QueryUtil.java deleted file mode 100644 index 5e84a34d..00000000 --- a/service/src/main/java/datawave/microservice/query/util/QueryUtil.java +++ /dev/null @@ -1,89 +0,0 @@ -package datawave.microservice.query.util; - -import java.util.Arrays; -import java.util.Set; - -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Mutation; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.security.ColumnVisibility; -import org.apache.hadoop.io.Text; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MultiValueMap; - -import com.google.protobuf.InvalidProtocolBufferException; - -import datawave.microservice.query.Query; -import datawave.microservice.query.QueryImpl; -import io.protostuff.LinkedBuffer; -import io.protostuff.ProtobufIOUtil; -import io.protostuff.Schema; -import io.protostuff.runtime.RuntimeSchema; - -public class QueryUtil { - public static final String PARAMETER_SEPARATOR = ";"; - public static final String PARAMETER_NAME_VALUE_SEPARATOR = ":"; - private static final String NULL_BYTE = "\u0000"; - - public static String getQueryImplClassName(Key key) { - String colq = key.getColumnQualifier().toString(); - String[] parts = colq.split(NULL_BYTE); - if (parts.length == 2) { - return parts[1]; - } else { - // TODO: Need to update the existing Queries table to include the class name in the COLQ. - throw new RuntimeException("Query impl class name not found in colq: " + colq); - } - } - - public static T deserialize(String queryImplClassName, Text columnVisibility, Value value) - throws InvalidProtocolBufferException, ClassNotFoundException { - @SuppressWarnings("unchecked") - Class queryClass = (Class) Class.forName(queryImplClassName); - byte[] b = value.get(); - Schema schema = RuntimeSchema.getSchema(queryClass); - T queryImpl = schema.newMessage(); - ProtobufIOUtil.mergeFrom(b, queryImpl, schema); - queryImpl.setColumnVisibility(columnVisibility.toString()); - return queryImpl; - } - - public static MultiValueMap parseParameters(final String paramsString) { - final MultiValueMap parameters = new LinkedMultiValueMap<>(); - Arrays.stream(paramsString.split(PARAMETER_SEPARATOR)).map(x -> x.split(PARAMETER_NAME_VALUE_SEPARATOR)).filter(x -> x.length == 2) - .forEach(x -> parameters.add(x[0], x[1])); - return parameters; - } - - private static ThreadLocal BUFFER = ThreadLocal.withInitial(() -> LinkedBuffer.allocate(1024)); - - public static Mutation toMutation(T query, ColumnVisibility vis) { - // Store by sid for backwards compatibility - Mutation m = new Mutation(query.getOwner()); - try { - @SuppressWarnings("unchecked") - Schema schema = (Schema) RuntimeSchema.getSchema(query.getClass()); - byte[] bytes = ProtobufIOUtil.toByteArray(query, schema, BUFFER.get()); - m.put(query.getQueryName(), query.getId() + NULL_BYTE + query.getClass().getName(), vis, query.getExpirationDate().getTime(), new Value(bytes)); - return m; - } finally { - BUFFER.get().clear(); - } - } - - public static String toParametersString(final Set parameters) { - final StringBuilder params = new StringBuilder(); - if (null != parameters) { - for (final QueryImpl.Parameter param : parameters) { - if (params.length() > 0) { - params.append(PARAMETER_SEPARATOR); - } - - params.append(param.getParameterName()); - params.append(PARAMETER_NAME_VALUE_SEPARATOR); - params.append(param.getParameterValue()); - } - } - return params.toString(); - } -} diff --git a/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java b/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java index 5e1f03a3..5a25f0f2 100644 --- a/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java +++ b/service/src/main/java/datawave/microservice/query/web/BaseQueryResponseAdvice.java @@ -29,7 +29,6 @@ public boolean supports(@NonNull MethodParameter returnType, @NonNull Class conv public BaseQueryResponse beforeBodyWrite(BaseQueryResponse baseQueryResponse, @NonNull MethodParameter returnType, @NonNull MediaType selectedContentType, @NonNull Class selectedConverterType, @NonNull ServerHttpRequest request, ServerHttpResponse response) { response.getHeaders().add(Constants.PAGE_NUMBER, String.valueOf(baseQueryResponse.getPageNumber())); - // TODO: This isn't the most accurate way of determining whether this is the last page response.getHeaders().add(Constants.IS_LAST_PAGE, String.valueOf(!baseQueryResponse.getHasResults())); response.getHeaders().add(Constants.PARTIAL_RESULTS, String.valueOf(baseQueryResponse.isPartialResults())); return baseQueryResponse; diff --git a/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java b/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java index ce1ef8b0..f26f9bff 100644 --- a/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java +++ b/service/src/test/java/datawave/microservice/query/QueryServiceNextTest.java @@ -149,7 +149,7 @@ public void testNextSuccess_multiplePages() throws Exception { queryStorageCache.updateQueryStatus(queryStatus); for (int page = 1; page <= 2; page++) { - // TODO: We have to generate the results in between next calls because the test queue manager does not handle requeueing of unused messages :( + // NOTE: We have to generate the results in between next calls because the test queue manager does not handle requeueing of unused messages :( // @formatter:off publishEventsToQueue( queryId, @@ -346,7 +346,7 @@ public void testNextSuccess_maxResults() throws Exception { queryStorageCache.updateQueryStatus(queryStatus); for (int page = 1; page <= 4; page++) { - // TODO: We have to generate the results in between next calls because the test queue manager does not handle requeueing of unused messages :( + // NOTE: We have to generate the results in between next calls because the test queue manager does not handle requeueing of unused messages :( // @formatter:off publishEventsToQueue( queryId, From 23743b197b90ae73db92e0915221336674bbf6b1 Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Mon, 6 May 2024 19:39:41 +0000 Subject: [PATCH 216/218] pr feedback --- .../microservice/query/monitor/config/MonitorProperties.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java b/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java index 4fad524e..b8518a17 100644 --- a/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java +++ b/service/src/main/java/datawave/microservice/query/monitor/config/MonitorProperties.java @@ -66,7 +66,7 @@ public long getLockWaitTime() { } public long getLockWaitTimeMillis() { - return lockWaitTime; + return lockWaitTimeUnit.toMillis(lockWaitTime); } public void setLockWaitTime(long lockWaitTime) { From 15754b1cbbfd47335e8801df84b4f42771322deb Mon Sep 17 00:00:00 2001 From: Whitney O'Meara Date: Wed, 8 May 2024 17:34:48 +0000 Subject: [PATCH 217/218] pr feedback --- .../microservice/query/QueryImpl.java | 298 ++++++++++-------- 1 file changed, 158 insertions(+), 140 deletions(-) diff --git a/api/src/main/java/datawave/microservice/query/QueryImpl.java b/api/src/main/java/datawave/microservice/query/QueryImpl.java index 2e32d5dd..b3600803 100644 --- a/api/src/main/java/datawave/microservice/query/QueryImpl.java +++ b/api/src/main/java/datawave/microservice/query/QueryImpl.java @@ -47,10 +47,10 @@ @XmlRootElement(name = "QueryImpl") @XmlAccessorType(XmlAccessType.NONE) public class QueryImpl extends Query implements Serializable, Message { - + public static final String PARAMETER_SEPARATOR = ";"; public static final String PARAMETER_NAME_VALUE_SEPARATOR = ":"; - + public static final String USER_DN = "userDN"; public static final String DN_LIST = "dnList"; public static final String COLUMN_VISIBILITY = "columnVisibility"; @@ -70,40 +70,41 @@ public class QueryImpl extends Query implements Serializable, Message public static final String QUERY_USER_FIELD = "QUERY_USER"; public static final String QUERY_LOGIC_NAME_FIELD = "QUERY_LOGIC_NAME"; public static final String POOL = QueryParameters.QUERY_POOL; - + @XmlAccessorType(XmlAccessType.FIELD) public static final class Parameter implements Serializable, Message { - + private static final long serialVersionUID = 2L; - + @XmlElement(name = "name") private String parameterName; @XmlElement(name = "value") private String parameterValue; - - public Parameter() {} - + + public Parameter() { + } + public Parameter(String name, String value) { this.parameterName = name; this.parameterValue = value; } - + public String getParameterName() { return parameterName; } - + public void setParameterName(String parameterName) { this.parameterName = parameterName; } - + public String getParameterValue() { return parameterValue; } - + public void setParameterValue(String parameterValue) { this.parameterValue = parameterValue; } - + @Override public String toString() { StringBuilder sb = new StringBuilder(256); @@ -111,7 +112,7 @@ public String toString() { sb.append(",value=").append(this.parameterValue).append("]"); return sb.toString(); } - + @Override public boolean equals(Object o) { if (null == o) @@ -126,44 +127,44 @@ public boolean equals(Object o) { else return false; } - + @Override public int hashCode() { return getParameterName() == null ? 0 : getParameterName().hashCode(); } - + @XmlTransient public static final Schema SCHEMA = new Schema() { public Parameter newMessage() { return new Parameter(); } - + public Class typeClass() { return Parameter.class; } - + public String messageName() { return Parameter.class.getSimpleName(); } - + public String messageFullName() { return Parameter.class.getName(); } - + public boolean isInitialized(Parameter message) { return message.parameterName != null && message.parameterValue != null; } - + public void writeTo(Output output, Parameter message) throws IOException { if (message.parameterName == null) throw new UninitializedMessageException(message); output.writeString(1, message.parameterName, false); - + if (message.parameterValue == null) throw new UninitializedMessageException(message); output.writeString(2, message.parameterValue, false); } - + public void mergeFrom(Input input, Parameter message) throws IOException { int number; while ((number = input.readFieldNumber(this)) != 0) { @@ -180,7 +181,7 @@ public void mergeFrom(Input input, Parameter message) throws IOException { } } } - + public String getFieldName(int number) { switch (number) { case 1: @@ -191,32 +192,33 @@ public String getFieldName(int number) { return null; } } - + public int getFieldNumber(String name) { final Integer number = fieldMap.get(name); return number == null ? 0 : number.intValue(); } - - final java.util.HashMap fieldMap = new java.util.HashMap(); + + final java.util.HashMap fieldMap = new java.util.HashMap(); + { fieldMap.put("parameterName", 1); fieldMap.put("parameterValue", 2); } }; - + public static Schema getSchema() { return SCHEMA; } - + @Override public Schema cachedSchema() { return SCHEMA; } - + } - + private static final long serialVersionUID = 2L; - + @XmlElement protected String queryLogicName; @XmlElement @@ -258,114 +260,114 @@ public Schema cachedSchema() { @XmlElement protected String pool; @XmlTransient - protected Map> optionalQueryParameters; - + protected Map> optionalQueryParameters; + protected transient QueryUncaughtExceptionHandler uncaughtExceptionHandler; - - protected transient HashMap paramLookup = new HashMap(); - + + protected transient HashMap paramLookup = new HashMap(); + public String getQueryLogicName() { return queryLogicName; } - + public UUID getId() { if (null == id) return null; return java.util.UUID.fromString(id); } - + public String getQueryName() { return queryName; } - + public String getUserDN() { return userDN; } - + public String getQuery() { return query; } - + public String getQueryAuthorizations() { return queryAuthorizations; } - + public Date getExpirationDate() { return expirationDate; } - + public int getPagesize() { return pagesize; } - + public int getPageTimeout() { return pageTimeout; } - + public String getPool() { return pool; } - + public long getMaxResultsOverride() { return maxResultsOverride; } - + public boolean isMaxResultsOverridden() { return maxResultsOverridden; } - + public void setMaxResultsOverridden(boolean maxResultsOverridden) { this.maxResultsOverridden = maxResultsOverridden; } - + public Set getParameters() { return parameters == null ? null : Collections.unmodifiableSet(parameters); } - + public void setQueryLogicName(String name) { this.queryLogicName = name; } - + public void setId(UUID id) { this.id = id.toString(); } - + public void setQueryName(String queryName) { this.queryName = queryName; } - + public void setUserDN(String userDN) { this.userDN = userDN; } - + public void setQuery(String query) { this.query = query; } - + public void setQueryAuthorizations(String queryAuthorizations) { this.queryAuthorizations = queryAuthorizations; } - + public void setExpirationDate(Date expirationDate) { this.expirationDate = expirationDate; } - + public void setMaxResultsOverride(long maxResults) { this.maxResultsOverride = maxResults; } - + public void setPagesize(int pagesize) { this.pagesize = pagesize; } - + public void setPageTimeout(int pageTimeout) { this.pageTimeout = pageTimeout; } - + public void setPool(String pool) { this.pool = pool; } - + public void setParameters(Set parameters) { this.parameters.clear(); this.parameters.addAll(parameters); @@ -374,78 +376,78 @@ public void setParameters(Set parameters) { this.paramLookup.put(p.getParameterName(), p); } } - + public void addParameter(String key, String val) { Parameter p = new Parameter(key, val); this.parameters.add(p); this.paramLookup.put(p.getParameterName(), p); } - - public void addParameters(Map parameters) { - for (Entry p : parameters.entrySet()) { + + public void addParameters(Map parameters) { + for (Entry p : parameters.entrySet()) { addParameter(p.getKey(), p.getValue()); } } - - public void setParameters(Map parameters) { + + public void setParameters(Map parameters) { HashSet paramObjs = new HashSet(parameters.size()); - for (Entry param : parameters.entrySet()) { + for (Entry param : parameters.entrySet()) { Parameter p = new Parameter(param.getKey(), param.getValue()); paramObjs.add(p); } this.setParameters(paramObjs); } - + public List getDnList() { return dnList; } - + public void setDnList(List dnList) { this.dnList = dnList; } - + public String getColumnVisibility() { return columnVisibility; } - + public void setColumnVisibility(String columnVisibility) { this.columnVisibility = columnVisibility; } - + public Date getBeginDate() { return beginDate; } - + public void setBeginDate(Date beginDate) { this.beginDate = beginDate; } - + public Date getEndDate() { return endDate; } - + public void setEndDate(Date endDate) { this.endDate = endDate; } - + @Override public String getSystemFrom() { return systemFrom; } - + @Override public void setSystemFrom(String systemFrom) { this.systemFrom = systemFrom; } - - public Map> getOptionalQueryParameters() { + + public Map> getOptionalQueryParameters() { return optionalQueryParameters; } - - public void setOptionalQueryParameters(Map> optionalQueryParameters) { + + public void setOptionalQueryParameters(Map> optionalQueryParameters) { this.optionalQueryParameters = optionalQueryParameters; } - + @Override public QueryImpl duplicate(String newQueryName) { QueryImpl query = new QueryImpl(); @@ -470,14 +472,14 @@ public QueryImpl duplicate(String newQueryName) { query.setParameters(new HashSet(this.parameters)); query.setDnList(this.dnList); if (MapUtils.isNotEmpty(this.optionalQueryParameters)) { - Map> optionalDuplicate = new HashMap<>(); + Map> optionalDuplicate = new HashMap<>(); this.optionalQueryParameters.entrySet().stream().forEach(e -> optionalDuplicate.put(e.getKey(), new ArrayList(e.getValue()))); query.setOptionalQueryParameters(optionalDuplicate); } query.setUncaughtExceptionHandler(this.getUncaughtExceptionHandler()); return query; } - + @Override public String toString() { ToStringBuilder tsb = new ToStringBuilder(this); @@ -501,7 +503,7 @@ public String toString() { tsb.append(POOL, this.getPool()); return tsb.toString(); } - + @Override public boolean equals(Object o) { if (this == o) @@ -510,21 +512,21 @@ public boolean equals(Object o) { return false; QueryImpl that = (QueryImpl) o; return pagesize == that.pagesize && pageTimeout == that.pageTimeout && maxResultsOverridden == that.maxResultsOverridden - && maxResultsOverride == that.maxResultsOverride && Objects.equals(queryLogicName, that.queryLogicName) && Objects.equals(id, that.id) - && Objects.equals(queryName, that.queryName) && Objects.equals(userDN, that.userDN) && Objects.equals(query, that.query) - && Objects.equals(beginDate, that.beginDate) && Objects.equals(endDate, that.endDate) - && Objects.equals(queryAuthorizations, that.queryAuthorizations) && Objects.equals(expirationDate, that.expirationDate) - && Objects.equals(parameters, that.parameters) && Objects.equals(dnList, that.dnList) && Objects.equals(owner, that.owner) - && Objects.equals(columnVisibility, that.columnVisibility) && Objects.equals(optionalQueryParameters, that.optionalQueryParameters) - && Objects.equals(systemFrom, that.systemFrom) && Objects.equals(pool, that.pool); + && maxResultsOverride == that.maxResultsOverride && Objects.equals(queryLogicName, that.queryLogicName) && Objects.equals(id, that.id) + && Objects.equals(queryName, that.queryName) && Objects.equals(userDN, that.userDN) && Objects.equals(query, that.query) + && Objects.equals(beginDate, that.beginDate) && Objects.equals(endDate, that.endDate) + && Objects.equals(queryAuthorizations, that.queryAuthorizations) && Objects.equals(expirationDate, that.expirationDate) + && Objects.equals(parameters, that.parameters) && Objects.equals(dnList, that.dnList) && Objects.equals(owner, that.owner) + && Objects.equals(columnVisibility, that.columnVisibility) && Objects.equals(optionalQueryParameters, that.optionalQueryParameters) + && Objects.equals(systemFrom, that.systemFrom) && Objects.equals(pool, that.pool); } - + @Override public int hashCode() { return Objects.hash(queryLogicName, id, queryName, userDN, query, beginDate, endDate, queryAuthorizations, expirationDate, pagesize, pageTimeout, - maxResultsOverridden, maxResultsOverride, parameters, dnList, owner, columnVisibility, optionalQueryParameters, systemFrom, pool); + maxResultsOverridden, maxResultsOverride, parameters, dnList, owner, columnVisibility, optionalQueryParameters, systemFrom, pool); } - + public Parameter findParameter(String parameter) { if (!paramLookup.containsKey(parameter)) { return new Parameter(parameter, ""); @@ -532,99 +534,99 @@ public Parameter findParameter(String parameter) { return paramLookup.get(parameter); } } - + @XmlTransient private static final Schema SCHEMA = new Schema() { public QueryImpl newMessage() { return new QueryImpl(); } - + public Class typeClass() { return QueryImpl.class; } - + public String messageName() { return QueryImpl.class.getSimpleName(); } - + public String messageFullName() { return QueryImpl.class.getName(); } - + public boolean isInitialized(QueryImpl message) { return message.queryLogicName != null && message.id != null && message.userDN != null && message.query != null - && message.queryAuthorizations != null && message.expirationDate != null && message.pagesize > 0 && message.pageTimeout != 0; + && message.queryAuthorizations != null && message.expirationDate != null && message.pagesize > 0 && message.pageTimeout != 0; } - + public void writeTo(Output output, QueryImpl message) throws IOException { if (message.queryLogicName == null) throw new UninitializedMessageException(message, SCHEMA); output.writeString(1, message.queryLogicName, false); - + if (message.id == null) throw new UninitializedMessageException(message, SCHEMA); output.writeString(2, message.id, false); - + if (message.queryName != null) output.writeString(3, message.queryName, false); - + if (message.userDN == null) throw new UninitializedMessageException(message, SCHEMA); output.writeString(4, message.userDN, false); - + if (message.query == null) throw new UninitializedMessageException(message, SCHEMA); output.writeString(5, message.query, false); - + if (message.beginDate != null) output.writeInt64(6, message.beginDate.getTime(), false); - + if (message.endDate != null) output.writeInt64(7, message.endDate.getTime(), false); - + if (message.queryAuthorizations == null) throw new UninitializedMessageException(message, SCHEMA); output.writeString(8, message.queryAuthorizations, false); - + if (message.expirationDate == null) throw new UninitializedMessageException(message, SCHEMA); output.writeInt64(9, message.expirationDate.getTime(), false); - + if (message.pagesize <= 0) throw new UninitializedMessageException(message, SCHEMA); output.writeUInt32(10, message.pagesize, false); - + if (message.parameters != null) { for (Parameter p : message.parameters) { output.writeObject(11, p, Parameter.SCHEMA, true); } } - + if (message.owner == null) throw new UninitializedMessageException(message, SCHEMA); output.writeString(12, message.owner, false); - + if (null != message.dnList) { for (String dn : message.dnList) output.writeString(13, dn, true); } - + if (message.columnVisibility != null) { output.writeString(14, message.columnVisibility, false); } - + if (message.pageTimeout == 0) throw new UninitializedMessageException(message, SCHEMA); output.writeUInt32(15, message.pageTimeout, false); - + if (message.systemFrom != null) output.writeString(16, message.systemFrom, false); - + if (message.pool != null) { output.writeString(17, message.pool, false); } } - + public void mergeFrom(Input input, QueryImpl message) throws IOException { int number; while ((number = input.readFieldNumber(this)) != 0) { @@ -644,7 +646,7 @@ public void mergeFrom(Input input, QueryImpl message) throws IOException { case 5: message.query = input.readString(); break; - + case 6: message.beginDate = new Date(input.readInt64()); break; @@ -690,7 +692,7 @@ public void mergeFrom(Input input, QueryImpl message) throws IOException { } } } - + public String getFieldName(int number) { switch (number) { case 1: @@ -731,13 +733,14 @@ public String getFieldName(int number) { return null; } } - + public int getFieldNumber(String name) { final Integer number = fieldMap.get(name); return number == null ? 0 : number.intValue(); } - - final java.util.HashMap fieldMap = new java.util.HashMap(); + + final java.util.HashMap fieldMap = new java.util.HashMap(); + { fieldMap.put(QUERY_LOGIC_NAME, 1); fieldMap.put(QUERY_ID, 2); @@ -758,16 +761,16 @@ public int getFieldNumber(String name) { fieldMap.put(POOL, 17); } }; - + public QueryUncaughtExceptionHandler getUncaughtExceptionHandler() { return this.uncaughtExceptionHandler; } - + public void setUncaughtExceptionHandler(QueryUncaughtExceptionHandler uncaughtExceptionHandler) { this.uncaughtExceptionHandler = uncaughtExceptionHandler; } - - public void initialize(String userDN, List dnList, String queryLogicName, QueryParameters qp, Map> optionalQueryParameters) { + + public void initialize(String userDN, List dnList, String queryLogicName, QueryParameters qp, Map> optionalQueryParameters) { this.dnList = dnList; this.expirationDate = qp.getExpirationDate(); this.id = java.util.UUID.randomUUID().toString(); @@ -784,19 +787,19 @@ public void initialize(String userDN, List dnList, String queryLogicName this.systemFrom = qp.getSystemFrom(); this.pool = qp.getPool(); if (optionalQueryParameters != null) { - for (Entry> entry : optionalQueryParameters.entrySet()) { + for (Entry> entry : optionalQueryParameters.entrySet()) { if (entry.getValue().get(0) != null) { this.addParameter(entry.getKey(), entry.getValue().get(0)); } } } } - + private static String getCommonName(String dn) { String[] comps = getComponents(dn, "CN"); return comps.length >= 1 ? comps[0] : null; } - + private static String[] getComponents(String dn, String componentName) { componentName = componentName.toUpperCase(); ArrayList components = new ArrayList(); @@ -812,7 +815,7 @@ private static String[] getComponents(String dn, String componentName) { } return components.toArray(new String[0]); } - + public static String getOwner(String dn) { String sid = null; if (dn != null) { @@ -826,18 +829,17 @@ public static String getOwner(String dn) { } return sid; } - + public void setOwner(String owner) { this.owner = owner; } - + public String getOwner() { return this.owner; } - - public Map> toMap() { - // TODO: missing variables uuid and owner -- not going into map - MultiValueMap p = new LinkedMultiValueMap<>(); + + public Map> toMap() { + MultiValueMap p = new LinkedMultiValueMap<>(); if (this.id != null) { p.set(QUERY_ID, this.id); } @@ -868,6 +870,9 @@ public Map> toMap() { if (this.dnList != null) { p.put(DN_LIST, this.dnList); } + if (this.owner != null) { + p.set(OWNER, this.owner); + } if (this.columnVisibility != null) { p.set(COLUMN_VISIBILITY, this.columnVisibility); } @@ -886,6 +891,9 @@ public Map> toMap() { } } p.set(PAGE_TIMEOUT, Integer.toString(this.pageTimeout)); + if (this.isMaxResultsOverridden()) { + p.set(MAX_RESULTS_OVERRIDE,Long.toString(this.maxResultsOverride)); + } if (this.pool != null) { p.set(QueryParameters.QUERY_POOL, this.pool); } @@ -937,6 +945,9 @@ public void readMap(Map> map) throws ParseException { case DN_LIST: setDnList(map.get(key)); break; + case OWNER: + setOwner(map.get(key).get(0)); + break; case COLUMN_VISIBILITY: setColumnVisibility(map.get(key).get(0)); break; @@ -949,6 +960,13 @@ public void readMap(Map> map) throws ParseException { case PAGE_TIMEOUT: setPageTimeout(Integer.parseInt(map.get(key).get(0))); break; + case MAX_RESULTS_OVERRIDE: + String maxResultsOverride = map.get(key).get(0); + if (maxResultsOverride != null) { + setMaxResultsOverridden(true); + setMaxResultsOverride(Long.parseLong(maxResultsOverride)); + } + break; case POOL: setPool(map.get(key).get(0)); default: From 85cd4e8ec57a8e1f9eb5a2870959b08d3b39f058 Mon Sep 17 00:00:00 2001 From: Ivan Bella Date: Wed, 8 May 2024 19:58:45 +0000 Subject: [PATCH 218/218] formatting --- .../microservice/query/QueryImpl.java | 285 +++++++++--------- 1 file changed, 142 insertions(+), 143 deletions(-) diff --git a/api/src/main/java/datawave/microservice/query/QueryImpl.java b/api/src/main/java/datawave/microservice/query/QueryImpl.java index b3600803..3640ce1f 100644 --- a/api/src/main/java/datawave/microservice/query/QueryImpl.java +++ b/api/src/main/java/datawave/microservice/query/QueryImpl.java @@ -47,10 +47,10 @@ @XmlRootElement(name = "QueryImpl") @XmlAccessorType(XmlAccessType.NONE) public class QueryImpl extends Query implements Serializable, Message { - + public static final String PARAMETER_SEPARATOR = ";"; public static final String PARAMETER_NAME_VALUE_SEPARATOR = ":"; - + public static final String USER_DN = "userDN"; public static final String DN_LIST = "dnList"; public static final String COLUMN_VISIBILITY = "columnVisibility"; @@ -70,41 +70,40 @@ public class QueryImpl extends Query implements Serializable, Message public static final String QUERY_USER_FIELD = "QUERY_USER"; public static final String QUERY_LOGIC_NAME_FIELD = "QUERY_LOGIC_NAME"; public static final String POOL = QueryParameters.QUERY_POOL; - + @XmlAccessorType(XmlAccessType.FIELD) public static final class Parameter implements Serializable, Message { - + private static final long serialVersionUID = 2L; - + @XmlElement(name = "name") private String parameterName; @XmlElement(name = "value") private String parameterValue; - - public Parameter() { - } - + + public Parameter() {} + public Parameter(String name, String value) { this.parameterName = name; this.parameterValue = value; } - + public String getParameterName() { return parameterName; } - + public void setParameterName(String parameterName) { this.parameterName = parameterName; } - + public String getParameterValue() { return parameterValue; } - + public void setParameterValue(String parameterValue) { this.parameterValue = parameterValue; } - + @Override public String toString() { StringBuilder sb = new StringBuilder(256); @@ -112,7 +111,7 @@ public String toString() { sb.append(",value=").append(this.parameterValue).append("]"); return sb.toString(); } - + @Override public boolean equals(Object o) { if (null == o) @@ -127,44 +126,44 @@ public boolean equals(Object o) { else return false; } - + @Override public int hashCode() { return getParameterName() == null ? 0 : getParameterName().hashCode(); } - + @XmlTransient public static final Schema SCHEMA = new Schema() { public Parameter newMessage() { return new Parameter(); } - + public Class typeClass() { return Parameter.class; } - + public String messageName() { return Parameter.class.getSimpleName(); } - + public String messageFullName() { return Parameter.class.getName(); } - + public boolean isInitialized(Parameter message) { return message.parameterName != null && message.parameterValue != null; } - + public void writeTo(Output output, Parameter message) throws IOException { if (message.parameterName == null) throw new UninitializedMessageException(message); output.writeString(1, message.parameterName, false); - + if (message.parameterValue == null) throw new UninitializedMessageException(message); output.writeString(2, message.parameterValue, false); } - + public void mergeFrom(Input input, Parameter message) throws IOException { int number; while ((number = input.readFieldNumber(this)) != 0) { @@ -181,7 +180,7 @@ public void mergeFrom(Input input, Parameter message) throws IOException { } } } - + public String getFieldName(int number) { switch (number) { case 1: @@ -192,33 +191,33 @@ public String getFieldName(int number) { return null; } } - + public int getFieldNumber(String name) { final Integer number = fieldMap.get(name); return number == null ? 0 : number.intValue(); } - - final java.util.HashMap fieldMap = new java.util.HashMap(); - + + final java.util.HashMap fieldMap = new java.util.HashMap(); + { fieldMap.put("parameterName", 1); fieldMap.put("parameterValue", 2); } }; - + public static Schema getSchema() { return SCHEMA; } - + @Override public Schema cachedSchema() { return SCHEMA; } - + } - + private static final long serialVersionUID = 2L; - + @XmlElement protected String queryLogicName; @XmlElement @@ -260,114 +259,114 @@ public Schema cachedSchema() { @XmlElement protected String pool; @XmlTransient - protected Map> optionalQueryParameters; - + protected Map> optionalQueryParameters; + protected transient QueryUncaughtExceptionHandler uncaughtExceptionHandler; - - protected transient HashMap paramLookup = new HashMap(); - + + protected transient HashMap paramLookup = new HashMap(); + public String getQueryLogicName() { return queryLogicName; } - + public UUID getId() { if (null == id) return null; return java.util.UUID.fromString(id); } - + public String getQueryName() { return queryName; } - + public String getUserDN() { return userDN; } - + public String getQuery() { return query; } - + public String getQueryAuthorizations() { return queryAuthorizations; } - + public Date getExpirationDate() { return expirationDate; } - + public int getPagesize() { return pagesize; } - + public int getPageTimeout() { return pageTimeout; } - + public String getPool() { return pool; } - + public long getMaxResultsOverride() { return maxResultsOverride; } - + public boolean isMaxResultsOverridden() { return maxResultsOverridden; } - + public void setMaxResultsOverridden(boolean maxResultsOverridden) { this.maxResultsOverridden = maxResultsOverridden; } - + public Set getParameters() { return parameters == null ? null : Collections.unmodifiableSet(parameters); } - + public void setQueryLogicName(String name) { this.queryLogicName = name; } - + public void setId(UUID id) { this.id = id.toString(); } - + public void setQueryName(String queryName) { this.queryName = queryName; } - + public void setUserDN(String userDN) { this.userDN = userDN; } - + public void setQuery(String query) { this.query = query; } - + public void setQueryAuthorizations(String queryAuthorizations) { this.queryAuthorizations = queryAuthorizations; } - + public void setExpirationDate(Date expirationDate) { this.expirationDate = expirationDate; } - + public void setMaxResultsOverride(long maxResults) { this.maxResultsOverride = maxResults; } - + public void setPagesize(int pagesize) { this.pagesize = pagesize; } - + public void setPageTimeout(int pageTimeout) { this.pageTimeout = pageTimeout; } - + public void setPool(String pool) { this.pool = pool; } - + public void setParameters(Set parameters) { this.parameters.clear(); this.parameters.addAll(parameters); @@ -376,78 +375,78 @@ public void setParameters(Set parameters) { this.paramLookup.put(p.getParameterName(), p); } } - + public void addParameter(String key, String val) { Parameter p = new Parameter(key, val); this.parameters.add(p); this.paramLookup.put(p.getParameterName(), p); } - - public void addParameters(Map parameters) { - for (Entry p : parameters.entrySet()) { + + public void addParameters(Map parameters) { + for (Entry p : parameters.entrySet()) { addParameter(p.getKey(), p.getValue()); } } - - public void setParameters(Map parameters) { + + public void setParameters(Map parameters) { HashSet paramObjs = new HashSet(parameters.size()); - for (Entry param : parameters.entrySet()) { + for (Entry param : parameters.entrySet()) { Parameter p = new Parameter(param.getKey(), param.getValue()); paramObjs.add(p); } this.setParameters(paramObjs); } - + public List getDnList() { return dnList; } - + public void setDnList(List dnList) { this.dnList = dnList; } - + public String getColumnVisibility() { return columnVisibility; } - + public void setColumnVisibility(String columnVisibility) { this.columnVisibility = columnVisibility; } - + public Date getBeginDate() { return beginDate; } - + public void setBeginDate(Date beginDate) { this.beginDate = beginDate; } - + public Date getEndDate() { return endDate; } - + public void setEndDate(Date endDate) { this.endDate = endDate; } - + @Override public String getSystemFrom() { return systemFrom; } - + @Override public void setSystemFrom(String systemFrom) { this.systemFrom = systemFrom; } - - public Map> getOptionalQueryParameters() { + + public Map> getOptionalQueryParameters() { return optionalQueryParameters; } - - public void setOptionalQueryParameters(Map> optionalQueryParameters) { + + public void setOptionalQueryParameters(Map> optionalQueryParameters) { this.optionalQueryParameters = optionalQueryParameters; } - + @Override public QueryImpl duplicate(String newQueryName) { QueryImpl query = new QueryImpl(); @@ -472,14 +471,14 @@ public QueryImpl duplicate(String newQueryName) { query.setParameters(new HashSet(this.parameters)); query.setDnList(this.dnList); if (MapUtils.isNotEmpty(this.optionalQueryParameters)) { - Map> optionalDuplicate = new HashMap<>(); + Map> optionalDuplicate = new HashMap<>(); this.optionalQueryParameters.entrySet().stream().forEach(e -> optionalDuplicate.put(e.getKey(), new ArrayList(e.getValue()))); query.setOptionalQueryParameters(optionalDuplicate); } query.setUncaughtExceptionHandler(this.getUncaughtExceptionHandler()); return query; } - + @Override public String toString() { ToStringBuilder tsb = new ToStringBuilder(this); @@ -503,7 +502,7 @@ public String toString() { tsb.append(POOL, this.getPool()); return tsb.toString(); } - + @Override public boolean equals(Object o) { if (this == o) @@ -512,21 +511,21 @@ public boolean equals(Object o) { return false; QueryImpl that = (QueryImpl) o; return pagesize == that.pagesize && pageTimeout == that.pageTimeout && maxResultsOverridden == that.maxResultsOverridden - && maxResultsOverride == that.maxResultsOverride && Objects.equals(queryLogicName, that.queryLogicName) && Objects.equals(id, that.id) - && Objects.equals(queryName, that.queryName) && Objects.equals(userDN, that.userDN) && Objects.equals(query, that.query) - && Objects.equals(beginDate, that.beginDate) && Objects.equals(endDate, that.endDate) - && Objects.equals(queryAuthorizations, that.queryAuthorizations) && Objects.equals(expirationDate, that.expirationDate) - && Objects.equals(parameters, that.parameters) && Objects.equals(dnList, that.dnList) && Objects.equals(owner, that.owner) - && Objects.equals(columnVisibility, that.columnVisibility) && Objects.equals(optionalQueryParameters, that.optionalQueryParameters) - && Objects.equals(systemFrom, that.systemFrom) && Objects.equals(pool, that.pool); + && maxResultsOverride == that.maxResultsOverride && Objects.equals(queryLogicName, that.queryLogicName) && Objects.equals(id, that.id) + && Objects.equals(queryName, that.queryName) && Objects.equals(userDN, that.userDN) && Objects.equals(query, that.query) + && Objects.equals(beginDate, that.beginDate) && Objects.equals(endDate, that.endDate) + && Objects.equals(queryAuthorizations, that.queryAuthorizations) && Objects.equals(expirationDate, that.expirationDate) + && Objects.equals(parameters, that.parameters) && Objects.equals(dnList, that.dnList) && Objects.equals(owner, that.owner) + && Objects.equals(columnVisibility, that.columnVisibility) && Objects.equals(optionalQueryParameters, that.optionalQueryParameters) + && Objects.equals(systemFrom, that.systemFrom) && Objects.equals(pool, that.pool); } - + @Override public int hashCode() { return Objects.hash(queryLogicName, id, queryName, userDN, query, beginDate, endDate, queryAuthorizations, expirationDate, pagesize, pageTimeout, - maxResultsOverridden, maxResultsOverride, parameters, dnList, owner, columnVisibility, optionalQueryParameters, systemFrom, pool); + maxResultsOverridden, maxResultsOverride, parameters, dnList, owner, columnVisibility, optionalQueryParameters, systemFrom, pool); } - + public Parameter findParameter(String parameter) { if (!paramLookup.containsKey(parameter)) { return new Parameter(parameter, ""); @@ -534,99 +533,99 @@ public Parameter findParameter(String parameter) { return paramLookup.get(parameter); } } - + @XmlTransient private static final Schema SCHEMA = new Schema() { public QueryImpl newMessage() { return new QueryImpl(); } - + public Class typeClass() { return QueryImpl.class; } - + public String messageName() { return QueryImpl.class.getSimpleName(); } - + public String messageFullName() { return QueryImpl.class.getName(); } - + public boolean isInitialized(QueryImpl message) { return message.queryLogicName != null && message.id != null && message.userDN != null && message.query != null - && message.queryAuthorizations != null && message.expirationDate != null && message.pagesize > 0 && message.pageTimeout != 0; + && message.queryAuthorizations != null && message.expirationDate != null && message.pagesize > 0 && message.pageTimeout != 0; } - + public void writeTo(Output output, QueryImpl message) throws IOException { if (message.queryLogicName == null) throw new UninitializedMessageException(message, SCHEMA); output.writeString(1, message.queryLogicName, false); - + if (message.id == null) throw new UninitializedMessageException(message, SCHEMA); output.writeString(2, message.id, false); - + if (message.queryName != null) output.writeString(3, message.queryName, false); - + if (message.userDN == null) throw new UninitializedMessageException(message, SCHEMA); output.writeString(4, message.userDN, false); - + if (message.query == null) throw new UninitializedMessageException(message, SCHEMA); output.writeString(5, message.query, false); - + if (message.beginDate != null) output.writeInt64(6, message.beginDate.getTime(), false); - + if (message.endDate != null) output.writeInt64(7, message.endDate.getTime(), false); - + if (message.queryAuthorizations == null) throw new UninitializedMessageException(message, SCHEMA); output.writeString(8, message.queryAuthorizations, false); - + if (message.expirationDate == null) throw new UninitializedMessageException(message, SCHEMA); output.writeInt64(9, message.expirationDate.getTime(), false); - + if (message.pagesize <= 0) throw new UninitializedMessageException(message, SCHEMA); output.writeUInt32(10, message.pagesize, false); - + if (message.parameters != null) { for (Parameter p : message.parameters) { output.writeObject(11, p, Parameter.SCHEMA, true); } } - + if (message.owner == null) throw new UninitializedMessageException(message, SCHEMA); output.writeString(12, message.owner, false); - + if (null != message.dnList) { for (String dn : message.dnList) output.writeString(13, dn, true); } - + if (message.columnVisibility != null) { output.writeString(14, message.columnVisibility, false); } - + if (message.pageTimeout == 0) throw new UninitializedMessageException(message, SCHEMA); output.writeUInt32(15, message.pageTimeout, false); - + if (message.systemFrom != null) output.writeString(16, message.systemFrom, false); - + if (message.pool != null) { output.writeString(17, message.pool, false); } } - + public void mergeFrom(Input input, QueryImpl message) throws IOException { int number; while ((number = input.readFieldNumber(this)) != 0) { @@ -646,7 +645,7 @@ public void mergeFrom(Input input, QueryImpl message) throws IOException { case 5: message.query = input.readString(); break; - + case 6: message.beginDate = new Date(input.readInt64()); break; @@ -692,7 +691,7 @@ public void mergeFrom(Input input, QueryImpl message) throws IOException { } } } - + public String getFieldName(int number) { switch (number) { case 1: @@ -733,14 +732,14 @@ public String getFieldName(int number) { return null; } } - + public int getFieldNumber(String name) { final Integer number = fieldMap.get(name); return number == null ? 0 : number.intValue(); } - - final java.util.HashMap fieldMap = new java.util.HashMap(); - + + final java.util.HashMap fieldMap = new java.util.HashMap(); + { fieldMap.put(QUERY_LOGIC_NAME, 1); fieldMap.put(QUERY_ID, 2); @@ -761,16 +760,16 @@ public int getFieldNumber(String name) { fieldMap.put(POOL, 17); } }; - + public QueryUncaughtExceptionHandler getUncaughtExceptionHandler() { return this.uncaughtExceptionHandler; } - + public void setUncaughtExceptionHandler(QueryUncaughtExceptionHandler uncaughtExceptionHandler) { this.uncaughtExceptionHandler = uncaughtExceptionHandler; } - - public void initialize(String userDN, List dnList, String queryLogicName, QueryParameters qp, Map> optionalQueryParameters) { + + public void initialize(String userDN, List dnList, String queryLogicName, QueryParameters qp, Map> optionalQueryParameters) { this.dnList = dnList; this.expirationDate = qp.getExpirationDate(); this.id = java.util.UUID.randomUUID().toString(); @@ -787,19 +786,19 @@ public void initialize(String userDN, List dnList, String queryLogicName this.systemFrom = qp.getSystemFrom(); this.pool = qp.getPool(); if (optionalQueryParameters != null) { - for (Entry> entry : optionalQueryParameters.entrySet()) { + for (Entry> entry : optionalQueryParameters.entrySet()) { if (entry.getValue().get(0) != null) { this.addParameter(entry.getKey(), entry.getValue().get(0)); } } } } - + private static String getCommonName(String dn) { String[] comps = getComponents(dn, "CN"); return comps.length >= 1 ? comps[0] : null; } - + private static String[] getComponents(String dn, String componentName) { componentName = componentName.toUpperCase(); ArrayList components = new ArrayList(); @@ -815,7 +814,7 @@ private static String[] getComponents(String dn, String componentName) { } return components.toArray(new String[0]); } - + public static String getOwner(String dn) { String sid = null; if (dn != null) { @@ -829,17 +828,17 @@ public static String getOwner(String dn) { } return sid; } - + public void setOwner(String owner) { this.owner = owner; } - + public String getOwner() { return this.owner; } - - public Map> toMap() { - MultiValueMap p = new LinkedMultiValueMap<>(); + + public Map> toMap() { + MultiValueMap p = new LinkedMultiValueMap<>(); if (this.id != null) { p.set(QUERY_ID, this.id); } @@ -892,7 +891,7 @@ public Map> toMap() { } p.set(PAGE_TIMEOUT, Integer.toString(this.pageTimeout)); if (this.isMaxResultsOverridden()) { - p.set(MAX_RESULTS_OVERRIDE,Long.toString(this.maxResultsOverride)); + p.set(MAX_RESULTS_OVERRIDE, Long.toString(this.maxResultsOverride)); } if (this.pool != null) { p.set(QueryParameters.QUERY_POOL, this.pool);