From 3bfbc584511f9ad6ee269f36a61808e678e3240a Mon Sep 17 00:00:00 2001 From: Tomoyuki Morita Date: Thu, 20 Jun 2024 17:20:51 -0700 Subject: [PATCH] Add README to async-query-core Signed-off-by: Tomoyuki Morita --- async-query-core/README.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 async-query-core/README.md diff --git a/async-query-core/README.md b/async-query-core/README.md new file mode 100644 index 0000000000..61b6057269 --- /dev/null +++ b/async-query-core/README.md @@ -0,0 +1,32 @@ +# async-query-core library + +This directory contains async-query-core library, which implements the core logic of async-query and provide extension points to allow plugin different implementation of data storage, etc. +`async-query` module provides implementations for OpenSearch index based implementation. + +## Type of queries +There are following types of queries, and the type is automatically identified by analysing the query. +- BatchQuery: Execute single query in Spark +- InteractiveQuery: Establish session and execute queries in single Spark session +- IndexDMLQuery: Handles DROP/ALTER/VACUUM operation for Flint indices +- RefreshQuery: One time query request to refresh(update) Flint index +- StreamingQuery: Continuously update flint index in single Spark session + +## Extension points +Following is the list of extension points where the consumer of the library needs to provide their own implementation. + +- Data store interface + - [AsyncQueryJobMetadataStorageService](src/main/java/org/opensearch/sql/spark/asyncquery/AsyncQueryJobMetadataStorageService.java) + - [SessionStorageService](java/org/opensearch/sql/spark/execution/statestore/SessionStorageService.java) + - [StatementStorageService](src/main/java/org/opensearch/sql/spark/execution/statestore/StatementStorageService.java) + - [FlintIndexMetadataService](src/main/java/org/opensearch/sql/spark/flint/FlintIndexMetadataService.java) + - [FlintIndexStateModelService](src/main/java/org/opensearch/sql/spark/flint/FlintIndexStateModelService.java) + - [IndexDMLResultStorageService](src/main/java/org/opensearch/sql/spark/flint/IndexDMLResultStorageService.java) +- Other + - [LeaseManager](src/main/java/org/opensearch/sql/spark/leasemanager/LeaseManager.java) + - [JobExecutionResponseReader](src/main/java/org/opensearch/sql/spark/response/JobExecutionResponseReader.java) + - [QueryIdProvider](src/main/java/org/opensearch/sql/spark/dispatcher/QueryIdProvider.java) + - [SessionIdProvider](src/main/java/org/opensearch/sql/spark/execution/session/SessionIdProvider.java) + - [SessionConfigSupplier](src/main/java/org/opensearch/sql/spark/execution/session/SessionConfigSupplier.java) + - [SparkExecutionEngineConfigSupplier](src/main/java/org/opensearch/sql/spark/config/SparkExecutionEngineConfigSupplier.java) + - [SparkSubmitParameterModifier](src/main/java/org/opensearch/sql/spark/config/SparkSubmitParameterModifier.java) + - [EMRServerlessClientFactory](src/main/java/org/opensearch/sql/spark/client/EMRServerlessClientFactory.java)