Skip to content

Commit

Permalink
Enable session by default (#2373) (#2374)
Browse files Browse the repository at this point in the history
* enable session by default



* update doc



---------


(cherry picked from commit ff38081)

Signed-off-by: Peng Huo <penghuo@gmail.com>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
  • Loading branch information
1 parent b7c7506 commit 5d47786
Show file tree
Hide file tree
Showing 4 changed files with 48 additions and 9 deletions.
8 changes: 4 additions & 4 deletions docs/user/admin/settings.rst
Original file line number Diff line number Diff line change
Expand Up @@ -317,9 +317,9 @@ plugins.query.executionengine.spark.session.enabled
Description
-----------

By default, execution engine is executed in job mode. You can enable session mode by this setting.
By default, execution engine is executed in session mode. You can disable session mode by this setting.

1. The default value is false.
1. The default value is true.
2. This setting is node scope.
3. This setting can be updated dynamically.

Expand All @@ -328,7 +328,7 @@ You can update the setting with a new value like this.
SQL query::

sh$ curl -sS -H 'Content-Type: application/json' -X PUT localhost:9200/_plugins/_query/settings \
... -d '{"transient":{"plugins.query.executionengine.spark.session.enabled":"true"}}'
... -d '{"transient":{"plugins.query.executionengine.spark.session.enabled":"false"}}'
{
"acknowledged": true,
"persistent": {},
Expand All @@ -338,7 +338,7 @@ SQL query::
"executionengine": {
"spark": {
"session": {
"enabled": "true"
"enabled": "false"
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ public class OpenSearchSettings extends Settings {
public static final Setting<?> SPARK_EXECUTION_SESSION_ENABLED_SETTING =
Setting.boolSetting(
Key.SPARK_EXECUTION_SESSION_ENABLED.getKeyValue(),
false,
true,
Setting.Property.NodeScope,
Setting.Property.Dynamic);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,9 @@ public class StateStore {
public static String SETTINGS_FILE_NAME = "query_execution_request_settings.yml";
public static String MAPPING_FILE_NAME = "query_execution_request_mapping.yml";
public static Function<String, String> DATASOURCE_TO_REQUEST_INDEX =
datasourceName -> String.format("%s_%s", SPARK_REQUEST_BUFFER_INDEX_NAME, datasourceName);
datasourceName ->
String.format(
"%s_%s", SPARK_REQUEST_BUFFER_INDEX_NAME, datasourceName.toLowerCase(Locale.ROOT));

private static final Logger LOG = LogManager.getLogger();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import lombok.Getter;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.opensearch.action.search.SearchRequest;
import org.opensearch.action.search.SearchResponse;
Expand Down Expand Up @@ -211,9 +212,6 @@ public void withSessionCreateAsyncQueryThenGetResultThenCancel() {
AsyncQueryExecutorService asyncQueryExecutorService =
createAsyncQueryExecutorService(emrsClient);

// enable session
enableSession(true);

// 1. create async query.
CreateAsyncQueryResponse response =
asyncQueryExecutorService.createAsyncQuery(
Expand Down Expand Up @@ -313,6 +311,9 @@ public void interactiveQueryNoTimeout() {
assertEquals(0L, (long) emrsClient.getJobRequest().executionTimeout());
}

@Ignore(
"flaky test, java.lang.IllegalArgumentException: Right now only AES/GCM/NoPadding is"
+ " supported")
@Test
public void datasourceWithBasicAuth() {
Map<String, String> properties = new HashMap<>();
Expand Down Expand Up @@ -480,6 +481,42 @@ public void submitQueryInInvalidSessionThrowException() {
assertEquals("no session found. " + sessionId, exception.getMessage());
}

@Test
public void datasourceNameIncludeUppercase() {
dataSourceService.createDataSource(
new DataSourceMetadata(
"TESTS3",
DataSourceType.S3GLUE,
ImmutableList.of(),
ImmutableMap.of(
"glue.auth.type",
"iam_role",
"glue.auth.role_arn",
"arn:aws:iam::924196221507:role/FlintOpensearchServiceRole",
"glue.indexstore.opensearch.uri",
"http://localhost:9200",
"glue.indexstore.opensearch.auth",
"noauth"),
null));

LocalEMRSClient emrsClient = new LocalEMRSClient();
AsyncQueryExecutorService asyncQueryExecutorService =
createAsyncQueryExecutorService(emrsClient);

// enable session
enableSession(true);

CreateAsyncQueryResponse response =
asyncQueryExecutorService.createAsyncQuery(
new CreateAsyncQueryRequest("select 1", "TESTS3", LangType.SQL, null));
String params = emrsClient.getJobRequest().getSparkSubmitParams();

assertNotNull(response.getSessionId());
assertTrue(
params.contains(
"--conf spark.sql.catalog.TESTS3=org.opensearch.sql.FlintDelegatingSessionCatalog"));
}

private DataSourceServiceImpl createDataSourceService() {
String masterKey = "a57d991d9b573f75b9bba1df";
DataSourceMetadataStorage dataSourceMetadataStorage =
Expand Down

0 comments on commit 5d47786

Please sign in to comment.