From ba5cbf4eb7a8e1c9ea17bab685ff2e7739c51f69 Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Wed, 3 Jun 2020 17:20:33 +0800 Subject: [PATCH 1/8] linkis version, 0.9.3 to 0.9.4 --- assembly/pom.xml | 2 +- assembly/public-module/pom.xml | 2 +- bml/bml-engine-hook/pom.xml | 2 +- bml/bmlclient/pom.xml | 2 +- bml/bmlcommon/pom.xml | 2 +- bml/bmlserver/pom.xml | 2 +- conf/config.sh | 2 +- core/cloudModule/pom.xml | 2 +- core/cloudMybatis/pom.xml | 2 +- core/cloudProtocol/pom.xml | 2 +- core/cloudRPC/pom.xml | 2 +- core/common/pom.xml | 2 +- core/hadoop-common/pom.xml | 2 +- core/httpclient/pom.xml | 2 +- core/scheduler/pom.xml | 2 +- datasource/datasourcemanager/common/pom.xml | 2 +- datasource/metadatamanager/common/pom.xml | 2 +- .../metadatamanager/elasticsearch/pom.xml | 2 +- .../metadatamanager/mysql/service/pom.xml | 2 +- .../metadatamanager/service/hive/pom.xml | 2 +- datasourcemanager/server/pom.xml | 2 +- ...ith_a_new_computation_or_storage_engine.md | 172 ++--- docs/en_US/ch3/Linkis User Manual.md | 710 +++++++++--------- docs/en_US/ch3/Linkis_Java_SDK_doc.md | 314 ++++---- ...77\347\224\250\346\226\207\346\241\243.md" | 2 +- ...30\345\202\250\345\274\225\346\223\216.md" | 6 +- ...77\347\224\250\346\226\207\346\241\243.md" | 6 +- eurekaServer/pom.xml | 2 +- extensions/spark-excel/pom.xml | 2 +- gateway/core/pom.xml | 2 +- gateway/gateway-httpclient-support/pom.xml | 2 +- gateway/gateway-ujes-support/pom.xml | 2 +- gateway/springcloudgateway/pom.xml | 2 +- metadata/pom.xml | 2 +- pom.xml | 10 +- publicService/application/pom.xml | 4 +- publicService/configuration/pom.xml | 2 +- publicService/jobhistory/pom.xml | 2 +- publicService/pom.xml | 2 +- publicService/udf/pom.xml | 2 +- publicService/variable/pom.xml | 2 +- .../client/workspace-httpclient/pom.xml | 2 +- publicService/workspace/pom.xml | 2 +- resourceManager/resourcemanagerclient/pom.xml | 2 +- resourceManager/resourcemanagercommon/pom.xml | 2 +- resourceManager/resourcemanagerserver/pom.xml | 2 +- storage/pesIO/io-engine/pom.xml | 2 +- storage/pesIO/io-enginemanager/pom.xml | 2 +- storage/pesIO/io-entrance/pom.xml | 2 +- storage/storage/pom.xml | 2 +- ujes/client/pom.xml | 2 +- ujes/definedEngines/hive/engine/pom.xml | 2 +- .../definedEngines/hive/enginemanager/pom.xml | 2 +- ujes/definedEngines/hive/entrance/pom.xml | 2 +- ujes/definedEngines/jdbc/entrance/pom.xml | 4 +- ujes/definedEngines/mlsql/entrance/pom.xml | 2 +- ujes/definedEngines/pipeline/engine/pom.xml | 2 +- .../pipeline/enginemanager/pom.xml | 2 +- ujes/definedEngines/pipeline/entrance/pom.xml | 2 +- ujes/definedEngines/python/engine/pom.xml | 2 +- .../python/enginemanager/pom.xml | 6 +- ujes/definedEngines/python/entrance/pom.xml | 2 +- ujes/definedEngines/shell/engine/pom.xml | 2 +- .../shell/enginemanager/pom.xml | 2 +- ujes/definedEngines/shell/entrance/pom.xml | 2 +- ujes/definedEngines/spark/engine/pom.xml | 2 +- .../spark/enginemanager/pom.xml | 2 +- ujes/definedEngines/spark/entrance/pom.xml | 2 +- ujes/definedEngines/tispark/engine/pom.xml | 2 +- ujes/engine/pom.xml | 2 +- ujes/enginemanager/pom.xml | 2 +- ujes/entrance/pom.xml | 2 +- ujes/entranceclient/pom.xml | 2 +- ujes/jdbc/pom.xml | 4 +- userControl/pom.xml | 2 +- 75 files changed, 683 insertions(+), 683 deletions(-) diff --git a/assembly/pom.xml b/assembly/pom.xml index b9170de816..11c9ae554c 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,7 +21,7 @@ com.webank.wedatasphere.linkis linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/assembly/public-module/pom.xml b/assembly/public-module/pom.xml index ba3a041fe2..5cced71f20 100644 --- a/assembly/public-module/pom.xml +++ b/assembly/public-module/pom.xml @@ -5,7 +5,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/bml/bml-engine-hook/pom.xml b/bml/bml-engine-hook/pom.xml index 9b525ffaaa..3808605c76 100644 --- a/bml/bml-engine-hook/pom.xml +++ b/bml/bml-engine-hook/pom.xml @@ -5,7 +5,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/bml/bmlclient/pom.xml b/bml/bmlclient/pom.xml index cb188a61f9..5ea5c5c448 100644 --- a/bml/bmlclient/pom.xml +++ b/bml/bmlclient/pom.xml @@ -5,7 +5,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/bml/bmlcommon/pom.xml b/bml/bmlcommon/pom.xml index 7aca4b27ce..7b0fa54abe 100644 --- a/bml/bmlcommon/pom.xml +++ b/bml/bmlcommon/pom.xml @@ -5,7 +5,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/bml/bmlserver/pom.xml b/bml/bmlserver/pom.xml index aff1dcc076..ad028b13a8 100644 --- a/bml/bmlserver/pom.xml +++ b/bml/bmlserver/pom.xml @@ -5,7 +5,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/conf/config.sh b/conf/config.sh index 79e7848f10..c707d8e496 100644 --- a/conf/config.sh +++ b/conf/config.sh @@ -119,4 +119,4 @@ BML_PORT=9113 ## java application default jvm memory export SERVER_HEAP_SIZE="512M" -LINKIS_VERSION=0.9.3 +LINKIS_VERSION=0.9.4 diff --git a/core/cloudModule/pom.xml b/core/cloudModule/pom.xml index 576289b485..b8f9df12e8 100644 --- a/core/cloudModule/pom.xml +++ b/core/cloudModule/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/core/cloudMybatis/pom.xml b/core/cloudMybatis/pom.xml index 3b917bdc70..700ba3c83d 100644 --- a/core/cloudMybatis/pom.xml +++ b/core/cloudMybatis/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 jar diff --git a/core/cloudProtocol/pom.xml b/core/cloudProtocol/pom.xml index da11c8d427..724286aed7 100644 --- a/core/cloudProtocol/pom.xml +++ b/core/cloudProtocol/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/core/cloudRPC/pom.xml b/core/cloudRPC/pom.xml index e381f961f4..1e4467798f 100644 --- a/core/cloudRPC/pom.xml +++ b/core/cloudRPC/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/core/common/pom.xml b/core/common/pom.xml index c4c7998239..3523a51312 100644 --- a/core/common/pom.xml +++ b/core/common/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/core/hadoop-common/pom.xml b/core/hadoop-common/pom.xml index a691d8d99c..ef84036e1e 100644 --- a/core/hadoop-common/pom.xml +++ b/core/hadoop-common/pom.xml @@ -5,7 +5,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/core/httpclient/pom.xml b/core/httpclient/pom.xml index 6b501b1736..51df03ddf3 100644 --- a/core/httpclient/pom.xml +++ b/core/httpclient/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/core/scheduler/pom.xml b/core/scheduler/pom.xml index 9ec1a9f73e..487de51072 100644 --- a/core/scheduler/pom.xml +++ b/core/scheduler/pom.xml @@ -22,7 +22,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-scheduler diff --git a/datasource/datasourcemanager/common/pom.xml b/datasource/datasourcemanager/common/pom.xml index cf21b26366..34841cd53a 100644 --- a/datasource/datasourcemanager/common/pom.xml +++ b/datasource/datasourcemanager/common/pom.xml @@ -18,7 +18,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/datasource/metadatamanager/common/pom.xml b/datasource/metadatamanager/common/pom.xml index 7f0c5e981e..2d3755fcdb 100644 --- a/datasource/metadatamanager/common/pom.xml +++ b/datasource/metadatamanager/common/pom.xml @@ -18,7 +18,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/datasource/metadatamanager/elasticsearch/pom.xml b/datasource/metadatamanager/elasticsearch/pom.xml index 4727619725..21d5c0759c 100644 --- a/datasource/metadatamanager/elasticsearch/pom.xml +++ b/datasource/metadatamanager/elasticsearch/pom.xml @@ -18,7 +18,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/datasource/metadatamanager/mysql/service/pom.xml b/datasource/metadatamanager/mysql/service/pom.xml index bb3cba0677..65acc14528 100644 --- a/datasource/metadatamanager/mysql/service/pom.xml +++ b/datasource/metadatamanager/mysql/service/pom.xml @@ -18,7 +18,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/datasource/metadatamanager/service/hive/pom.xml b/datasource/metadatamanager/service/hive/pom.xml index 3f28e117f6..1f92b4514e 100644 --- a/datasource/metadatamanager/service/hive/pom.xml +++ b/datasource/metadatamanager/service/hive/pom.xml @@ -18,7 +18,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 linkis-metadatamanager-service-hive diff --git a/datasourcemanager/server/pom.xml b/datasourcemanager/server/pom.xml index ea7dff386e..a20439d415 100644 --- a/datasourcemanager/server/pom.xml +++ b/datasourcemanager/server/pom.xml @@ -18,7 +18,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/docs/en_US/ch3/How_to_adapt_Linkis_with_a_new_computation_or_storage_engine.md b/docs/en_US/ch3/How_to_adapt_Linkis_with_a_new_computation_or_storage_engine.md index d7b679a769..ed67b027b6 100644 --- a/docs/en_US/ch3/How_to_adapt_Linkis_with_a_new_computation_or_storage_engine.md +++ b/docs/en_US/ch3/How_to_adapt_Linkis_with_a_new_computation_or_storage_engine.md @@ -1,86 +1,86 @@ -# How to adapt Linkis with a new computation or storage engine - -## 1. Introduction - -Except using the engines developed by Linkis directly, backend developers can also develop their own applications based on their requirements. -Divided into Entrance, EngineManager and Engine modules, one can easily split an application to adapt to Linkis. -The purpose and archetecture of these three modules please refer to Linkis Archetect Design Docs[UJES架构设计文档](../ch4/Linkis-UJES设计文档.md). - - -## 2. Declaration - -Linkis uses Spring framework as the underlying technique. So the Spring development specs must be obeyed. - -Linkis has an elastic underlying achitecture and provides common implementations for almost all of its top-level interfaces. If customized classes are needed by users, they can be directly injected and replace the current implementations. - - -### 2.1 Entrance module adaption - -**1) maven dependency** - -```xml - - com.webank.wedatasphere.Linkis - Linkis-ujes-entrance - 0.9.3 - -``` - -**2)Interfaces to be implemented** - -There is no compulsary interface in Entrance. Below interfaces can be implemented on demand. -- EntranceParser. Used to parse request maps from frontend to a persistable Task. Class AbstractEntranceParser is already provided and only parseToTask method needs to be overrided. Linkis provides CommonEntranceParser as the default implementation. -- EngineRequester. Used to build a RequestEngine object, which can be used to request a new engine from the EngineManager. -- Scheduler. Used to schedule tasks. The default implementation provides parallel mode for multi-user situations and FIFO mode for single user pattern. It is not suggested to be customized without special purposes. - -### 2.2 EngineManager module adaption - -**1) maven dependency** - -```xml - - com.webank.wedatasphere.Linkis - Linkis-ujes-enginemanager - 0.9.3 - -``` - -**2)Interfaces to be implemented** - -Below interfaces are required to be implemented in EngineManager: -- EngineCreator. Method createProcessEngineBuilder needs to be overridden in the existing AbstractEngineCreator to create an EngineBuilder. -Here ProcessEngineBuilder has already provided a class called JavaProcessEngineBuilder, which is an abstract class accomplishes configurations of classpath, JavaOpts, GC file path and log path, and opening DEBUG port in test mode. To implement JavaProcessEngineBuilder, only extra classpath and JavaOpts are needed to be specified. -- EngineResourceFactory. Method getRequestResource needs to be overridden in the existing AbstractEngineResourceFactory to declare user customized resource requirements. -- resources. A Spring bean used to register resources to RM. Users need to specify an instance of ModuleInfo for dependency injection. - -Below interfaces/beans are optional in EngineManager: -- hooks. A Spring bean used to add pre and post hooks around the Engine startup procedure. Users need to declare an Spring bean in type EngineHook[] hooks to make new hooks effective. For details please refer to com.webank.wedatasphere.linkis.enginemanager.impl.EngineManagerSpringConfiguration. - - -### 2.3 Engine module adaption - -**1) maven dependency** - -```xml - - com.webank.wedatasphere.Linkis - Linkis-ujes-engine - 0.9.3 - -``` - - -**2)Interfaces to be implemented** - -Below interfaces are required to be implemented in Engine： -- EngineExecutorFactory. Used to build an EngineExecutor from a Map by implementing method createExecutor. This map contains evironment variables and engine arguments. -- EngineExecutor. The actual executor to execute the code submitted from the entrance. - Methods need to be implemented: - 1. getActualUsedResources(the resource an engine acually used) - 2. executeLine(execute a line of the code parsed by CodeParser) - 3. executeCompletely(the suplementary method for executeLine. If executeLine returns ExecuteIncomplete, new code will be submitted with the previous code together to the engine) - -Below interfaces/beans are optional in Engine: -- engineHooks: Array[EngineHook], a Spring bean used to add pre and post hooks around the Engine startup procedure. Currently the system provides 2 hooks: CodeGeneratorEngineHook for UDF/Function loading and ReleaseEngineHook for releasing spare engines. The system registers engineHooks=Array(ReleaseEngineHook) only by default. -- CodeParser. Used to parse code into lines and submit one line only for each execution loop. The system registers a CodeParser returns all the code at once by default. -- EngineParser. Used to convert a RequestTask to a Job that is acceptable by Scheduler. If not specified, the system registers an EngineParser that converts RequestTask to CommonEngineJob. +# How to adapt Linkis with a new computation or storage engine + +## 1. Introduction + +Except using the engines developed by Linkis directly, backend developers can also develop their own applications based on their requirements. +Divided into Entrance, EngineManager and Engine modules, one can easily split an application to adapt to Linkis. +The purpose and archetecture of these three modules please refer to Linkis Archetect Design Docs[UJES架构设计文档](../ch4/Linkis-UJES设计文档.md). + + +## 2. Declaration + +Linkis uses Spring framework as the underlying technique. So the Spring development specs must be obeyed. + +Linkis has an elastic underlying achitecture and provides common implementations for almost all of its top-level interfaces. If customized classes are needed by users, they can be directly injected and replace the current implementations. + + +### 2.1 Entrance module adaption + +**1) maven dependency** + +```xml + + com.webank.wedatasphere.Linkis + Linkis-ujes-entrance + 0.9.4 + +``` + +**2)Interfaces to be implemented** + +There is no compulsary interface in Entrance. Below interfaces can be implemented on demand. +- EntranceParser. Used to parse request maps from frontend to a persistable Task. Class AbstractEntranceParser is already provided and only parseToTask method needs to be overrided. Linkis provides CommonEntranceParser as the default implementation. +- EngineRequester. Used to build a RequestEngine object, which can be used to request a new engine from the EngineManager. +- Scheduler. Used to schedule tasks. The default implementation provides parallel mode for multi-user situations and FIFO mode for single user pattern. It is not suggested to be customized without special purposes. + +### 2.2 EngineManager module adaption + +**1) maven dependency** + +```xml + + com.webank.wedatasphere.Linkis + Linkis-ujes-enginemanager + 0.9.4 + +``` + +**2)Interfaces to be implemented** + +Below interfaces are required to be implemented in EngineManager: +- EngineCreator. Method createProcessEngineBuilder needs to be overridden in the existing AbstractEngineCreator to create an EngineBuilder. +Here ProcessEngineBuilder has already provided a class called JavaProcessEngineBuilder, which is an abstract class accomplishes configurations of classpath, JavaOpts, GC file path and log path, and opening DEBUG port in test mode. To implement JavaProcessEngineBuilder, only extra classpath and JavaOpts are needed to be specified. +- EngineResourceFactory. Method getRequestResource needs to be overridden in the existing AbstractEngineResourceFactory to declare user customized resource requirements. +- resources. A Spring bean used to register resources to RM. Users need to specify an instance of ModuleInfo for dependency injection. + +Below interfaces/beans are optional in EngineManager: +- hooks. A Spring bean used to add pre and post hooks around the Engine startup procedure. Users need to declare an Spring bean in type EngineHook[] hooks to make new hooks effective. For details please refer to com.webank.wedatasphere.linkis.enginemanager.impl.EngineManagerSpringConfiguration. + + +### 2.3 Engine module adaption + +**1) maven dependency** + +```xml + + com.webank.wedatasphere.Linkis + Linkis-ujes-engine + 0.9.4 + +``` + + +**2)Interfaces to be implemented** + +Below interfaces are required to be implemented in Engine： +- EngineExecutorFactory. Used to build an EngineExecutor from a Map by implementing method createExecutor. This map contains evironment variables and engine arguments. +- EngineExecutor. The actual executor to execute the code submitted from the entrance. + Methods need to be implemented: + 1. getActualUsedResources(the resource an engine acually used) + 2. executeLine(execute a line of the code parsed by CodeParser) + 3. executeCompletely(the suplementary method for executeLine. If executeLine returns ExecuteIncomplete, new code will be submitted with the previous code together to the engine) + +Below interfaces/beans are optional in Engine: +- engineHooks: Array[EngineHook], a Spring bean used to add pre and post hooks around the Engine startup procedure. Currently the system provides 2 hooks: CodeGeneratorEngineHook for UDF/Function loading and ReleaseEngineHook for releasing spare engines. The system registers engineHooks=Array(ReleaseEngineHook) only by default. +- CodeParser. Used to parse code into lines and submit one line only for each execution loop. The system registers a CodeParser returns all the code at once by default. +- EngineParser. Used to convert a RequestTask to a Job that is acceptable by Scheduler. If not specified, the system registers an EngineParser that converts RequestTask to CommonEngineJob. diff --git a/docs/en_US/ch3/Linkis User Manual.md b/docs/en_US/ch3/Linkis User Manual.md index 6ab6c1853d..c5fac81444 100644 --- a/docs/en_US/ch3/Linkis User Manual.md +++ b/docs/en_US/ch3/Linkis User Manual.md @@ -1,356 +1,356 @@ -## Linkis User Manual - -#### 1.Summary -        Designed by Webank independently, Linkis is an extensible framework and a sophisticated solution for big data task submission. Conveniently, it could be used directly together with Scriptest, which is another open-source project powered by Webank. And those frontend APIs are also available for users. A client implementation is also provided as an SDK to interact directly with background services. As an highly extensible framework, users can leverage the SDK to develop their own applications. - -#### 2.Frontend Adaption -        Two protocols are supported for frontend APIs: HTTP and WebSocket. Compared with HTTP, Websocket is more friendly to servers and behaved more efficiently in message pushing. But WebSocket is unstable and users are easily to be disconnected by accident. So Scriptest combined both ways to adapt with Linkis. It communicates with Linkis by websocket in normal circumstances, and failover to HTTP protocol in case the Websocket connection was down. -##### 2.1 API Specs -Linkis has its own specs for front-backend adaption.
- -**1).URL specs** -``` -/api/rest_j/v1/{applicationName}/.+ -/api/rest_s/v1/{applicationName}/.+ -``` - -- rest_j means the API is conformed to Jersey standards -- rest_s means the API is conformed to springMVC Rest standards -- v1 is the version of services，**The version will be upgraded with Linkis releases** -- {applicationName} is the microservice name - -**2).Request specs** -```json -{ - "method":"/api/rest_j/v1/entrance/execute", - "data":{}, - "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" -} -``` - -**3).Response specs** -```json -{"method":"/api/rest_j/v1/entrance/execute","status":0, "message":Success！","data":{}} -``` -- method：Return the Restful API URL requested, basically used by websocket protocol。 -- status：Return the status info, in which -1 means login failed, 0 means succeeded, 1 means error, 2 mean validation failed, and 3 means no permission. -- data：Return detailed data. -- message：Return hint message of the request. If the status is not 0, this message returns error messages. At the same time 'data' may return the stack information in its 'stack' column. - - -##### 2.2WebSocket API Description - - -**1).Establish connection**
-
-Used to establish a WebSocket connection with Linkis. -- API `/api/rest_j/entrance/connect` -- HTTP Method **GET** -- Status Code **101**
- -**2).Request execution**
-
-Used to submit user jobs to Linkis for execution. -- API `/api/rest_j/entrance/execute` -- HTTP Method `POST`
-- Sample Json request body -```json -{ - "method":"/api/rest_j/v1/entrance/execute", - "data":{ - "params": { - "variable":{ - "k1":"v1" - }, - "configuration":{ - "special":{ - "k2":"v2" - }, - "runtime":{ - "k3":"v3" - }, - "startup":{ - "k4":"v4" - } - } - }, - "executeApplicationName":"spark", - "executionCode":"show tables", - "runType":"sql", - "source":{ - "scriptPath": "/home//linkis.sql" - }, - "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" - } -} -``` -- Descriptions for parameters of request body data
- -| Parameter Name | Parameter Definition | Type | Comments | -| ------------ | ------------ | ------------ | ------------ | -| executeApplicationName | The Engine service expected by the user, such as Spark or hive| String | Not null | -| requestApplicationName | The name of the system launching this request | String | Nullable | -| params | User-defined parameters to run services | Map | Required, but values are nullable | -| executionCode | The execution code submitted by the user | String |Not null | -| runType | Assuming that the user executes a spark job, he may choose python, R or SQL as runType| String | Not null | -| scriptPath | The script path of the execution code | String | For Scriptest, it shouldn't be null with executionCode at the same time | - Table 1 Descriptions for the parameters - - -- Sample Json response body -```json -{ - "method": "/api/rest_j/v1/entrance/execute", - "status": 0, - "message": "Execution request succeeded", - "data": { - "execID": "030418IDEhivebdpdwc010004:10087IDE_johnnwang_21", - "taskID": "123" - } -} -``` -- execID is a unique ID of String type generated for each user task after submitted to Linkis. It is only used during the execution period, like PID. The format of execID is (length of requestApplicationName)(length of executeAppName)(length of Instance)${requestApplicationName}${executeApplicationName}${entranceInstance infomation ip+port}${requestApplicationName}_${umUser}_${index} -- taskID is a unique ID of Long type genenrated incrementally by the database for each task. - -**3).The push mechanism for task status, logs and progress**
- -After submission, the status, logs and progress information will be pushed by the server. They could be retrieved by websocket protocol. -The API is consistent with HTTP protocol as mentioned below. The only difference is that the schema of websocket is ws://, but http:// for HTTP protocol. - -Sample response of WebSocket API -- Logs -```json -{ - "method": "/api/rest_j/v1/entrance/${execID}/log", - "status": 0, - "message": "Returned log information", - "data": { - "execID": "${execID}", - "log": ["errorLog","warnLog","infoLog", "allLog"], - "taskID":28594, - "fromLine": 56 - }, - "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" -} -``` -- Status -```json -{ - "method": "/api/rest_j/v1/entrance/${execID}/status", - "status": 0, - "message": "Return status information", - "data": { - "execID": "${execID}", - "taskID":28594, - "status": "Running", - }, - "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" -} -``` -- Progress -```json -{ - "method": "/api/rest_j/v1/entrance/${execID}/log", - "status": 0, - "message": "Return progress information", - "data": { - "execID": "${execID}", - "taskID":28594, - "progress": 0.2, - "progressInfo": [ - { - "id": "job-1", - "succeedTasks": 2, - "failedTasks": 0, - "runningTasks": 5, - "totalTasks": 10 - }, - { - "id": "job-2", - "succeedTasks": 5, - "failedTasks": 0, - "runningTasks": 5, - "totalTasks": 10 - } - ] - }, - "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" -} -``` - - - -##### 2.3HTTP API description -For HTTP API, polling should be used to retrieve the status, logs and progress information after submission. - - -**1).Request execution** - -- API `/api/rest_j/entrance/execute` -- HTTP Method `POST`
-- Sample JSON request body, its description is the same with Table 1 -```json -{ - "params": {}, - "executeApplicationName":"spark", - "executionCode":"show tables", - "runType":"sql", - "source":{ - "scriptPath": "/home/linkis/linkis.sql" - } -} -``` -- The response is consistent with websocket. Both execId and taskId will be obtained. - -**2).Retrieve status**
-
-- API `/api/rest_j/entrance/${execID}/status` -- HTTP Method `GET`
-- Sample response body -```json -{ - "method": "/api/rest_j/v1/entrance/{execID}/status", - "status": 0, - "message": "Succeeded to retrieve status", - "data": { - "execID": "${execID}", - "status": "Running" - } -} -``` - -**3).Retrieve logs**
-
-- API `/api/rest_j/entrance/${execID}/log?fromLine=${fromLine}&size=${size}` -- HTTP Method `GET` -- Parameter fromLine specifies from which line to start. Parameter size specifies the number of lines should be retrieved for this request. -- Sample response body, the returned fromLine indicates the value of parameter fromLine for next request. -```json -{ - "method": "/api/rest_j/v1/entrance/${execID}/log", - "status": 0, - "message": "Return logs information", - "data": { - "execID": "${execID}", - "log": ["errorLogs","warnLogs","infoLogs", "allLogs], - "fromLine": 56 - } -} -``` - -**4).Retrieve progress**
-
-- API `/api/rest_j/entrance/${execID}/progress` -- HTTP Method `GET`
-- Sample response body -```json -{ - "method": "/api/rest_j/v1/entrance/{execID}/progress", - "status": 0, - "message": "Return progress information", - "data": { - "execID": "${execID}", - "progress": 0.2, - "progressInfo": [ - { - "id": "job-1", - "succeedTasks": 2, - "failedTasks": 0, - "runningTasks": 5, - "totalTasks": 10 - }, - { - "id": "job-2", - "succeedTasks": 5, - "failedTasks": 0, - "runningTasks": 5, - "totalTasks": 10 - } - ] - } -} -``` -**5).kill task**
-
-- API `/api/rest_j/entrance/${execID}/kill` -- HTTP Method `POST` -- Sample response body -```json -{ - "method": "/api/rest_j/v1/entrance/{execID}/kill", - "status": 0, - "message": "OK", - "data": { - "execID":"${execID}" - } -} -``` - - - -### 3.Client SDK Adaption -[Please see Linkis Quick Start](/docs/en_US/ch2/Linkis%20Quick%20Start.md) - - - -#### 4. Multiple engine type support -Except using the engines developed by Linkis directly, backend developers can also develop their own applications based on their requirements. Divided into Entrance, EngineManager and Engine modules, one can easily split an application to adapt to Linkis. The purpose and architecture of these three modules please refer to Linkis Architect Design Docs. - -#### Convention - -Linkis uses Spring framework as the underlying technique. So instances of some classes can be injected using Spring annotations. Linkis provides some default common implementations. If customized classes are needed by users, they can be directly injected and replace the current implementations. - -##### 4.1Entrance module adaption -**1)maven dependency** -```xml - - com.webank.wedatasphere.linkis - linkis-ujes-entrance - 0.9.3 - -``` -**2)Interfaces to be implemented** - -There is no compulsory interface in Entrance. Below interfaces can be implemented on demand. -- EntranceParser. Used to parse request maps from frontend to a persistable Task. Class AbstractEntranceParser is already provided and only parseToTask method needs to be overridden. Linkis provides CommonEntranceParser as the default implementation. -- EngineRequester. Used to build a RequestEngine object, which can be used to request a new engine from the EngineManager. -- Scheduler. Used to schedule tasks. The default implementation provides parallel mode for multi-user situations and FIFO mode for single user pattern. It is not suggested to be customized without special purposes. - -##### 4.2EngineManager module adaption -**1)maven dependency** -```xml - - com.webank.wedatasphere.linkis - linkis-ujes-enginemanager - 0.9.3 - -``` - -**2)Interfaces to be implemented** - -Below interfaces are required to be implemented in EngineManager: -- EngineCreator. Method createProcessEngineBuilder needs to be overridden in the existing AbstractEngineCreator to create an EngineBuilder. -Here ProcessEngineBuilder has already provided a class called JavaProcessEngineBuilder, which is an abstract class accomplishes configurations of classpath, JavaOpts, GC file path and log path, and opening DEBUG port in test mode. To implement JavaProcessEngineBuilder, only extra classpath and JavaOpts are needed to be specified. -- EngineResourceFactory. Method getRequestResource needs to be overridden in the existing AbstractEngineResourceFactory to declare user customized resource requirements. -- hooks. A Spring bean used to add pre and post hooks around the Engine startup procedure. Users need to specify an Array[EngineHook] for dependency injection. -- resources. A Spring bean used to register resources to RM. Users need to specify an instance of ModuleInfo for dependency injection. - -##### 4.3Engine module adaption -**1)maven dependency** -```xml - - com.webank.wedatasphere.linkis - linkis-ujes-engine - 0.9.3 - -``` -**2)Interfaces to be implemented** - -Below interfaces are required to be implemented in Engine： -- EngineExecutorFactory. Used to build an EngineExecutor from a Map by implementing method createExecutor. -- EngineExecutor. The actual executor to execute the code submitted from the entrance. Methods need to be implemented: getActualUsedResources(the resource an engine acually used), executeLine(execute a line of the code parsed by CodeParser), executeCompletely(the suplementary method for executeLine. If executeLine returns ExecuteIncomplete, new code will be submitted with the previous code together to the engine) - -Below interfaces/beans are optional in Engine: -- engineHooks: Array[EngineHook], a Spring bean used to add pre and post hooks around the Engine startup procedure. Currently the system provides 2 hooks: CodeGeneratorEngineHook for UDF/Function loading and ReleaseEngineHook for releasing spare engines. The system registers engineHooks=Array(ReleaseEngineHook) only by default. -- CodeParser. Used to parse code into lines and submit one line only for each execution loop. The system registers a CodeParser returns all the code at once by default. +## Linkis User Manual + +#### 1.Summary +        Designed by Webank independently, Linkis is an extensible framework and a sophisticated solution for big data task submission. Conveniently, it could be used directly together with Scriptest, which is another open-source project powered by Webank. And those frontend APIs are also available for users. A client implementation is also provided as an SDK to interact directly with background services. As an highly extensible framework, users can leverage the SDK to develop their own applications. + +#### 2.Frontend Adaption +        Two protocols are supported for frontend APIs: HTTP and WebSocket. Compared with HTTP, Websocket is more friendly to servers and behaved more efficiently in message pushing. But WebSocket is unstable and users are easily to be disconnected by accident. So Scriptest combined both ways to adapt with Linkis. It communicates with Linkis by websocket in normal circumstances, and failover to HTTP protocol in case the Websocket connection was down. +##### 2.1 API Specs +Linkis has its own specs for front-backend adaption.
+ +**1).URL specs** +``` +/api/rest_j/v1/{applicationName}/.+ +/api/rest_s/v1/{applicationName}/.+ +``` + +- rest_j means the API is conformed to Jersey standards +- rest_s means the API is conformed to springMVC Rest standards +- v1 is the version of services，**The version will be upgraded with Linkis releases** +- {applicationName} is the microservice name + +**2).Request specs** +```json +{ + "method":"/api/rest_j/v1/entrance/execute", + "data":{}, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` + +**3).Response specs** +```json +{"method":"/api/rest_j/v1/entrance/execute","status":0, "message":Success！","data":{}} +``` +- method：Return the Restful API URL requested, basically used by websocket protocol。 +- status：Return the status info, in which -1 means login failed, 0 means succeeded, 1 means error, 2 mean validation failed, and 3 means no permission. +- data：Return detailed data. +- message：Return hint message of the request. If the status is not 0, this message returns error messages. At the same time 'data' may return the stack information in its 'stack' column. + + +##### 2.2WebSocket API Description + + +**1).Establish connection**
+
+Used to establish a WebSocket connection with Linkis. +- API `/api/rest_j/entrance/connect` +- HTTP Method **GET** +- Status Code **101**
+ +**2).Request execution**
+
+Used to submit user jobs to Linkis for execution. +- API `/api/rest_j/entrance/execute` +- HTTP Method `POST`
+- Sample Json request body +```json +{ + "method":"/api/rest_j/v1/entrance/execute", + "data":{ + "params": { + "variable":{ + "k1":"v1" + }, + "configuration":{ + "special":{ + "k2":"v2" + }, + "runtime":{ + "k3":"v3" + }, + "startup":{ + "k4":"v4" + } + } + }, + "executeApplicationName":"spark", + "executionCode":"show tables", + "runType":"sql", + "source":{ + "scriptPath": "/home//linkis.sql" + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" + } +} +``` +- Descriptions for parameters of request body data
+ +| Parameter Name | Parameter Definition | Type | Comments | +| ------------ | ------------ | ------------ | ------------ | +| executeApplicationName | The Engine service expected by the user, such as Spark or hive| String | Not null | +| requestApplicationName | The name of the system launching this request | String | Nullable | +| params | User-defined parameters to run services | Map | Required, but values are nullable | +| executionCode | The execution code submitted by the user | String |Not null | +| runType | Assuming that the user executes a spark job, he may choose python, R or SQL as runType| String | Not null | +| scriptPath | The script path of the execution code | String | For Scriptest, it shouldn't be null with executionCode at the same time | + Table 1 Descriptions for the parameters + + +- Sample Json response body +```json +{ + "method": "/api/rest_j/v1/entrance/execute", + "status": 0, + "message": "Execution request succeeded", + "data": { + "execID": "030418IDEhivebdpdwc010004:10087IDE_johnnwang_21", + "taskID": "123" + } +} +``` +- execID is a unique ID of String type generated for each user task after submitted to Linkis. It is only used during the execution period, like PID. The format of execID is (length of requestApplicationName)(length of executeAppName)(length of Instance)${requestApplicationName}${executeApplicationName}${entranceInstance infomation ip+port}${requestApplicationName}_${umUser}_${index} +- taskID is a unique ID of Long type genenrated incrementally by the database for each task. + +**3).The push mechanism for task status, logs and progress**
+ +After submission, the status, logs and progress information will be pushed by the server. They could be retrieved by websocket protocol. +The API is consistent with HTTP protocol as mentioned below. The only difference is that the schema of websocket is ws://, but http:// for HTTP protocol. + +Sample response of WebSocket API +- Logs +```json +{ + "method": "/api/rest_j/v1/entrance/${execID}/log", + "status": 0, + "message": "Returned log information", + "data": { + "execID": "${execID}", + "log": ["errorLog","warnLog","infoLog", "allLog"], + "taskID":28594, + "fromLine": 56 + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` +- Status +```json +{ + "method": "/api/rest_j/v1/entrance/${execID}/status", + "status": 0, + "message": "Return status information", + "data": { + "execID": "${execID}", + "taskID":28594, + "status": "Running", + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` +- Progress +```json +{ + "method": "/api/rest_j/v1/entrance/${execID}/log", + "status": 0, + "message": "Return progress information", + "data": { + "execID": "${execID}", + "taskID":28594, + "progress": 0.2, + "progressInfo": [ + { + "id": "job-1", + "succeedTasks": 2, + "failedTasks": 0, + "runningTasks": 5, + "totalTasks": 10 + }, + { + "id": "job-2", + "succeedTasks": 5, + "failedTasks": 0, + "runningTasks": 5, + "totalTasks": 10 + } + ] + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` + + + +##### 2.3HTTP API description +For HTTP API, polling should be used to retrieve the status, logs and progress information after submission. + + +**1).Request execution** + +- API `/api/rest_j/entrance/execute` +- HTTP Method `POST`
+- Sample JSON request body, its description is the same with Table 1 +```json +{ + "params": {}, + "executeApplicationName":"spark", + "executionCode":"show tables", + "runType":"sql", + "source":{ + "scriptPath": "/home/linkis/linkis.sql" + } +} +``` +- The response is consistent with websocket. Both execId and taskId will be obtained. + +**2).Retrieve status**
+
+- API `/api/rest_j/entrance/${execID}/status` +- HTTP Method `GET`
+- Sample response body +```json +{ + "method": "/api/rest_j/v1/entrance/{execID}/status", + "status": 0, + "message": "Succeeded to retrieve status", + "data": { + "execID": "${execID}", + "status": "Running" + } +} +``` + +**3).Retrieve logs**
+
+- API `/api/rest_j/entrance/${execID}/log?fromLine=${fromLine}&size=${size}` +- HTTP Method `GET` +- Parameter fromLine specifies from which line to start. Parameter size specifies the number of lines should be retrieved for this request. +- Sample response body, the returned fromLine indicates the value of parameter fromLine for next request. +```json +{ + "method": "/api/rest_j/v1/entrance/${execID}/log", + "status": 0, + "message": "Return logs information", + "data": { + "execID": "${execID}", + "log": ["errorLogs","warnLogs","infoLogs", "allLogs], + "fromLine": 56 + } +} +``` + +**4).Retrieve progress**
+
+- API `/api/rest_j/entrance/${execID}/progress` +- HTTP Method `GET`
+- Sample response body +```json +{ + "method": "/api/rest_j/v1/entrance/{execID}/progress", + "status": 0, + "message": "Return progress information", + "data": { + "execID": "${execID}", + "progress": 0.2, + "progressInfo": [ + { + "id": "job-1", + "succeedTasks": 2, + "failedTasks": 0, + "runningTasks": 5, + "totalTasks": 10 + }, + { + "id": "job-2", + "succeedTasks": 5, + "failedTasks": 0, + "runningTasks": 5, + "totalTasks": 10 + } + ] + } +} +``` +**5).kill task**
+
+- API `/api/rest_j/entrance/${execID}/kill` +- HTTP Method `POST` +- Sample response body +```json +{ + "method": "/api/rest_j/v1/entrance/{execID}/kill", + "status": 0, + "message": "OK", + "data": { + "execID":"${execID}" + } +} +``` + + + +### 3.Client SDK Adaption +[Please see Linkis Quick Start](/docs/en_US/ch2/Linkis%20Quick%20Start.md) + + + +#### 4. Multiple engine type support +Except using the engines developed by Linkis directly, backend developers can also develop their own applications based on their requirements. Divided into Entrance, EngineManager and Engine modules, one can easily split an application to adapt to Linkis. The purpose and architecture of these three modules please refer to Linkis Architect Design Docs. + +#### Convention + +Linkis uses Spring framework as the underlying technique. So instances of some classes can be injected using Spring annotations. Linkis provides some default common implementations. If customized classes are needed by users, they can be directly injected and replace the current implementations. + +##### 4.1Entrance module adaption +**1)maven dependency** +```xml + + com.webank.wedatasphere.linkis + linkis-ujes-entrance + 0.9.4 + +``` +**2)Interfaces to be implemented** + +There is no compulsory interface in Entrance. Below interfaces can be implemented on demand. +- EntranceParser. Used to parse request maps from frontend to a persistable Task. Class AbstractEntranceParser is already provided and only parseToTask method needs to be overridden. Linkis provides CommonEntranceParser as the default implementation. +- EngineRequester. Used to build a RequestEngine object, which can be used to request a new engine from the EngineManager. +- Scheduler. Used to schedule tasks. The default implementation provides parallel mode for multi-user situations and FIFO mode for single user pattern. It is not suggested to be customized without special purposes. + +##### 4.2EngineManager module adaption +**1)maven dependency** +```xml + + com.webank.wedatasphere.linkis + linkis-ujes-enginemanager + 0.9.4 + +``` + +**2)Interfaces to be implemented** + +Below interfaces are required to be implemented in EngineManager: +- EngineCreator. Method createProcessEngineBuilder needs to be overridden in the existing AbstractEngineCreator to create an EngineBuilder. +Here ProcessEngineBuilder has already provided a class called JavaProcessEngineBuilder, which is an abstract class accomplishes configurations of classpath, JavaOpts, GC file path and log path, and opening DEBUG port in test mode. To implement JavaProcessEngineBuilder, only extra classpath and JavaOpts are needed to be specified. +- EngineResourceFactory. Method getRequestResource needs to be overridden in the existing AbstractEngineResourceFactory to declare user customized resource requirements. +- hooks. A Spring bean used to add pre and post hooks around the Engine startup procedure. Users need to specify an Array[EngineHook] for dependency injection. +- resources. A Spring bean used to register resources to RM. Users need to specify an instance of ModuleInfo for dependency injection. + +##### 4.3Engine module adaption +**1)maven dependency** +```xml + + com.webank.wedatasphere.linkis + linkis-ujes-engine + 0.9.4 + +``` +**2)Interfaces to be implemented** + +Below interfaces are required to be implemented in Engine： +- EngineExecutorFactory. Used to build an EngineExecutor from a Map by implementing method createExecutor. +- EngineExecutor. The actual executor to execute the code submitted from the entrance. Methods need to be implemented: getActualUsedResources(the resource an engine acually used), executeLine(execute a line of the code parsed by CodeParser), executeCompletely(the suplementary method for executeLine. If executeLine returns ExecuteIncomplete, new code will be submitted with the previous code together to the engine) + +Below interfaces/beans are optional in Engine: +- engineHooks: Array[EngineHook], a Spring bean used to add pre and post hooks around the Engine startup procedure. Currently the system provides 2 hooks: CodeGeneratorEngineHook for UDF/Function loading and ReleaseEngineHook for releasing spare engines. The system registers engineHooks=Array(ReleaseEngineHook) only by default. +- CodeParser. Used to parse code into lines and submit one line only for each execution loop. The system registers a CodeParser returns all the code at once by default. - EngineParser. Used to convert a RequestTask to a Job that is acceptable by Scheduler. If not specified, the system registers an EngineParser that converts RequestTask to CommonEngineJob. \ No newline at end of file diff --git a/docs/en_US/ch3/Linkis_Java_SDK_doc.md b/docs/en_US/ch3/Linkis_Java_SDK_doc.md index 244e7a4fde..487d782755 100644 --- a/docs/en_US/ch3/Linkis_Java_SDK_doc.md +++ b/docs/en_US/ch3/Linkis_Java_SDK_doc.md @@ -1,157 +1,157 @@ -### Linkis Java SDK Doc -#### 1.Introduction - -Linkis provides a Java client implementation for users to have quick-access to Linkis background services using UJESClient. - -#### 2 Quick start - - We provided two test classes under dir ujes/client/src/test： - - com.webank.wedatasphere.linkis.ujes.client.UJESClientImplTestJ # Test class based on Java - com.webank.wedatasphere.linkis.ujes.client.UJESClientImplTest # Test class based on Scala - - If you have cloned the source code of Linkis, you can directly run these two test classes. - - - **Below sections introduce about how to write the code to complete a single execution on Linkis** - -#### 3 Fast implementation - -##### 3.1 maven dependency - -```xml - - com.webank.wedatasphere.Linkis - Linkis-ujes-client - 0.9.3 - -``` - -#### 3.2 Sample implementation - -- **JAVA** -```java -package com.webank.bdp.dataworkcloud.ujes.client; - -import com.webank.wedatasphere.Linkis.common.utils.Utils; -import com.webank.wedatasphere.Linkis.httpclient.dws.authentication.StaticAuthenticationStrategy; -import com.webank.wedatasphere.Linkis.httpclient.dws.config.DWSClientConfig; -import com.webank.wedatasphere.Linkis.httpclient.dws.config.DWSClientConfigBuilder; -import com.webank.wedatasphere.Linkis.ujes.client.UJESClient; -import com.webank.wedatasphere.Linkis.ujes.client.UJESClientImpl; -import com.webank.wedatasphere.Linkis.ujes.client.request.JobExecuteAction; -import com.webank.wedatasphere.Linkis.ujes.client.request.ResultSetAction; -import com.webank.wedatasphere.Linkis.ujes.client.response.JobExecuteResult; -import com.webank.wedatasphere.Linkis.ujes.client.response.JobInfoResult; -import com.webank.wedatasphere.Linkis.ujes.client.response.JobProgressResult; -import com.webank.wedatasphere.Linkis.ujes.client.response.JobStatusResult; -import org.apache.commons.io.IOUtils; - -import java.util.concurrent.TimeUnit; - - -public class UJESClientImplTestJ{ - public static void main(String[] args){ - // 1. To do the configuration, an instance of DWSClientConfig should be obtained from DWSClientBuilder. - DWSClientConfig clientConfig = ((DWSClientConfigBuilder) (DWSClientConfigBuilder.newBuilder() - .addUJESServerUrl("http://${ip}:${port}") //Specify the ServerUrl，The address of Willink gateway, i.e. http://{ip}:{port} - .connectionTimeout(30000) //connectionTimeOut: The connection timeout of the client - .discoveryEnabled(true).discoveryFrequency(1, TimeUnit.MINUTES) //Enable service discovery. Once enabled, newly started Gateway will be auto-dicovered. - .loadbalancerEnabled(true) // Enable load balancing. Cannot be enabled alone without service discovery enabled. - .maxConnectionSize(5) //Max connection size, aka the max concurrent threshold - .retryEnabled(false).readTimeout(30000) //whether to retry after failure - .setAuthenticationStrategy(new StaticAuthenticationStrategy()) //AuthenticationStrategy, The authentication strategy of Linkis - .setAuthTokenKey("${username}").setAuthTokenValue("${password}"))) //The authentication key，usually the username;The authentication value，usually the password - .setDWSVersion("v1").build(); //The version of Linkis background protocol, currently v1 - - // 2. Create a UJESClient from DWSClientConfig - UJESClient client = new UJESClientImpl(clientConfig); - - // 3. Begin to execute the code - JobExecuteResult jobExecuteResult = client.execute(JobExecuteAction.builder() - .setCreator("LinkisClient-Test") //creator. The name of the system which holds the UJES client, used for system level isolation. - .addExecuteCode("show tables") //ExecutionCode. The code which is requested to be executed - .setEngineType(JobExecuteAction.EngineType$.MODULE$.HIVE()) // The engine type expected by the client, i.e. Spark, Hive, etc... - .setUser("johnnwang") //User, The user who makes this request；Used for user level multi-tenant isolation - .build()); - System.out.println("execId: " + jobExecuteResult.getExecID() + ", taskId: " + jobExecuteResult.taskID()); - - // 4. Synch the status of script execution - JobStatusResult status = client.status(jobExecuteResult); - while(!status.isCompleted()) { - // 5. Synch the status of script execution - JobProgressResult progress = client.progress(jobExecuteResult); - Utils.sleepQuietly(500); - status = client.status(jobExecuteResult); - } - - // 6. Synch the job information of script execution - JobInfoResult jobInfo = client.getJobInfo(jobExecuteResult); - // 7. Fetch the list of result sets(Multiple result sets will be generated if a user submitted multiple SQL at once) - String resultSet = jobInfo.getResultSetList(client)[0]; - // 8. Fetch detailed result set content with a particular result set info - Object fileContents = client.resultSet(ResultSetAction.builder().setPath(resultSet).setUser(jobExecuteResult.getUser()).build()).getFileContent(); - System.out.println("fileContents: " + fileContents); - IOUtils.closeQuietly(client); - } -} -``` - -- **SCALA** -```scala - -import java.util.concurrent.TimeUnit - -import com.webank.wedatasphere.Linkis.common.utils.Utils -import com.webank.wedatasphere.Linkis.httpclient.dws.authentication.StaticAuthenticationStrategy -import com.webank.wedatasphere.Linkis.httpclient.dws.config.DWSClientConfigBuilder -import com.webank.wedatasphere.Linkis.ujes.client.request.JobExecuteAction.EngineType -import com.webank.wedatasphere.Linkis.ujes.client.request.{JobExecuteAction, ResultSetAction} -import org.apache.commons.io.IOUtils - -object UJESClientImplTest extends App { - - // 1. To do the configuration, an instance of DWSClientConfig should be obtained from DWSClientBuilder. - val clientConfig = DWSClientConfigBuilder.newBuilder() - .addUJESServerUrl("http://${ip}:${port}") //Specify the ServerUrl，The address of Willink gateway, i.e. http://{ip}:{port} - .connectionTimeout(30000) //connectionTimeOut: The connection timeout of the client - .discoveryEnabled(true).discoveryFrequency(1, TimeUnit.MINUTES) //Enable service discovery. Once enabled, newly started Gateway will be auto-dicovered. - .loadbalancerEnabled(true) // Enable load balancing. Cannot be enabled alone without service discovery enabled. - .maxConnectionSize(5) //Max connection size, aka the max concurrent threshold - .retryEnabled(false).readTimeout(30000) //whether to retry after failure - .setAuthenticationStrategy(new StaticAuthenticationStrategy()) //AuthenticationStrategy, The authentication strategy of Linkis - .setAuthTokenKey("${username}").setAuthTokenValue("${password}") //The authentication key，usually the username;The authentication value，usually the password - .setDWSVersion("v1").build() //The version of Linkis background protocol, currently v1 - - // 2. Create a UJESClient from DWSClientConfig - val client = UJESClient(clientConfig) - - // 3. Begin to execute the code - val jobExecuteResult = client.execute(JobExecuteAction.builder() - .setCreator("LinkisClient-Test") //creator. The name of the system which holds the UJES client, used for system level isolation. - .addExecuteCode("show tables") //ExecutionCode. The code which is requested to be executed - .setEngineType(EngineType.SPARK) // The engine type expected by the client, i.e. Spark, Hive, etc... - .setUser("${username}").build()) //User, The user who makes this request；Used for user level multi-tenant isolation - println("execId: " + jobExecuteResult.getExecID + ", taskId: " + jobExecuteResult.taskID) - - // 4. Synch the status of script execution - var status = client.status(jobExecuteResult) - while(!status.isCompleted) { - // 5. Synch the status of script execution - val progress = client.progress(jobExecuteResult) - val progressInfo = if(progress.getProgressInfo != null) progress.getProgressInfo.toList else List.empty - println("progress: " + progress.getProgress + ", progressInfo: " + progressInfo) - Utils.sleepQuietly(500) - status = client.status(jobExecuteResult) - } - - // 6. Synch the job information of script execution - val jobInfo = client.getJobInfo(jobExecuteResult) - // 7. Fetch the list of result sets(Multiple result sets will be generated if a user submitted multiple SQL at once) - val resultSet = jobInfo.getResultSetList(client).head - // 8. Fetch detailed result set content with a particular result set info - val fileContents = client.resultSet(ResultSetAction.builder().setPath(resultSet).setUser(jobExecuteResult.getUser).build()).getFileContent - println("fileContents: " + fileContents) - IOUtils.closeQuietly(client) -} -``` +### Linkis Java SDK Doc +#### 1.Introduction + +Linkis provides a Java client implementation for users to have quick-access to Linkis background services using UJESClient. + +#### 2 Quick start + + We provided two test classes under dir ujes/client/src/test： + + com.webank.wedatasphere.linkis.ujes.client.UJESClientImplTestJ # Test class based on Java + com.webank.wedatasphere.linkis.ujes.client.UJESClientImplTest # Test class based on Scala + + If you have cloned the source code of Linkis, you can directly run these two test classes. + + + **Below sections introduce about how to write the code to complete a single execution on Linkis** + +#### 3 Fast implementation + +##### 3.1 maven dependency + +```xml + + com.webank.wedatasphere.Linkis + Linkis-ujes-client + 0.9.4 + +``` + +#### 3.2 Sample implementation + +- **JAVA** +```java +package com.webank.bdp.dataworkcloud.ujes.client; + +import com.webank.wedatasphere.Linkis.common.utils.Utils; +import com.webank.wedatasphere.Linkis.httpclient.dws.authentication.StaticAuthenticationStrategy; +import com.webank.wedatasphere.Linkis.httpclient.dws.config.DWSClientConfig; +import com.webank.wedatasphere.Linkis.httpclient.dws.config.DWSClientConfigBuilder; +import com.webank.wedatasphere.Linkis.ujes.client.UJESClient; +import com.webank.wedatasphere.Linkis.ujes.client.UJESClientImpl; +import com.webank.wedatasphere.Linkis.ujes.client.request.JobExecuteAction; +import com.webank.wedatasphere.Linkis.ujes.client.request.ResultSetAction; +import com.webank.wedatasphere.Linkis.ujes.client.response.JobExecuteResult; +import com.webank.wedatasphere.Linkis.ujes.client.response.JobInfoResult; +import com.webank.wedatasphere.Linkis.ujes.client.response.JobProgressResult; +import com.webank.wedatasphere.Linkis.ujes.client.response.JobStatusResult; +import org.apache.commons.io.IOUtils; + +import java.util.concurrent.TimeUnit; + + +public class UJESClientImplTestJ{ + public static void main(String[] args){ + // 1. To do the configuration, an instance of DWSClientConfig should be obtained from DWSClientBuilder. + DWSClientConfig clientConfig = ((DWSClientConfigBuilder) (DWSClientConfigBuilder.newBuilder() + .addUJESServerUrl("http://${ip}:${port}") //Specify the ServerUrl，The address of Willink gateway, i.e. http://{ip}:{port} + .connectionTimeout(30000) //connectionTimeOut: The connection timeout of the client + .discoveryEnabled(true).discoveryFrequency(1, TimeUnit.MINUTES) //Enable service discovery. Once enabled, newly started Gateway will be auto-dicovered. + .loadbalancerEnabled(true) // Enable load balancing. Cannot be enabled alone without service discovery enabled. + .maxConnectionSize(5) //Max connection size, aka the max concurrent threshold + .retryEnabled(false).readTimeout(30000) //whether to retry after failure + .setAuthenticationStrategy(new StaticAuthenticationStrategy()) //AuthenticationStrategy, The authentication strategy of Linkis + .setAuthTokenKey("${username}").setAuthTokenValue("${password}"))) //The authentication key，usually the username;The authentication value，usually the password + .setDWSVersion("v1").build(); //The version of Linkis background protocol, currently v1 + + // 2. Create a UJESClient from DWSClientConfig + UJESClient client = new UJESClientImpl(clientConfig); + + // 3. Begin to execute the code + JobExecuteResult jobExecuteResult = client.execute(JobExecuteAction.builder() + .setCreator("LinkisClient-Test") //creator. The name of the system which holds the UJES client, used for system level isolation. + .addExecuteCode("show tables") //ExecutionCode. The code which is requested to be executed + .setEngineType(JobExecuteAction.EngineType$.MODULE$.HIVE()) // The engine type expected by the client, i.e. Spark, Hive, etc... + .setUser("johnnwang") //User, The user who makes this request；Used for user level multi-tenant isolation + .build()); + System.out.println("execId: " + jobExecuteResult.getExecID() + ", taskId: " + jobExecuteResult.taskID()); + + // 4. Synch the status of script execution + JobStatusResult status = client.status(jobExecuteResult); + while(!status.isCompleted()) { + // 5. Synch the status of script execution + JobProgressResult progress = client.progress(jobExecuteResult); + Utils.sleepQuietly(500); + status = client.status(jobExecuteResult); + } + + // 6. Synch the job information of script execution + JobInfoResult jobInfo = client.getJobInfo(jobExecuteResult); + // 7. Fetch the list of result sets(Multiple result sets will be generated if a user submitted multiple SQL at once) + String resultSet = jobInfo.getResultSetList(client)[0]; + // 8. Fetch detailed result set content with a particular result set info + Object fileContents = client.resultSet(ResultSetAction.builder().setPath(resultSet).setUser(jobExecuteResult.getUser()).build()).getFileContent(); + System.out.println("fileContents: " + fileContents); + IOUtils.closeQuietly(client); + } +} +``` + +- **SCALA** +```scala + +import java.util.concurrent.TimeUnit + +import com.webank.wedatasphere.Linkis.common.utils.Utils +import com.webank.wedatasphere.Linkis.httpclient.dws.authentication.StaticAuthenticationStrategy +import com.webank.wedatasphere.Linkis.httpclient.dws.config.DWSClientConfigBuilder +import com.webank.wedatasphere.Linkis.ujes.client.request.JobExecuteAction.EngineType +import com.webank.wedatasphere.Linkis.ujes.client.request.{JobExecuteAction, ResultSetAction} +import org.apache.commons.io.IOUtils + +object UJESClientImplTest extends App { + + // 1. To do the configuration, an instance of DWSClientConfig should be obtained from DWSClientBuilder. + val clientConfig = DWSClientConfigBuilder.newBuilder() + .addUJESServerUrl("http://${ip}:${port}") //Specify the ServerUrl，The address of Willink gateway, i.e. http://{ip}:{port} + .connectionTimeout(30000) //connectionTimeOut: The connection timeout of the client + .discoveryEnabled(true).discoveryFrequency(1, TimeUnit.MINUTES) //Enable service discovery. Once enabled, newly started Gateway will be auto-dicovered. + .loadbalancerEnabled(true) // Enable load balancing. Cannot be enabled alone without service discovery enabled. + .maxConnectionSize(5) //Max connection size, aka the max concurrent threshold + .retryEnabled(false).readTimeout(30000) //whether to retry after failure + .setAuthenticationStrategy(new StaticAuthenticationStrategy()) //AuthenticationStrategy, The authentication strategy of Linkis + .setAuthTokenKey("${username}").setAuthTokenValue("${password}") //The authentication key，usually the username;The authentication value，usually the password + .setDWSVersion("v1").build() //The version of Linkis background protocol, currently v1 + + // 2. Create a UJESClient from DWSClientConfig + val client = UJESClient(clientConfig) + + // 3. Begin to execute the code + val jobExecuteResult = client.execute(JobExecuteAction.builder() + .setCreator("LinkisClient-Test") //creator. The name of the system which holds the UJES client, used for system level isolation. + .addExecuteCode("show tables") //ExecutionCode. The code which is requested to be executed + .setEngineType(EngineType.SPARK) // The engine type expected by the client, i.e. Spark, Hive, etc... + .setUser("${username}").build()) //User, The user who makes this request；Used for user level multi-tenant isolation + println("execId: " + jobExecuteResult.getExecID + ", taskId: " + jobExecuteResult.taskID) + + // 4. Synch the status of script execution + var status = client.status(jobExecuteResult) + while(!status.isCompleted) { + // 5. Synch the status of script execution + val progress = client.progress(jobExecuteResult) + val progressInfo = if(progress.getProgressInfo != null) progress.getProgressInfo.toList else List.empty + println("progress: " + progress.getProgress + ", progressInfo: " + progressInfo) + Utils.sleepQuietly(500) + status = client.status(jobExecuteResult) + } + + // 6. Synch the job information of script execution + val jobInfo = client.getJobInfo(jobExecuteResult) + // 7. Fetch the list of result sets(Multiple result sets will be generated if a user submitted multiple SQL at once) + val resultSet = jobInfo.getResultSetList(client).head + // 8. Fetch detailed result set content with a particular result set info + val fileContents = client.resultSet(ResultSetAction.builder().setPath(resultSet).setUser(jobExecuteResult.getUser).build()).getFileContent + println("fileContents: " + fileContents) + IOUtils.closeQuietly(client) +} +``` diff --git "a/docs/zh_CN/ch2/linkis\345\277\253\351\200\237\344\275\277\347\224\250\346\226\207\346\241\243.md" "b/docs/zh_CN/ch2/linkis\345\277\253\351\200\237\344\275\277\347\224\250\346\226\207\346\241\243.md" index 3553a9e5e9..41727944b3 100644 --- "a/docs/zh_CN/ch2/linkis\345\277\253\351\200\237\344\275\277\347\224\250\346\226\207\346\241\243.md" +++ "b/docs/zh_CN/ch2/linkis\345\277\253\351\200\237\344\275\277\347\224\250\346\226\207\346\241\243.md" @@ -21,7 +21,7 @@ Linkis为用户提供了Java客户端的实现，用户可以使用UJESClient对 com.webank.wedatasphere.Linkis Linkis-ujes-client - 0.9.3 + 0.9.4 ``` diff --git "a/docs/zh_CN/ch3/Linkis\345\246\202\344\275\225\346\216\245\345\205\245\346\226\260\347\232\204\345\272\225\345\261\202\350\256\241\347\256\227\345\255\230\345\202\250\345\274\225\346\223\216.md" "b/docs/zh_CN/ch3/Linkis\345\246\202\344\275\225\346\216\245\345\205\245\346\226\260\347\232\204\345\272\225\345\261\202\350\256\241\347\256\227\345\255\230\345\202\250\345\274\225\346\223\216.md" index 6b3c972de2..42f3cdbc24 100644 --- "a/docs/zh_CN/ch3/Linkis\345\246\202\344\275\225\346\216\245\345\205\245\346\226\260\347\232\204\345\272\225\345\261\202\350\256\241\347\256\227\345\255\230\345\202\250\345\274\225\346\223\216.md" +++ "b/docs/zh_CN/ch3/Linkis\345\246\202\344\275\225\346\216\245\345\205\245\346\226\260\347\232\204\345\272\225\345\261\202\350\256\241\347\256\227\345\255\230\345\202\250\345\274\225\346\223\216.md" @@ -22,7 +22,7 @@ Linkis的底层架构非常灵活，所有的顶层接口几乎都提供了一 com.webank.wedatasphere.Linkis Linkis-ujes-entrance - 0.9.3 + 0.9.4 ``` @@ -42,7 +42,7 @@ Linkis的底层架构非常灵活，所有的顶层接口几乎都提供了一 com.webank.wedatasphere.Linkis Linkis-ujes-enginemanager - 0.9.3 + 0.9.4 ``` @@ -74,7 +74,7 @@ Linkis的底层架构非常灵活，所有的顶层接口几乎都提供了一 com.webank.wedatasphere.Linkis Linkis-ujes-engine - 0.9.3 + 0.9.4 ``` diff --git "a/docs/zh_CN/ch3/linkis\344\275\277\347\224\250\346\226\207\346\241\243.md" "b/docs/zh_CN/ch3/linkis\344\275\277\347\224\250\346\226\207\346\241\243.md" index 0695c79ea6..1c61591a77 100644 --- "a/docs/zh_CN/ch3/linkis\344\275\277\347\224\250\346\226\207\346\241\243.md" +++ "b/docs/zh_CN/ch3/linkis\344\275\277\347\224\250\346\226\207\346\241\243.md" @@ -312,7 +312,7 @@ Linkis项目使用了Spring框架作为底层技术，所以一些类实例可 com.webank.wedatasphere.Linkis Linkis-ujes-entrance - 0.9.3 + 0.9.4 ``` **2)需要实现的接口** @@ -328,7 +328,7 @@ Entrance没有必须要实例化的接口，以下接口可以根据需要进行 com.webank.wedatasphere.Linkis Linkis-ujes-enginemanager - 0.9.3 + 0.9.4 ``` @@ -348,7 +348,7 @@ EngineManager需要对以下接口根据需要进行实现: com.webank.wedatasphere.Linkis Linkis-ujes-engine - 0.9.3 + 0.9.4 ``` **2)需要实现的接口** diff --git a/eurekaServer/pom.xml b/eurekaServer/pom.xml index 2f4128093c..2999d30588 100644 --- a/eurekaServer/pom.xml +++ b/eurekaServer/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-eureka-server diff --git a/extensions/spark-excel/pom.xml b/extensions/spark-excel/pom.xml index 10ce736575..0c920b697d 100644 --- a/extensions/spark-excel/pom.xml +++ b/extensions/spark-excel/pom.xml @@ -6,7 +6,7 @@ com.webank.weDataSphere spark-excel_2.11 - 0.9.3 + 0.9.4 UTF-8 diff --git a/gateway/core/pom.xml b/gateway/core/pom.xml index 26bacc8d3e..2b83561737 100644 --- a/gateway/core/pom.xml +++ b/gateway/core/pom.xml @@ -22,7 +22,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-gateway-core diff --git a/gateway/gateway-httpclient-support/pom.xml b/gateway/gateway-httpclient-support/pom.xml index ca6a715f31..dce31146f9 100644 --- a/gateway/gateway-httpclient-support/pom.xml +++ b/gateway/gateway-httpclient-support/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-gateway-httpclient-support diff --git a/gateway/gateway-ujes-support/pom.xml b/gateway/gateway-ujes-support/pom.xml index 9613d47fd3..9a44a17cac 100644 --- a/gateway/gateway-ujes-support/pom.xml +++ b/gateway/gateway-ujes-support/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-gateway-ujes-support diff --git a/gateway/springcloudgateway/pom.xml b/gateway/springcloudgateway/pom.xml index 15de471629..068c01c5a5 100644 --- a/gateway/springcloudgateway/pom.xml +++ b/gateway/springcloudgateway/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-gateway-springcloudgateway diff --git a/metadata/pom.xml b/metadata/pom.xml index 42dcc78826..d4a05f473d 100644 --- a/metadata/pom.xml +++ b/metadata/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 jar diff --git a/pom.xml b/pom.xml index 35d08eb761..79b182cef7 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ com.webank.wedatasphere.linkis linkis - 0.9.3 + 0.9.4 pom Linkis Project Parent POM @@ -134,7 +134,7 @@ 9.4.11.v20180605 4.5.4 1.7.12 - 0.9.3 + 0.9.4 3.3.3 UTF-8 0.11.2 @@ -238,7 +238,7 @@ true - 0.9.3 + 0.9.4 4.1.17.Final 3.5.3 @@ -246,7 +246,7 @@ spark2.3 - 0.9.3 + 0.9.4 4.1.17.Final 3.2.11 @@ -254,7 +254,7 @@ spark2.2-2.0 - 0.9.3 + 0.9.4 4.0.47.Final 3.2.11 diff --git a/publicService/application/pom.xml b/publicService/application/pom.xml index 01af0feeef..7e921f7643 100644 --- a/publicService/application/pom.xml +++ b/publicService/application/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-application @@ -47,7 +47,7 @@ com.webank.wedatasphere.linkis linkis-storage - 0.9.3 + 0.9.4 provided diff --git a/publicService/configuration/pom.xml b/publicService/configuration/pom.xml index 3150bd4ff9..7fae3ee568 100644 --- a/publicService/configuration/pom.xml +++ b/publicService/configuration/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-configuration diff --git a/publicService/jobhistory/pom.xml b/publicService/jobhistory/pom.xml index 195f3cb161..43421fcaa1 100644 --- a/publicService/jobhistory/pom.xml +++ b/publicService/jobhistory/pom.xml @@ -24,7 +24,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-jobhistory diff --git a/publicService/pom.xml b/publicService/pom.xml index 5168a7063d..53617f2515 100644 --- a/publicService/pom.xml +++ b/publicService/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/publicService/udf/pom.xml b/publicService/udf/pom.xml index 137c5eaeea..5320cfe31a 100644 --- a/publicService/udf/pom.xml +++ b/publicService/udf/pom.xml @@ -21,7 +21,7 @@ com.webank.wedatasphere.linkis linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/publicService/variable/pom.xml b/publicService/variable/pom.xml index 16b2a48f94..97c38b28cf 100644 --- a/publicService/variable/pom.xml +++ b/publicService/variable/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-variable diff --git a/publicService/workspace/client/workspace-httpclient/pom.xml b/publicService/workspace/client/workspace-httpclient/pom.xml index ec446371b7..967e45f3b8 100644 --- a/publicService/workspace/client/workspace-httpclient/pom.xml +++ b/publicService/workspace/client/workspace-httpclient/pom.xml @@ -7,7 +7,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-workspace-httpclient diff --git a/publicService/workspace/pom.xml b/publicService/workspace/pom.xml index 3d937529d2..8a5b98daae 100644 --- a/publicService/workspace/pom.xml +++ b/publicService/workspace/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-workspace jar diff --git a/resourceManager/resourcemanagerclient/pom.xml b/resourceManager/resourcemanagerclient/pom.xml index 7f9341f145..fda5b9ca7d 100644 --- a/resourceManager/resourcemanagerclient/pom.xml +++ b/resourceManager/resourcemanagerclient/pom.xml @@ -20,7 +20,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - 0.9.3 + 0.9.4 com.webank.wedatasphere.linkis linkis diff --git a/resourceManager/resourcemanagercommon/pom.xml b/resourceManager/resourcemanagercommon/pom.xml index 051cec5c1f..25e0b3ffef 100644 --- a/resourceManager/resourcemanagercommon/pom.xml +++ b/resourceManager/resourcemanagercommon/pom.xml @@ -21,7 +21,7 @@ 4.0.0 - 0.9.3 + 0.9.4 com.webank.wedatasphere.linkis linkis diff --git a/resourceManager/resourcemanagerserver/pom.xml b/resourceManager/resourcemanagerserver/pom.xml index a02e0bae80..848acf9e1b 100644 --- a/resourceManager/resourcemanagerserver/pom.xml +++ b/resourceManager/resourcemanagerserver/pom.xml @@ -21,7 +21,7 @@ 4.0.0 - 0.9.3 + 0.9.4 com.webank.wedatasphere.linkis linkis diff --git a/storage/pesIO/io-engine/pom.xml b/storage/pesIO/io-engine/pom.xml index 718e279cf7..2daa412cb3 100644 --- a/storage/pesIO/io-engine/pom.xml +++ b/storage/pesIO/io-engine/pom.xml @@ -21,7 +21,7 @@ 4.0.0 - 0.9.3 + 0.9.4 com.webank.wedatasphere.linkis linkis diff --git a/storage/pesIO/io-enginemanager/pom.xml b/storage/pesIO/io-enginemanager/pom.xml index 8d585629d7..10bd4caa7c 100644 --- a/storage/pesIO/io-enginemanager/pom.xml +++ b/storage/pesIO/io-enginemanager/pom.xml @@ -21,7 +21,7 @@ 4.0.0 - 0.9.3 + 0.9.4 com.webank.wedatasphere.linkis linkis diff --git a/storage/pesIO/io-entrance/pom.xml b/storage/pesIO/io-entrance/pom.xml index 233bb4860b..a5c66f663c 100644 --- a/storage/pesIO/io-entrance/pom.xml +++ b/storage/pesIO/io-entrance/pom.xml @@ -20,7 +20,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - 0.9.3 + 0.9.4 com.webank.wedatasphere.linkis linkis diff --git a/storage/storage/pom.xml b/storage/storage/pom.xml index 19302dd605..f12d217ae1 100644 --- a/storage/storage/pom.xml +++ b/storage/storage/pom.xml @@ -21,7 +21,7 @@ 4.0.0 - 0.9.3 + 0.9.4 com.webank.wedatasphere.linkis linkis diff --git a/ujes/client/pom.xml b/ujes/client/pom.xml index b1f1b673ac..a293cfd068 100644 --- a/ujes/client/pom.xml +++ b/ujes/client/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-ujes-client diff --git a/ujes/definedEngines/hive/engine/pom.xml b/ujes/definedEngines/hive/engine/pom.xml index 68260f23f7..54357a87e5 100644 --- a/ujes/definedEngines/hive/engine/pom.xml +++ b/ujes/definedEngines/hive/engine/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/ujes/definedEngines/hive/enginemanager/pom.xml b/ujes/definedEngines/hive/enginemanager/pom.xml index 84c66a43fd..949d601a41 100644 --- a/ujes/definedEngines/hive/enginemanager/pom.xml +++ b/ujes/definedEngines/hive/enginemanager/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/ujes/definedEngines/hive/entrance/pom.xml b/ujes/definedEngines/hive/entrance/pom.xml index b39b7c1f1a..bb771d41e1 100644 --- a/ujes/definedEngines/hive/entrance/pom.xml +++ b/ujes/definedEngines/hive/entrance/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/ujes/definedEngines/jdbc/entrance/pom.xml b/ujes/definedEngines/jdbc/entrance/pom.xml index 582c39ab9e..ba17552dda 100644 --- a/ujes/definedEngines/jdbc/entrance/pom.xml +++ b/ujes/definedEngines/jdbc/entrance/pom.xml @@ -22,7 +22,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 @@ -77,7 +77,7 @@ org.apache.thrift libthrift - 0.9.3 + 0.9.4 pom diff --git a/ujes/definedEngines/mlsql/entrance/pom.xml b/ujes/definedEngines/mlsql/entrance/pom.xml index 62f924d1bd..31d60e783c 100644 --- a/ujes/definedEngines/mlsql/entrance/pom.xml +++ b/ujes/definedEngines/mlsql/entrance/pom.xml @@ -5,7 +5,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/ujes/definedEngines/pipeline/engine/pom.xml b/ujes/definedEngines/pipeline/engine/pom.xml index 956cac82b0..67c2155380 100644 --- a/ujes/definedEngines/pipeline/engine/pom.xml +++ b/ujes/definedEngines/pipeline/engine/pom.xml @@ -21,7 +21,7 @@ 4.0.0 - 0.9.3 + 0.9.4 com.webank.wedatasphere.linkis linkis diff --git a/ujes/definedEngines/pipeline/enginemanager/pom.xml b/ujes/definedEngines/pipeline/enginemanager/pom.xml index 7efdddbc56..1aea5d59b9 100644 --- a/ujes/definedEngines/pipeline/enginemanager/pom.xml +++ b/ujes/definedEngines/pipeline/enginemanager/pom.xml @@ -21,7 +21,7 @@ 4.0.0 - 0.9.3 + 0.9.4 com.webank.wedatasphere.linkis linkis diff --git a/ujes/definedEngines/pipeline/entrance/pom.xml b/ujes/definedEngines/pipeline/entrance/pom.xml index ca5f811711..cc95e786ef 100644 --- a/ujes/definedEngines/pipeline/entrance/pom.xml +++ b/ujes/definedEngines/pipeline/entrance/pom.xml @@ -21,7 +21,7 @@ 4.0.0 - 0.9.3 + 0.9.4 com.webank.wedatasphere.linkis linkis diff --git a/ujes/definedEngines/python/engine/pom.xml b/ujes/definedEngines/python/engine/pom.xml index 33b6fba552..a08864540c 100644 --- a/ujes/definedEngines/python/engine/pom.xml +++ b/ujes/definedEngines/python/engine/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-python-engine diff --git a/ujes/definedEngines/python/enginemanager/pom.xml b/ujes/definedEngines/python/enginemanager/pom.xml index 9e623b97cf..cac5209654 100644 --- a/ujes/definedEngines/python/enginemanager/pom.xml +++ b/ujes/definedEngines/python/enginemanager/pom.xml @@ -23,17 +23,17 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-python-enginemanager - 0.9.3 + 0.9.4 jar - 0.9.3 + 0.9.4 diff --git a/ujes/definedEngines/python/entrance/pom.xml b/ujes/definedEngines/python/entrance/pom.xml index b06dae1a9a..0e1b31a078 100644 --- a/ujes/definedEngines/python/entrance/pom.xml +++ b/ujes/definedEngines/python/entrance/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-python-entracne diff --git a/ujes/definedEngines/shell/engine/pom.xml b/ujes/definedEngines/shell/engine/pom.xml index f18edefe01..f9f2129a14 100644 --- a/ujes/definedEngines/shell/engine/pom.xml +++ b/ujes/definedEngines/shell/engine/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/ujes/definedEngines/shell/enginemanager/pom.xml b/ujes/definedEngines/shell/enginemanager/pom.xml index 4987448bbb..8e821e28a6 100644 --- a/ujes/definedEngines/shell/enginemanager/pom.xml +++ b/ujes/definedEngines/shell/enginemanager/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/ujes/definedEngines/shell/entrance/pom.xml b/ujes/definedEngines/shell/entrance/pom.xml index 40f9f50178..1c6765b911 100644 --- a/ujes/definedEngines/shell/entrance/pom.xml +++ b/ujes/definedEngines/shell/entrance/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/ujes/definedEngines/spark/engine/pom.xml b/ujes/definedEngines/spark/engine/pom.xml index 92fcda873b..704c9110a2 100644 --- a/ujes/definedEngines/spark/engine/pom.xml +++ b/ujes/definedEngines/spark/engine/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-ujes-spark-engine diff --git a/ujes/definedEngines/spark/enginemanager/pom.xml b/ujes/definedEngines/spark/enginemanager/pom.xml index d10aec39ed..ae8a3b488f 100644 --- a/ujes/definedEngines/spark/enginemanager/pom.xml +++ b/ujes/definedEngines/spark/enginemanager/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-ujes-spark-enginemanager diff --git a/ujes/definedEngines/spark/entrance/pom.xml b/ujes/definedEngines/spark/entrance/pom.xml index 7641bd5f88..070592a12f 100644 --- a/ujes/definedEngines/spark/entrance/pom.xml +++ b/ujes/definedEngines/spark/entrance/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-ujes-spark-entracne diff --git a/ujes/definedEngines/tispark/engine/pom.xml b/ujes/definedEngines/tispark/engine/pom.xml index 09723524bd..0e334bb3a9 100644 --- a/ujes/definedEngines/tispark/engine/pom.xml +++ b/ujes/definedEngines/tispark/engine/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 4.0.0 diff --git a/ujes/engine/pom.xml b/ujes/engine/pom.xml index 387ce2ddbb..9fa05dbcbb 100644 --- a/ujes/engine/pom.xml +++ b/ujes/engine/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-ujes-engine diff --git a/ujes/enginemanager/pom.xml b/ujes/enginemanager/pom.xml index 196f1dbe7c..9843ac997c 100644 --- a/ujes/enginemanager/pom.xml +++ b/ujes/enginemanager/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-ujes-enginemanager diff --git a/ujes/entrance/pom.xml b/ujes/entrance/pom.xml index 575430f5cc..90294dcacb 100644 --- a/ujes/entrance/pom.xml +++ b/ujes/entrance/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-ujes-entrance diff --git a/ujes/entranceclient/pom.xml b/ujes/entranceclient/pom.xml index 01158a663d..0657ff0d72 100644 --- a/ujes/entranceclient/pom.xml +++ b/ujes/entranceclient/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-ujes-entranceclient diff --git a/ujes/jdbc/pom.xml b/ujes/jdbc/pom.xml index 15239041db..ebbdfee216 100644 --- a/ujes/jdbc/pom.xml +++ b/ujes/jdbc/pom.xml @@ -7,7 +7,7 @@ linkis com.webank.wedatasphere.linkis - 0.9.3 + 0.9.4 linkis-ujes-jdbc @@ -15,7 +15,7 @@ com.webank.wedatasphere.linkis linkis-ujes-client - 0.9.3 + 0.9.4 diff --git a/userControl/pom.xml b/userControl/pom.xml index fca67462c4..67c41c11b2 100644 --- a/userControl/pom.xml +++ b/userControl/pom.xml @@ -21,7 +21,7 @@ com.webank.wedatasphere.linkis linkis - 0.9.3 + 0.9.4 4.0.0 From cc7fb728a9765f31a54d86bda3d518769f508d23 Mon Sep 17 00:00:00 2001 From: johnnywang Date: Wed, 3 Jun 2020 17:48:41 +0800 Subject: [PATCH 2/8] Update pom.xml --- ujes/definedEngines/jdbc/entrance/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ujes/definedEngines/jdbc/entrance/pom.xml b/ujes/definedEngines/jdbc/entrance/pom.xml index ba17552dda..0f214b667c 100644 --- a/ujes/definedEngines/jdbc/entrance/pom.xml +++ b/ujes/definedEngines/jdbc/entrance/pom.xml @@ -77,7 +77,7 @@ org.apache.thrift libthrift - 0.9.4 + 0.9.3 pom @@ -139,4 +139,4 @@ ${project.artifactId}-${project.version} - \ No newline at end of file + From cddd08eb4fcec9142caecfdabdf147c355543ba2 Mon Sep 17 00:00:00 2001 From: johnnywang Date: Wed, 3 Jun 2020 17:52:48 +0800 Subject: [PATCH 3/8] Update pom.xml Remove useless properties --- ujes/definedEngines/python/enginemanager/pom.xml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ujes/definedEngines/python/enginemanager/pom.xml b/ujes/definedEngines/python/enginemanager/pom.xml index cac5209654..b81d0f3cad 100644 --- a/ujes/definedEngines/python/enginemanager/pom.xml +++ b/ujes/definedEngines/python/enginemanager/pom.xml @@ -32,9 +32,6 @@ - - 0.9.4 - @@ -128,4 +125,4 @@ - \ No newline at end of file + From 2badadf6345bee2405be9cfe517f2c124a0944e1 Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Wed, 3 Jun 2020 20:44:12 +0800 Subject: [PATCH 4/8] add context service client --- contextservice/cs-client/pom.xml | 104 ++++ .../cs/client/AbstractContextClient.java | 9 + .../linkis/cs/client/Context.java | 83 +++ .../linkis/cs/client/ContextClient.java | 120 ++++ .../linkis/cs/client/LinkisContext.java | 9 + .../cs/client/LinkisWorkFlowContext.java | 134 ++++ .../client/builder/ContextClientConfig.java | 9 + .../client/builder/ContextClientFactory.java | 29 + .../builder/HttpContextClientConfig.java | 60 ++ .../builder/RPCHttpContextClientConfig.java | 8 + .../client/http/ContextPostActionBuilder.java | 64 ++ .../cs/client/http/HttpContextClient.java | 588 ++++++++++++++++++ .../cs/client/http/HttpHeartBeater.java | 141 +++++ .../client/listener/ContextClientEvent.java | 11 + .../listener/ContextClientListener.java | 14 + .../listener/ContextClientListenerBus.java | 32 + .../ContextClientListenerManager.java | 25 + .../cs/client/listener/ContextIDListener.java | 75 +++ .../client/listener/ContextKeyListener.java | 68 ++ .../client/listener/ContextUpdateEvent.java | 8 + .../cs/client/listener/HeartBeater.java | 21 + .../cs/client/service/CSMetaDataService.java | 54 ++ .../cs/client/service/CSNodeService.java | 23 + .../cs/client/service/CSNodeServiceImpl.java | 63 ++ .../cs/client/service/CSResourceService.java | 75 +++ .../client/service/CSResultDataService.java | 32 + .../service/CSResultDataServiceImpl.java | 109 ++++ .../cs/client/service/CSTableService.java | 205 ++++++ .../cs/client/service/CSVariableService.java | 94 +++ .../cs/client/service/CSWorkService.java | 30 + .../cs/client/service/CSWorkServiceImpl.java | 93 +++ .../client/service/DefaultSearchService.java | 147 +++++ .../client/service/LinkisJobDataService.java | 30 + .../service/LinkisJobDataServiceImpl.java | 88 +++ .../cs/client/service/MetaDataService.java | 25 + .../cs/client/service/ResourceService.java | 26 + .../cs/client/service/SearchService.java | 39 ++ .../cs/client/service/TableService.java | 59 ++ .../cs/client/service/VariableService.java | 30 + .../cs/client/utils/ContextServiceUtils.java | 192 ++++++ .../cs/client/utils/ExceptionHelper.java | 15 + .../cs/client/utils/SerializeHelper.java | 63 ++ .../exception/ProtocolNotMatchException.scala | 9 + .../linkis/cs/client/http/ContextAction.scala | 97 +++ .../linkis/cs/client/http/ContextResult.scala | 113 ++++ .../cs/client/utils/ContextClientConf.scala | 47 ++ .../cs/client/utils/ContextClientUtils.scala | 18 + .../client/utils/ContextServerHttpConf.scala | 65 ++ .../linkis/cs/client/test/Test.java | 113 ++++ .../test/bean/ClientTestContextKey.java | 68 ++ .../test/bean/ClientTestContextValue.java | 37 ++ .../client/test/bean/ResourceValueBean.java | 34 + .../listener/CommonContextKeyListener.java | 30 + .../cs/client/test/restful/ListenerTest.java | 8 + .../cs/client/test/restful/RestfulTest.java | 92 +++ .../cs/client/test/service/TestInfo.java | 83 +++ .../cs/client/test/service/TestRemove.java | 114 ++++ .../test/service/TestSearchService.java | 112 ++++ .../test_multiuser/TestChangeContext.java | 94 +++ .../test_multiuser/TestCreateContext.java | 91 +++ 60 files changed, 4329 insertions(+) create mode 100644 contextservice/cs-client/pom.xml create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/AbstractContextClient.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/Context.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/ContextClient.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/LinkisContext.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/LinkisWorkFlowContext.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/ContextClientConfig.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/ContextClientFactory.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/HttpContextClientConfig.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/RPCHttpContextClientConfig.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/ContextPostActionBuilder.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/HttpContextClient.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/HttpHeartBeater.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientEvent.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListener.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListenerBus.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListenerManager.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextIDListener.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextKeyListener.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextUpdateEvent.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/HeartBeater.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSMetaDataService.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSNodeService.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSNodeServiceImpl.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResourceService.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResultDataService.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResultDataServiceImpl.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSTableService.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSVariableService.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSWorkService.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSWorkServiceImpl.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/DefaultSearchService.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/LinkisJobDataService.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/LinkisJobDataServiceImpl.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/MetaDataService.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/ResourceService.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/SearchService.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/TableService.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/VariableService.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/ContextServiceUtils.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/ExceptionHelper.java create mode 100644 contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/SerializeHelper.java create mode 100644 contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/exception/ProtocolNotMatchException.scala create mode 100644 contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/http/ContextAction.scala create mode 100644 contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/http/ContextResult.scala create mode 100644 contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextClientConf.scala create mode 100644 contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextClientUtils.scala create mode 100644 contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextServerHttpConf.scala create mode 100644 contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/Test.java create mode 100644 contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ClientTestContextKey.java create mode 100644 contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ClientTestContextValue.java create mode 100644 contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ResourceValueBean.java create mode 100644 contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/listener/CommonContextKeyListener.java create mode 100644 contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/restful/ListenerTest.java create mode 100644 contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/restful/RestfulTest.java create mode 100644 contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestInfo.java create mode 100644 contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestRemove.java create mode 100644 contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestSearchService.java create mode 100644 contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/test_multiuser/TestChangeContext.java create mode 100644 contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/test_multiuser/TestCreateContext.java diff --git a/contextservice/cs-client/pom.xml b/contextservice/cs-client/pom.xml new file mode 100644 index 0000000000..eff02ce3e9 --- /dev/null +++ b/contextservice/cs-client/pom.xml @@ -0,0 +1,104 @@ + + + + + + linkis + com.webank.wedatasphere.linkis + 0.9.4 + ../../pom.xml + + 4.0.0 + + linkis-cs-client + + + + com.webank.wedatasphere.linkis + linkis-common + ${linkis.version} + + + + com.webank.wedatasphere.linkis + linkis-gateway-httpclient-support + ${linkis.version} + + + com.webank.wedatasphere.linkis + linkis-storage + + + + + + com.webank.wedatasphere.linkis + linkis-httpclient + ${linkis.version} + + + + com.webank.wedatasphere.linkis + linkis-cs-common + ${linkis.version} + + + + + + + + + + com.webank.wedatasphere.linkis + linkis-cs-listener + ${linkis.version} + + + + junit + junit + 4.12 + test + + + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + ${basedir}/src/main/resources + + + + + \ No newline at end of file diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/AbstractContextClient.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/AbstractContextClient.java new file mode 100644 index 0000000000..53ad3c4261 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/AbstractContextClient.java @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.linkis.cs.client; + +/** + * created by cooperyang on 2020/2/11 + * Description: + */ +public abstract class AbstractContextClient implements ContextClient{ + +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/Context.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/Context.java new file mode 100644 index 0000000000..4226d43344 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/Context.java @@ -0,0 +1,83 @@ +package com.webank.wedatasphere.linkis.cs.client; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.cs.client.listener.ContextIDListener; +import com.webank.wedatasphere.linkis.cs.client.listener.ContextKeyListener; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue; + +import java.util.List; + + +/** + * created by cooperyang on 2020/2/10 + * Description:Context是一个接口让用户能够在微服务中进行对一个工作流的Context进行操作 + * 他应该有一个默认的实现，比如LinkisWorkFlowContext + */ +public interface Context { + + /** + * 每一个Context都应该有一个ContextID与之对应d + * @return contextID是该Context对应的ContextID + */ + public ContextID getContextID(); + + /** + * 设置ContextID + * @param contextID contextID + */ + public void setContextID(ContextID contextID); + + /** + * 通过contextKey来获取对应的contextValue + * @param contextKey 是标识一个ContextValue的一个key值，比如资源文件的名称 + * @return 返回key对应的value, 比如resourceid 和 version + */ + public ContextValue getContextValue(ContextKey contextKey) throws ErrorException; + + /** + * 设置contextKeyValue + * @param contextKeyAndValue 需要设置的ContextKeyValue + * @throws ErrorException 可能由于网络原因出现的 + */ + + public void setContextKeyAndValue(ContextKeyValue contextKeyAndValue) throws ErrorException; + + + + public void set(ContextKey contextKey, ContextValue contextValue) throws ErrorException; + + public void setLocal(ContextKey contextKey, ContextValue contextValue); + + public void setLocal(ContextKeyValue contextKeyValue); + + /** + * todo 是通过ContextCondition进行搜索 + * @return + */ + public List searchContext(List contextTypes, + List contextScopes, + List contains, + List regex) throws ErrorException; + + + public void reset(ContextKey contextKey) throws ErrorException; + + public void reset() throws ErrorException; + + + public void remove(ContextKey contextKey) throws ErrorException; + + public void removeAll()throws ErrorException; + + public void onBind(ContextIDListener contextIDListener) throws ErrorException; + + public void onBind(ContextKey contextKey, ContextKeyListener contextKeyListener) throws ErrorException; + + + +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/ContextClient.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/ContextClient.java new file mode 100644 index 0000000000..cc9b431fe4 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/ContextClient.java @@ -0,0 +1,120 @@ +package com.webank.wedatasphere.linkis.cs.client; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.cs.client.listener.ContextIDListener; +import com.webank.wedatasphere.linkis.cs.client.listener.ContextKeyListener; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue; + +import java.io.Closeable; +import java.util.List; +import java.util.Map; + +/** + * created by cooperyang on 2020/2/10 + * Description: + */ +public interface ContextClient extends Closeable { + /** + * 通过与cs-server进行交互生成一个工作流的Context + * 传入的信息应该有工程名 工作流名 + * user是传入的用户名 + * @return 用户可以使用的Context + */ + @Deprecated + Context createContext(String projectName, String flowName, String use, Map params) throws ErrorException; + + + Context createContext(ContextID contextID) throws ErrorException; + + + /** + * 通过ContextID获取 + * @param contextID + * @return + * @throws ErrorException + */ + Context getContext(ContextID contextID) throws ErrorException; + + Context getContext(String contextIDStr) throws ErrorException; + + ContextValue getContextValue(ContextID contextID, ContextKey contextKey) throws ErrorException; + + + + void update(ContextID contextID, ContextKey contextKey, ContextValue contextValue) throws ErrorException; + + + /** + * 通过contextID和contextKey进行对某一个contextKey进行reset + * @param contextID contextId + * @param contextKey contexKey + * @throws ErrorException 可能捕获的异常 + */ + void reset(ContextID contextID, ContextKey contextKey) throws ErrorException; + + + /** + * 将整个contextID所有的contextKey进行reset,这个会用在工作流实时执行之前的一个reset + * @param contextID contextID + * @throws ErrorException + */ + void reset(ContextID contextID) throws ErrorException; + + + + + + /** + * 删除的操作是为了能够将contextid下面的contextkey进行删除 + * 如果contextKey是空的话，则全部删除 + * @param contextID + * @param contextKey + * @throws ErrorException + */ + void remove(ContextID contextID, ContextKey contextKey) throws ErrorException; + + void setContextKeyValue(ContextID contextID, ContextKeyValue contextKeyValue) throws ErrorException; + + void bindContextIDListener(ContextIDListener contextIDListener) throws ErrorException; + + void bindContextKeyListener(ContextKeyListener contextKeyListener) throws ErrorException; + + /** + * 通过各种condition搜索contextkeyValue + * @return 一个contextKeyValue数组 + * @throws ErrorException 可能出现的error + */ + List search(ContextID contextID, List contextTypes, + List contextScopes, + List contains, + List regex) throws ErrorException; + + /** + * 通过各种condition搜索contextkeyValue + * upstreamOnly 表示取跟当前节点上游中查询 + * nodeName 获取该节点的关系节点 + * num 获取的节点数量，1 返回最近的一个， 无穷大表示所有节点 + * @return 一个contextKeyValue数组 + * @throws ErrorException 可能出现的error + */ + List search(ContextID contextID, List contextTypes, + List contextScopes, + List contains, + List regex, + boolean upstreamOnly, + String nodeName, + int num, + List contextValueTypes) throws ErrorException; + + + + void removeAllValueByKeyPrefixAndContextType(ContextID contextID, ContextType contextType, String keyPrefix) throws ErrorException; + + void removeAllValueByKeyPrefix(ContextID contextID, String keyPrefix) throws ErrorException; + +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/LinkisContext.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/LinkisContext.java new file mode 100644 index 0000000000..565d6f1dec --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/LinkisContext.java @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.linkis.cs.client; + +/** + * created by cooperyang on 2020/2/17 + * Description: + */ +public abstract class LinkisContext implements Context{ + +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/LinkisWorkFlowContext.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/LinkisWorkFlowContext.java new file mode 100644 index 0000000000..ccc0eecd78 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/LinkisWorkFlowContext.java @@ -0,0 +1,134 @@ +package com.webank.wedatasphere.linkis.cs.client; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.cs.client.listener.ContextIDListener; +import com.webank.wedatasphere.linkis.cs.client.listener.ContextKeyListener; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; +import com.webank.wedatasphere.linkis.cs.common.entity.source.*; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * created by cooperyang on 2020/2/17 + * Description: + */ +public class LinkisWorkFlowContext extends LinkisContext{ + + private ContextID contextID; + + private ContextClient contextClient; + + private String user; + + + private Map keyValues = new ConcurrentHashMap<>(); + + @Override + public ContextID getContextID() { + return this.contextID; + } + + @Override + public void setContextID(ContextID contextID) { + this.contextID = contextID; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + @Override + public ContextValue getContextValue(ContextKey contextKey) throws ErrorException{ + if(keyValues.containsKey(contextKey)){ + return keyValues.get(contextKey); + }else{ + return this.contextClient.getContextValue(this.contextID, contextKey); + } + } + + + @Override + public void setContextKeyAndValue(ContextKeyValue contextKeyValue) throws ErrorException{ + //todo 这个地方是为了进行一次缓存 不过有待商榷 + //this.setLocal(contextKeyValue); + this.contextClient.setContextKeyValue(this.contextID, contextKeyValue); + } + + @Override + public void set(ContextKey contextKey, ContextValue contextValue) throws ErrorException{ + setLocal(contextKey, contextValue); + ContextKeyValue contextKeyValue = new CommonContextKeyValue(); + contextKeyValue.setContextKey(contextKey); + contextKeyValue.setContextValue(contextValue); + this.setContextKeyAndValue(contextKeyValue); + } + + + @Override + public void setLocal(ContextKey contextKey, ContextValue contextValue) { + this.keyValues.put(contextKey, contextValue); + } + + @Override + public void setLocal(ContextKeyValue contextKeyValue) { + this.keyValues.put(contextKeyValue.getContextKey(), contextKeyValue.getContextValue()); + } + + @Override + public List searchContext(List contextTypes, + List contextScopes, + List contains, + List regex) throws ErrorException { + return this.contextClient.search(contextID, contextTypes, contextScopes, contains, regex); + } + + + @Override + public void reset(ContextKey contextKey) throws ErrorException{ + this.contextClient.reset(contextID, contextKey); + } + + @Override + public void reset() throws ErrorException { + this.contextClient.reset(this.contextID); + } + + @Override + public void remove(ContextKey contextKey) throws ErrorException{ + this.contextClient.remove(contextID, contextKey); + } + + @Override + public void removeAll()throws ErrorException { + this.contextClient.remove(contextID, null); + } + + @Override + public void onBind(ContextIDListener contextIDListener) throws ErrorException{ + contextIDListener.setContextID(this.contextID); + contextIDListener.setContext(this); + this.contextClient.bindContextIDListener(contextIDListener); + } + + @Override + public void onBind(ContextKey contextKey, ContextKeyListener contextKeyListener) throws ErrorException { + contextKeyListener.setContextKey(contextKey); + contextKeyListener.setContext(this); + this.contextClient.bindContextKeyListener(contextKeyListener); + } + + public ContextClient getContextClient() { + return contextClient; + } + + public void setContextClient(ContextClient contextClient) { + this.contextClient = contextClient; + } +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/ContextClientConfig.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/ContextClientConfig.java new file mode 100644 index 0000000000..1818bd37f9 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/ContextClientConfig.java @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.linkis.cs.client.builder; + +/** + * created by cooperyang on 2020/2/10 + * Description: + */ +public abstract class ContextClientConfig { + +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/ContextClientFactory.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/ContextClientFactory.java new file mode 100644 index 0000000000..889af65013 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/ContextClientFactory.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.linkis.cs.client.builder; + +import com.webank.wedatasphere.linkis.cs.client.ContextClient; +import com.webank.wedatasphere.linkis.cs.client.http.HttpContextClient; + +/** + * created by cooperyang on 2020/2/10 + * Description: + */ +public class ContextClientFactory { + + + private static final ContextClientConfig DEFAULT_CONTEXT_CLIENT_CONFIG; + + static{ + DEFAULT_CONTEXT_CLIENT_CONFIG = new HttpContextClientConfig(); + } + + public static ContextClient getOrCreateContextClient(){ + return getOrCreateContextClient(DEFAULT_CONTEXT_CLIENT_CONFIG); + } + + public static ContextClient getOrCreateContextClient(ContextClientConfig contextClientConfig){ + return HttpContextClient.getInstance(contextClientConfig); + } + + + +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/HttpContextClientConfig.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/HttpContextClientConfig.java new file mode 100644 index 0000000000..962527a631 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/HttpContextClientConfig.java @@ -0,0 +1,60 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.cs.client.builder; + +import com.webank.wedatasphere.linkis.common.conf.Configuration; +import com.webank.wedatasphere.linkis.cs.client.utils.ContextClientConf; +import com.webank.wedatasphere.linkis.httpclient.authentication.AuthenticationStrategy; +import com.webank.wedatasphere.linkis.httpclient.config.ClientConfig; +import com.webank.wedatasphere.linkis.httpclient.config.ClientConfigBuilder; +import com.webank.wedatasphere.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy; + +/** + * created by cooperyang on 2020/2/10 + * Description: 以http的方式和cs-server进行交互的配置,包括http的诸多配置 + */ +public class HttpContextClientConfig extends ContextClientConfig{ + + + private ClientConfig clientConfig; + + + public HttpContextClientConfig(){ + //初始化clientConfig + String gatewayUrl = Configuration.getGateWayURL(); + AuthenticationStrategy authenticationStrategy = new TokenAuthenticationStrategy(); + int maxConnection = 10; + int connectionTimeout = 3000; + int readTimeout = 10000; + clientConfig = ClientConfigBuilder.newBuilder().addUJESServerUrl(gatewayUrl). + connectionTimeout(connectionTimeout).discoveryEnabled(false).loadbalancerEnabled(false). + maxConnectionSize(maxConnection).retryEnabled(false).readTimeout(readTimeout) + .setAuthenticationStrategy(authenticationStrategy).setAuthTokenKey(ContextClientConf.CONTEXT_CLIENT_AUTH_KEY().getValue()) + .setAuthTokenValue(ContextClientConf.CONTEXT_CLIENT_AUTH_VALUE().getValue()).build(); + + } + + + + /** + * + * @return 返回一个的 + */ + public ClientConfig getClientConfig(){ + return this.clientConfig; + } + + + +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/RPCHttpContextClientConfig.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/RPCHttpContextClientConfig.java new file mode 100644 index 0000000000..fdd3b1ab12 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/builder/RPCHttpContextClientConfig.java @@ -0,0 +1,8 @@ +package com.webank.wedatasphere.linkis.cs.client.builder; + +/** + * created by cooperyang on 2020/2/10 + * Description: + */ +public class RPCHttpContextClientConfig { +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/ContextPostActionBuilder.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/ContextPostActionBuilder.java new file mode 100644 index 0000000000..576eec10f7 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/ContextPostActionBuilder.java @@ -0,0 +1,64 @@ +package com.webank.wedatasphere.linkis.cs.client.http; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID; +import com.webank.wedatasphere.linkis.cs.common.protocol.ContextHTTPConstant; + +import java.util.HashMap; +import java.util.Map; + +/** + * Created by v_wbjftang on 2020/3/4. + */ +public class ContextPostActionBuilder { + + private final DefaultContextPostAction action; + + private Map requestParams = new HashMap(4); + private Map headerParams = new HashMap<>(2); + + public ContextPostActionBuilder(String url) { + action = new DefaultContextPostAction(url); + } + + public static ContextPostActionBuilder of(String url) { + return new ContextPostActionBuilder(url); + } + + public ContextPostActionBuilder with(ContextID contextID) throws ErrorException { + String contextIDStr = SerializeHelper.serializeContextID(contextID); + requestParams.put(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr); + return this; + } + + + public ContextPostActionBuilder with(String key, Object object) throws ErrorException { + requestParams.put(key, object); + return this; + } + + public ContextPostActionBuilder addHeader(String key, String value) throws ErrorException { + headerParams.put(key, value); + return this; + } + + /* public ContextPostActionBuilder with(ContextKey contextKey) { + return this; + } + + public ContextPostActionBuilder with(ContextKeyValue keyValue) { + return this; + } + + public ContextPostActionBuilder with(ContextValue value) { + return this; + }*/ + + public DefaultContextPostAction build() { + action.getRequestPayloads().putAll(requestParams); + action.getHeaders().putAll(headerParams); + return action; + } + +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/HttpContextClient.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/HttpContextClient.java new file mode 100644 index 0000000000..c4c47c7a0d --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/HttpContextClient.java @@ -0,0 +1,588 @@ +package com.webank.wedatasphere.linkis.cs.client.http; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.cs.client.AbstractContextClient; +import com.webank.wedatasphere.linkis.cs.client.Context; +import com.webank.wedatasphere.linkis.cs.client.LinkisWorkFlowContext; +import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientConfig; +import com.webank.wedatasphere.linkis.cs.client.builder.HttpContextClientConfig; +import com.webank.wedatasphere.linkis.cs.client.listener.ContextIDListener; +import com.webank.wedatasphere.linkis.cs.client.listener.ContextKeyListener; +import com.webank.wedatasphere.linkis.cs.client.listener.HeartBeater; +import com.webank.wedatasphere.linkis.cs.client.utils.ContextClientConf; +import com.webank.wedatasphere.linkis.cs.client.utils.ContextServerHttpConf; +import com.webank.wedatasphere.linkis.cs.client.utils.ExceptionHelper; +import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; +import com.webank.wedatasphere.linkis.cs.common.entity.source.*; +import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException; +import com.webank.wedatasphere.linkis.cs.common.protocol.ContextHTTPConstant; +import com.webank.wedatasphere.linkis.cs.common.search.ContextSearchConditionMapBuilder; +import com.webank.wedatasphere.linkis.httpclient.config.ClientConfig; +import com.webank.wedatasphere.linkis.httpclient.dws.DWSHttpClient; +import com.webank.wedatasphere.linkis.httpclient.dws.config.DWSClientConfig; +import com.webank.wedatasphere.linkis.httpclient.dws.response.DWSResult; +import com.webank.wedatasphere.linkis.httpclient.request.Action; +import com.webank.wedatasphere.linkis.httpclient.response.Result; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * created by cooperyang on 2020/2/11 + * Description: HttpContextClient是ContextClient的使用Http方式进行通信的具体实现 + * 一般可以将其做成单例 + */ +public class HttpContextClient extends AbstractContextClient { + + + private static final Logger LOGGER = LoggerFactory.getLogger(HttpContextClient.class); + + private DWSHttpClient dwsHttpClient; + private ContextClientConfig contextClientConfig; + private final String linkis_version = ContextClientConf.LINKIS_WEB_VERSION().getValue(); + + private final String name = "HttpContextClient"; + + private static HttpContextClient httpContextClient; + + private HeartBeater heartBeater; + + + + + private HttpContextClient(){ + + } + + private HttpContextClient(ContextClientConfig contextClientConfig){ + //初始化dwsHttpClient + this.contextClientConfig = contextClientConfig; + if (contextClientConfig instanceof HttpContextClientConfig){ + HttpContextClientConfig httpContextClientConfig = (HttpContextClientConfig)contextClientConfig; + ClientConfig clientConfig = httpContextClientConfig.getClientConfig(); + DWSClientConfig dwsClientConfig = new DWSClientConfig(clientConfig); + dwsClientConfig.setDWSVersion(linkis_version); + dwsHttpClient = new DWSHttpClient(dwsClientConfig, name); + } + if ("true".equals(ContextClientConf.HEART_BEAT_ENABLED().getValue())){ + this.heartBeater = new HttpHeartBeater(contextClientConfig); + heartBeater.start(); + } + } + + + public static HttpContextClient getInstance(ContextClientConfig contextClientConfig){ + if (httpContextClient == null){ + synchronized (HttpContextClient.class){ + if (httpContextClient == null){ + httpContextClient = new HttpContextClient(contextClientConfig); + } + } + } + return httpContextClient; + } + + + @Override + @Deprecated + public Context createContext(String projectName, String flowName, String user, Map params) throws ErrorException{ + ContextCreateAction contextCreateAction = new ContextCreateAction(); + LinkisHAWorkFlowContextID contextID = new LinkisHAWorkFlowContextID(); + contextID.setProject(projectName); + contextID.setFlow(flowName); + String contextIDStr = SerializeHelper.serializeContextID(contextID); + contextCreateAction.addHeader(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr); + contextCreateAction.getRequestPayloads().put(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr); + Result result = null; + try{ + result = dwsHttpClient.execute(contextCreateAction); + }catch(Exception e) { + LOGGER.error("create context failed", e); + ExceptionHelper.throwErrorException(80015, "create context failed", e); + } + if (result instanceof ContextCreateResult){ + ContextCreateResult contextCreateResult = (ContextCreateResult)result; + int status = contextCreateResult.getStatus(); + if (status != 0){ + String errMsg = contextCreateResult.getMessage(); + LOGGER.error("create context for project {}, flow {} failed, msg is {}", projectName, flowName, errMsg); + throw new ErrorException(80014, "create context failed" + errMsg); + }else{ + LinkisWorkFlowContext context = new LinkisWorkFlowContext(); + Map map = contextCreateResult.getData(); + contextID.setContextId(map.get("contextId").toString()); + context.setContextID(contextID); + context.setContextClient(this); + context.setUser(user); + return context; + } + }else if (result != null){ + LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName()); + throw new ErrorException(80015, "result is not a correct type"); + }else{ + LOGGER.error("result is null"); + throw new ErrorException(80015, "result is null"); + } + } + + + @Override + public Context createContext(ContextID contextID) throws ErrorException { + ContextCreateAction contextCreateAction = new ContextCreateAction(); + String contextIDStr = SerializeHelper.serializeContextID(contextID); + contextCreateAction.addHeader(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr); + contextCreateAction.getRequestPayloads().put(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr); + Result result = null; + try{ + result = dwsHttpClient.execute(contextCreateAction); + }catch(Exception e) { + LOGGER.error("create context failed", e); + ExceptionHelper.throwErrorException(80015, "create context failed", e); + } + if (result instanceof ContextCreateResult){ + ContextCreateResult contextCreateResult = (ContextCreateResult)result; + int status = contextCreateResult.getStatus(); + if (status != 0){ + String errMsg = contextCreateResult.getMessage(); + LOGGER.error("create context failed, msg is {}", errMsg); + throw new ErrorException(80014, "create context failed" + errMsg); + }else{ + LinkisWorkFlowContext context = new LinkisWorkFlowContext(); + Map map = contextCreateResult.getData(); + contextID.setContextId(map.get("contextId").toString()); + context.setContextID(contextID); + context.setContextClient(this); + return context; + } + }else if (result != null){ + LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName()); + throw new ErrorException(80015, "result is not a correct type"); + }else{ + LOGGER.error("result is null"); + throw new ErrorException(80015, "result is null"); + } + } + + @Override + public Context getContext(ContextID contextId) throws ErrorException{ + LinkisWorkFlowContext context = new LinkisWorkFlowContext(); + context.setContextID(contextId); + context.setContextClient(this); + return context; + } + + @Override + public Context getContext(String contextIDStr) throws ErrorException { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + return getContext(contextID); + } + + @Override + public ContextValue getContextValue(ContextID contextID, ContextKey contextKey) throws ErrorException { + String contextIDStr = SerializeHelper.serializeContextID(contextID); + String contextKeyStr = SerializeHelper.serializeContextKey(contextKey); + ContextGetValueAction contextGetValueAction = new ContextGetValueAction(); + contextGetValueAction.addHeader(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr); + contextGetValueAction.getRequestPayloads().put(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr); + contextGetValueAction.getRequestPayloads().put(ContextHTTPConstant.CONTEXT_KEY_STR, contextKeyStr); + Result result = null; + try{ + result = dwsHttpClient.execute(contextGetValueAction); + }catch(Exception e) { + LOGGER.error("get context value id: {} , key: {} failed", contextIDStr, contextKeyStr, e); + ExceptionHelper.throwErrorException(80015, "get context value failed", e); + } + if (result instanceof ContextGetValueResult){ + ContextGetValueResult contextGetValueResult = (ContextGetValueResult)result; + int status = contextGetValueResult.getStatus(); + if (status != 0){ + String errMsg = contextGetValueResult.getMessage(); + LOGGER.error("get context value id: {} , key: {} failed, msg is {}", contextIDStr, contextKeyStr, errMsg); + throw new ErrorException(80014, "create context failed" + errMsg); + }else{ + Map map = contextGetValueResult.getData(); + if( null == map || null == map.get("contextValue") ){ + return null; + } + String contextValueStr = map.get("contextValue").toString(); + return SerializeHelper.deserializeContextValue(contextValueStr); + } + }else if (result != null){ + LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName()); + throw new ErrorException(80015, "result is not a correct type"); + }else{ + LOGGER.error("result is null"); + throw new ErrorException(80015, "result is null"); + } + } + + + @Override + public void update(ContextID contextID, ContextKey contextKey, ContextValue contextValue) throws ErrorException { + String contextIdStr = SerializeHelper.SERIALIZE_HELPER.serialize(contextID); + String contextKeyValueStr = SerializeHelper.SERIALIZE_HELPER.serialize(new CommonContextKeyValue(contextKey, contextValue)); + ContextSetKeyValueAction contextSetKeyValueAction = new ContextSetKeyValueAction(); + contextSetKeyValueAction.addHeader(ContextHTTPConstant.CONTEXT_ID_STR, contextIdStr); + contextSetKeyValueAction.getRequestPayloads().put("contextID", contextIdStr); + contextSetKeyValueAction.getRequestPayloads().put("contextKeyValue", contextKeyValueStr); + Result result = null; + try{ + result = dwsHttpClient.execute(contextSetKeyValueAction); + }catch(Exception e) { + LOGGER.error("update context failed", e); + ExceptionHelper.throwErrorException(80015, "update context failed", e); + } + if (result instanceof ContextSetKeyValueResult){ + ContextSetKeyValueResult contextSetKeyValueResult = (ContextSetKeyValueResult)result; + int status = contextSetKeyValueResult.getStatus(); + if (status != 0){ + String errMsg = contextSetKeyValueResult.getMessage(); + LOGGER.error("调用客户端去更新contextId {} 失败, 返回的错误信息是 {} ", contextIdStr, errMsg); + } + }else if (result != null){ + LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName()); + throw new ErrorException(80015, "result is not a correct type"); + }else{ + LOGGER.error("result is null"); + throw new ErrorException(80015, "result is null"); + } + } + + @Override + public void reset(ContextID contextID, ContextKey contextKey) throws ErrorException{ + String contextIdStr = SerializeHelper.SERIALIZE_HELPER.serialize(contextID); + String contextKeyStr = SerializeHelper.SERIALIZE_HELPER.serialize(contextKey); + ContextResetValueAction contextResetValueAction = new ContextResetValueAction(); + contextResetValueAction.getRequestPayloads().put("contextKey", contextKeyStr); + contextResetValueAction.getRequestPayloads().put("contextID", contextIdStr); + // contextResetValueAction.getParameters().put("contextId", contextID.getContextId()); + Result result = null; + try{ + result = dwsHttpClient.execute(contextResetValueAction); + }catch(Exception e) { + LOGGER.error("reset contextID {}, contextKey {} failed", contextIdStr, contextKeyStr, e); + ExceptionHelper.throwErrorException(80015, "reset context failed", e); + } + if (result instanceof ContextResetResult){ + ContextResetResult contextResetResult = (ContextResetResult)result; + int status = contextResetResult.getStatus(); + if (status != 0){ + String errMsg = contextResetResult.getMessage(); + LOGGER.error("调用客户端去reset contextId {}, contextKey {} 失败, 返回的错误信息是 {} ", contextIdStr, contextKeyStr,errMsg); + throw new ErrorException(80015, "reset contextID failed"); + } + }else if (result != null){ + LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName()); + throw new ErrorException(80015, "result is not a correct type"); + }else{ + LOGGER.error("result is null"); + throw new ErrorException(80015, "result is null"); + } + } + + @Override + public void reset(ContextID contextID) throws ErrorException { + String contextIdStr = SerializeHelper.serializeContextID(contextID); + ContextResetIDAction contextResetIDAction = new ContextResetIDAction(); + contextResetIDAction.getRequestPayloads().put(ContextHTTPConstant.CONTEXT_ID_STR, contextIdStr); + //contextResetIDAction.getParameters().put("contextId", contextID.getContextId()); + Result result = null; + try{ + result = dwsHttpClient.execute(contextResetIDAction); + }catch(Exception e) { + LOGGER.error("reset contextID {} failed", contextIdStr, e); + ExceptionHelper.throwErrorException(80015, "reset context failed", e); + } + if (result instanceof ContextResetIDResult){ + ContextResetIDResult contextResetResult = (ContextResetIDResult)result; + int status = contextResetResult.getStatus(); + if (status != 0){ + String errMsg = contextResetResult.getMessage(); + LOGGER.error("调用客户端去reset contextId {} 失败, 返回的错误信息是 {} ", contextIdStr,errMsg); + throw new ErrorException(80015, "reset contextID failed"); + } + }else if (result != null){ + LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName()); + throw new ErrorException(80015, "result is not a correct type"); + }else{ + LOGGER.error("result is null"); + throw new ErrorException(80015, "result is null"); + } + } + + @Override + public void remove(ContextID contextID, ContextKey contextKey) throws ErrorException{ + String contextIdStr = SerializeHelper.serializeContextID(contextID); + String contextKeyStr = SerializeHelper.serializeContextKey(contextKey); + ContextRemoveAction contextRemoveAction = new ContextRemoveAction(contextIdStr, contextKeyStr); + contextRemoveAction.addHeader(ContextHTTPConstant.CONTEXT_ID_STR, contextIdStr); + contextRemoveAction.getRequestPayloads().put(ContextHTTPConstant.CONTEXT_KEY_STR, contextKeyStr); + contextRemoveAction.getRequestPayloads().put(ContextHTTPConstant.CONTEXT_ID_STR, contextIdStr); + contextRemoveAction.getRequestPayloads().put("contextId", contextID.getContextId()); + Result result = null; + try{ + result = dwsHttpClient.execute(contextRemoveAction); + }catch(Exception e) { + LOGGER.error("remove context id {} context key {} failed", contextIdStr, contextIdStr, e); + ExceptionHelper.throwErrorException(80015, "remove context failed", e); + } + if (result instanceof ContextRemoveResult){ + ContextRemoveResult contextRemoveResult = (ContextRemoveResult)result; + int status = contextRemoveResult.getStatus(); + if (status != 0){ + String errMsg = contextRemoveResult.getMessage(); + LOGGER.error("remove context failed contextID {}, contextKey {} ", contextIdStr, contextKey.getKey()); + } + }else if (result != null){ + LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName()); + throw new ErrorException(80015, "result is not a correct type"); + }else{ + LOGGER.error("result is null"); + throw new ErrorException(80015, "result is null"); + } + } + + @Override + public void setContextKeyValue(ContextID contextID, ContextKeyValue contextKeyValue) throws ErrorException { + String contextIDStr = SerializeHelper.SERIALIZE_HELPER.serialize(contextID); + String contextKeyValueStr = SerializeHelper.SERIALIZE_HELPER.serialize(contextKeyValue); + ContextSetKeyValueAction action = new ContextSetKeyValueAction(); + action.addHeader(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr); + action.getRequestPayloads().put("contextID", contextIDStr); + action.getRequestPayloads().put("contextKeyValue", contextKeyValueStr); + Result result = null; + try{ + result = dwsHttpClient.execute(action); + }catch(Exception e) { + LOGGER.error("set value failed", e); + ExceptionHelper.throwErrorException(80015, "update context failed", e); + } + if (result instanceof ContextSetKeyValueResult){ + ContextSetKeyValueResult contextSetKeyValueResult = (ContextSetKeyValueResult)result; + int status = contextSetKeyValueResult.getStatus(); + if (status != 0){ + String errMsg = contextSetKeyValueResult.getMessage(); + LOGGER.error("set value failed {} ,err is {}", contextIDStr, errMsg); + } + }else if (result != null){ + LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName()); + throw new ErrorException(80015, "result is not a correct type"); + }else{ + LOGGER.error("result is null"); + throw new ErrorException(80015, "result is null"); + } + } + + @Override + public void bindContextIDListener(ContextIDListener contextIDListener) throws ErrorException{ + ContextID contextID = contextIDListener.getContextID(); + String contextIDStr = SerializeHelper.serializeContextID(contextID); + ContextBindIDAction contextBindIDAction = new ContextBindIDAction(); + contextBindIDAction.getRequestPayloads().put(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr); + // contextBindIDAction.getParameters().put("contextId", contextID.getContextId()); + Result result = null; + try{ + result = dwsHttpClient.execute(contextBindIDAction); + }catch(Exception e) { + LOGGER.error("bind context id {} failed", contextIDStr, e); + ExceptionHelper.throwErrorException(80015, "bind context id failed", e); + } + if (result instanceof ContextBindIDResult){ + ContextBindIDResult contextBindIDResult = (ContextBindIDResult)result; + int status = contextBindIDResult.getStatus(); + if (status != 0){ + String errMsg = contextBindIDResult.getMessage(); + LOGGER.error("bind context id failed {} ,err is {}", contextIDStr, errMsg); + } + }else if (result != null){ + LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName()); + throw new ErrorException(80015, "result is not a correct type"); + }else{ + LOGGER.error("result is null"); + throw new ErrorException(80015, "result is null"); + } + } + + @Override + public void bindContextKeyListener(ContextKeyListener contextKeyListener) throws ErrorException{ + ContextID contextID = contextKeyListener.getContext().getContextID(); + ContextKey contextKey =contextKeyListener.getContextKey(); + String contextIDStr = SerializeHelper.serializeContextID(contextID); + String contextKeyStr = SerializeHelper.serializeContextKey(contextKey); + ContextBindKeyAction contextBindKeyAction = new ContextBindKeyAction(); + contextBindKeyAction.addHeader(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr); + contextBindKeyAction.getRequestPayloads().put(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr); + contextBindKeyAction.getRequestPayloads().put(ContextHTTPConstant.CONTEXT_KEY_STR, contextKeyStr); + //todo 这里要改一下source的来历 + contextBindKeyAction.getRequestPayloads().put("source", name); + Result result = null; + try{ + result = dwsHttpClient.execute(contextBindKeyAction); + }catch(Exception e) { + LOGGER.error("bind context id {} context key {} failed", contextIDStr, contextKeyStr, e); + ExceptionHelper.throwErrorException(80015, "bind context key failed", e); + } + if (result instanceof ContextBindKeyResult){ + ContextBindKeyResult contextBindKeyResult = (ContextBindKeyResult)result; + int status = contextBindKeyResult.getStatus(); + if (status != 0){ + String errMsg = contextBindKeyResult.getMessage(); + LOGGER.error("bind context id {} context key {} failed ,err is {}", contextIDStr, contextKeyStr, errMsg); + } + }else if (result != null){ + LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName()); + throw new ErrorException(80015, "result is not a correct type"); + }else{ + LOGGER.error("result is null"); + throw new ErrorException(80015, "result is null"); + } + } + + @Override + public List search(ContextID contextID, + List contextTypes, + List contextScopes, + List contains, + List regex) throws ErrorException { + return search(contextID, contextTypes, contextScopes, contains, regex, false, null, Integer.MAX_VALUE, null); + } + + @Override + public List search(ContextID contextID, + List contextTypes, + List contextScopes, + List contains, + List regex, + boolean upstream, + String nodeName, + int num, + List contextValueTypes) throws ErrorException { + ContextSearchConditionMapBuilder builder = ContextSearchConditionMapBuilder.newBuilder(); + if (contextTypes != null){ + contextTypes.forEach(builder::contextTypes); + } + if (contextScopes != null){ + contextScopes.forEach(builder::contextScopes); + } + if (contains != null){ + contains.forEach(builder::contains); + } + if (regex != null){ + regex.forEach(builder::regex); + } + builder.nearest(nodeName, num, upstream); + if (contextValueTypes != null) { + contextValueTypes.forEach(builder::contextValueTypes); + } + String contextIDStr = SerializeHelper.serializeContextID(contextID); + ContextSearchContextAction contextSearchContextAction = new ContextSearchContextAction(); + contextSearchContextAction.addHeader(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr); + contextSearchContextAction.getRequestPayloads().put("condition", builder.build()); + contextSearchContextAction.getRequestPayloads().put("contextID", contextIDStr); + Result result = null; + try{ + result = dwsHttpClient.execute(contextSearchContextAction); + }catch(Exception e) { + LOGGER.error("search condition failed", e); + ExceptionHelper.throwErrorException(80015, "search condition failed", e); + } + if (result instanceof ContextSearchResult){ + ContextSearchResult contextSearchResult = (ContextSearchResult)result; + int status = contextSearchResult.getStatus(); + if (status != 0){ + String errMsg = contextSearchResult.getMessage(); + LOGGER.error("search condition failed, err is {}", errMsg); + }else{ + Map data = contextSearchResult.getData(); + if(data.get("contextKeyValue") != null){ + List retKvs = new ArrayList<>(); + Object o = data.get("contextKeyValue"); + List list = (List)o; + list.stream().map(s -> { + try{ + return SerializeHelper.deserializeContextKeyValue(s); + }catch(ErrorException e){ + LOGGER.error("failed to deserialize {} to a contextKeyValue", s, e); + return null; + } + }).filter(Objects::nonNull).forEach(retKvs::add); + return retKvs; + } + } + }else if (result != null){ + LOGGER.error("result is not a correct type, result type is {}", result.getClass().getSimpleName()); + throw new ErrorException(80015, "result is not a correct type"); + }else{ + LOGGER.error("result is null"); + throw new ErrorException(80015, "result is null"); + } + return null; + } + + + + @Override + public void removeAllValueByKeyPrefixAndContextType(ContextID contextID, ContextType contextType, String keyPrefix) throws ErrorException { + String contextIDStr = SerializeHelper.serializeContextID(contextID); + DefaultContextPostAction action = ContextPostActionBuilder.of(ContextServerHttpConf.removeAllValueByKeyPrefixAndContextTypeURL()) + .with(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr).with(ContextHTTPConstant.CONTEXT_KEY_TYPE_STR, contextType.toString()) + .with(ContextHTTPConstant.CONTEXT_KEY_PREFIX_STR, keyPrefix).addHeader(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr).build(); + checkDWSResult(execute(action)); + } + + @Override + public void removeAllValueByKeyPrefix(ContextID contextID, String keyPrefix) throws ErrorException { + String contextIDStr = SerializeHelper.serializeContextID(contextID); + DefaultContextPostAction action = ContextPostActionBuilder.of(ContextServerHttpConf.removeAllValueByKeyPrefixURL()) + .with(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr).with(ContextHTTPConstant.CONTEXT_KEY_PREFIX_STR, keyPrefix) + .addHeader(ContextHTTPConstant.CONTEXT_ID_STR, contextIDStr).build(); + checkDWSResult(execute(action)); + } + + private Result execute(Action action) throws ErrorException { + try{ + return dwsHttpClient.execute(action); + }catch(Exception e) { + LOGGER.error("execute failed", e); + ExceptionHelper.throwErrorException(80015, "execute failed", e); + } + return null; + } + + private DWSResult checkDWSResult(Result result) throws CSErrorException { + if(result instanceof DWSResult){ + int status = ((DWSResult) result).getStatus(); + if (status != 0){ + String errMsg = ((DWSResult) result).getMessage(); + LOGGER.error("request failed, err is {}", errMsg); + throw new CSErrorException(80015,errMsg); + }else { + return (DWSResult)result; + } + }else { + throw new CSErrorException(80015,"resulet is not instance of DWSResult"); + } + } + + @Override + public void close() throws IOException { + try{ + LOGGER.info("client close"); + if (null != this.dwsHttpClient){ + this.dwsHttpClient.close(); + this.heartBeater.close(); + } + } catch (Exception e){ + LOGGER.error("Failed to close httpContextClient", e); + throw new IOException(e); + } + + } +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/HttpHeartBeater.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/HttpHeartBeater.java new file mode 100644 index 0000000000..ddee61cad8 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/http/HttpHeartBeater.java @@ -0,0 +1,141 @@ +package com.webank.wedatasphere.linkis.cs.client.http; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.common.listener.Event; +import com.webank.wedatasphere.linkis.common.utils.Utils; +import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientConfig; +import com.webank.wedatasphere.linkis.cs.client.builder.HttpContextClientConfig; +import com.webank.wedatasphere.linkis.cs.client.listener.*; +import com.webank.wedatasphere.linkis.cs.client.utils.ContextClientConf; +import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper; +import com.webank.wedatasphere.linkis.cs.common.entity.source.*; +import com.webank.wedatasphere.linkis.cs.listener.callback.imp.ContextKeyValueBean; +import com.webank.wedatasphere.linkis.cs.listener.event.enumeration.OperateType; +import com.webank.wedatasphere.linkis.cs.listener.event.impl.DefaultContextKeyEvent; +import com.webank.wedatasphere.linkis.httpclient.config.ClientConfig; +import com.webank.wedatasphere.linkis.httpclient.dws.DWSHttpClient; +import com.webank.wedatasphere.linkis.httpclient.dws.config.DWSClientConfig; +import com.webank.wedatasphere.linkis.httpclient.dws.response.DWSResult; +import com.webank.wedatasphere.linkis.httpclient.response.Result; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +/** + * created by cooperyang on 2020/2/18 + * Description: heartbeater类的作用是为了csclient能够和csserver进行每秒钟交互的一个类，从server中获取内容， + * 然后封装成事件投递到 事件总线，来让监听器进行消费 + */ +public class HttpHeartBeater implements HeartBeater { + + + private static final Logger LOGGER = LoggerFactory.getLogger(HttpHeartBeater.class); + + private ContextClientListenerBus contextClientListenerBus = + ContextClientListenerManager.getContextClientListenerBus(); + + private final String name = "ContextClientHTTPHeatBeater"; + + //todo 要改成某一个微服务的标识 + private final String client_source = "TestClient"; + + + private DWSHttpClient dwsHttpClient; + + + + public HttpHeartBeater(ContextClientConfig contextClientConfig){ + if (contextClientConfig instanceof HttpContextClientConfig){ + HttpContextClientConfig httpContextClientConfig = (HttpContextClientConfig)contextClientConfig; + ClientConfig clientConfig = httpContextClientConfig.getClientConfig(); + DWSClientConfig dwsClientConfig = new DWSClientConfig(clientConfig); + dwsClientConfig.setDWSVersion(ContextClientConf.LINKIS_WEB_VERSION().getValue()); + dwsHttpClient = new DWSHttpClient(dwsClientConfig, name); + } + } + + + + + + @Override + @SuppressWarnings("unchecked") + public void heartBeat() { + ContextHeartBeatAction contextHeartBeatAction = new ContextHeartBeatAction(client_source); + contextHeartBeatAction.getRequestPayloads().put("source", client_source); + Result result = null; + try{ + result = dwsHttpClient.execute(contextHeartBeatAction); + }catch(Exception e){ + LOGGER.error("执行heartbeat出现失败", e); + return ; + } + if (result instanceof ContextHeartBeatResult){ + ContextHeartBeatResult contextHeartBeatResult = (ContextHeartBeatResult)result; + Map data = contextHeartBeatResult.getData(); + Object object = data.get("ContextKeyValueBean"); + List kvBeans = new ArrayList<>(); + if (object instanceof List){ + List list = (List)object; + list.stream(). + filter(Objects::nonNull). + map(Object::toString). + map(str -> { + try{ + return SerializeHelper.deserializeContextKVBean(str); + }catch(ErrorException e){ + return null; + } + }).filter(Objects::nonNull).forEach(kvBeans::add); + } + if (kvBeans.size() > 0){ + dealCallBack(kvBeans); + } + } + } + + @Override + public void dealCallBack(List kvs) { + for(ContextKeyValueBean kv : kvs){ + //todo 先忽略掉contextIDEvent + ContextKeyValue contextKeyValue = new CommonContextKeyValue(); + contextKeyValue.setContextKey(kv.getCsKey()); + contextKeyValue.setContextValue(kv.getCsValue()); + DefaultContextKeyEvent event = new DefaultContextKeyEvent(); + event.setContextID(kv.getCsID()); + event.setOperateType(OperateType.UPDATE); + event.setContextKeyValue(contextKeyValue); + contextClientListenerBus.post(event); + } + } + + + @Override + public void start() { + Utils.defaultScheduler().scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + heartBeat(); + } + }, 0, 1, TimeUnit.HOURS); + } + + @Override + public void close() throws IOException { + try{ + if (null != this.dwsHttpClient){ + this.dwsHttpClient.close(); + } + } catch (Exception e){ + LOGGER.error("Failed to close httpContextClient", e); + throw new IOException(e); + } + } +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientEvent.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientEvent.java new file mode 100644 index 0000000000..2043deb871 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientEvent.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.linkis.cs.client.listener; + +import com.webank.wedatasphere.linkis.common.listener.Event; + +/** + * created by cooperyang on 2020/2/18 + * Description: + */ +public interface ContextClientEvent extends Event { + +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListener.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListener.java new file mode 100644 index 0000000000..245e520646 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListener.java @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.linkis.cs.client.listener; + +import com.webank.wedatasphere.linkis.common.listener.Event; +import com.webank.wedatasphere.linkis.common.listener.EventListener; + +/** + * created by cooperyang on 2020/2/11 + * Description: + */ +public interface ContextClientListener extends EventListener { + void onContextCreated(Event event); + void onContextUpdated(Event event); + void onEvent(Event event); +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListenerBus.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListenerBus.java new file mode 100644 index 0000000000..6de73ec3cb --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListenerBus.java @@ -0,0 +1,32 @@ +package com.webank.wedatasphere.linkis.cs.client.listener; + +import com.webank.wedatasphere.linkis.common.listener.Event; +import com.webank.wedatasphere.linkis.common.listener.ListenerBus; +import com.webank.wedatasphere.linkis.common.listener.ListenerEventBus; + +/** + * created by cooperyang on 2020/2/11 + * Description: + */ +public class ContextClientListenerBus extends ListenerEventBus { + + + private static final String NAME = "ContextClientListenerBus"; + + private static final int CAPACITY = 10; + + private static final int THREAD_SIZE = 20; + + private static final int MAX_FREE_TIME = 5000; + + public ContextClientListenerBus(){ + super(CAPACITY, NAME,THREAD_SIZE,MAX_FREE_TIME); + } + + + + @Override + public void doPostEvent(L listener, E event) { + listener.onEvent(event); + } +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListenerManager.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListenerManager.java new file mode 100644 index 0000000000..b8ef4e4237 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextClientListenerManager.java @@ -0,0 +1,25 @@ +package com.webank.wedatasphere.linkis.cs.client.listener; + +import com.webank.wedatasphere.linkis.common.listener.Event; + +/** + * created by cooperyang on 2020/2/11 + * Description: + * Manager的作用是为了方便用户将的 + */ +public class ContextClientListenerManager { + + private static ContextClientListenerBus contextClientListenerBus; + + public static ContextClientListenerBus getContextClientListenerBus(){ + if (contextClientListenerBus == null){ + synchronized (ContextClientListenerManager.class){ + if (contextClientListenerBus == null){ + contextClientListenerBus = new ContextClientListenerBus(); + } + } + } + return contextClientListenerBus; + } + +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextIDListener.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextIDListener.java new file mode 100644 index 0000000000..896a8dbc8d --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextIDListener.java @@ -0,0 +1,75 @@ +package com.webank.wedatasphere.linkis.cs.client.listener; + +import com.webank.wedatasphere.linkis.common.listener.Event; +import com.webank.wedatasphere.linkis.cs.client.Context; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID; +import com.webank.wedatasphere.linkis.cs.listener.event.enumeration.OperateType; +import com.webank.wedatasphere.linkis.cs.listener.event.impl.DefaultContextIDEvent; + + +/** + * created by cooperyang on 2020/2/17 + * Description: 这个listener是用来监听contextID的，用户可以进行实现 + */ +public abstract class ContextIDListener implements ContextClientListener{ + + + private ContextID contextID; + + private Context context; + + + public ContextIDListener(){ + + } + + public ContextIDListener(ContextID contextID){ + this.contextID = contextID; + } + + public ContextID getContextID() { + return contextID; + } + + public void setContextID(ContextID contextID) { + this.contextID = contextID; + } + + + public Context getContext() { + return context; + } + + public void setContext(Context context) { + this.context = context; + } + + @Override + public void onContextCreated(Event event) { + + } + + @Override + public void onContextUpdated(Event event) { + + } + + public abstract void onContextRemoved(Event event); + + + @Override + public void onEvent(Event event) { + if (event instanceof DefaultContextIDEvent){ + DefaultContextIDEvent defaultContextKeyEvent = (DefaultContextIDEvent)event; + if (defaultContextKeyEvent.getContextID().equals(contextID)){ + switch(defaultContextKeyEvent.getOperateType()){ + case UPDATE : onContextUpdated(defaultContextKeyEvent); + break; + case CREATE: onContextCreated(defaultContextKeyEvent);break; + case REMOVE: onContextRemoved(defaultContextKeyEvent);break; + default: break; + } + } + } + } +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextKeyListener.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextKeyListener.java new file mode 100644 index 0000000000..8df1e47142 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextKeyListener.java @@ -0,0 +1,68 @@ +package com.webank.wedatasphere.linkis.cs.client.listener; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.common.listener.Event; +import com.webank.wedatasphere.linkis.cs.client.Context; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey; +import com.webank.wedatasphere.linkis.cs.listener.event.impl.DefaultContextKeyEvent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * created by cooperyang on 2020/2/18 + * Description:一个微服务对contextKey的监听器 + */ +public abstract class ContextKeyListener implements ContextClientListener{ + + private static final Logger LOGGER = LoggerFactory.getLogger(ContextKeyListener.class); + + private ContextKey contextKey; + + private Context context; + + public ContextKeyListener(){ + + } + + public ContextKeyListener(ContextKey contextKey){ + this.contextKey = contextKey; + } + + public ContextKey getContextKey() { + return contextKey; + } + + public void setContextKey(ContextKey contextKey) { + this.contextKey = contextKey; + } + + public Context getContext() { + return context; + } + + public void setContext(Context context) { + this.context = context; + } + + @Override + public void onContextUpdated(Event event) { + if (event instanceof DefaultContextKeyEvent){ + context.setLocal(((DefaultContextKeyEvent) event).getContextKeyValue()); + } + } + + @Override + public void onEvent(Event event) { + if (event instanceof DefaultContextKeyEvent){ + DefaultContextKeyEvent defaultContextKeyEvent = (DefaultContextKeyEvent)event; + if (defaultContextKeyEvent.getContextKeyValue().getContextKey().equals(contextKey)){ + switch(defaultContextKeyEvent.getOperateType()){ + case UPDATE:onContextUpdated(defaultContextKeyEvent);break; + case CREATE:onContextCreated(defaultContextKeyEvent);break; + default:break; + } + } + } + } +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextUpdateEvent.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextUpdateEvent.java new file mode 100644 index 0000000000..9878c47e1a --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/ContextUpdateEvent.java @@ -0,0 +1,8 @@ +package com.webank.wedatasphere.linkis.cs.client.listener; + +/** + * created by cooperyang on 2020/2/18 + * Description: + */ +public class ContextUpdateEvent implements ContextClientEvent{ +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/HeartBeater.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/HeartBeater.java new file mode 100644 index 0000000000..2d7490705e --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/listener/HeartBeater.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.linkis.cs.client.listener; + +import com.webank.wedatasphere.linkis.cs.listener.callback.imp.ContextKeyValueBean; + +import java.io.Closeable; +import java.util.List; + +/** + * created by cooperyang on 2020/2/11 + * Description: + */ +public interface HeartBeater extends Closeable { + /** + * + */ + public void heartBeat(); + public void dealCallBack(List kvs); + + public void start(); + +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSMetaDataService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSMetaDataService.java new file mode 100644 index 0000000000..221cbad9ee --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSMetaDataService.java @@ -0,0 +1,54 @@ +package com.webank.wedatasphere.linkis.cs.client.service; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.common.io.MetaData; +import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey; +import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException; +import com.webank.wedatasphere.linkis.cs.common.exception.ErrorCode; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +/** + * @Author alexyang + * @Date 2020/3/9 + */ +public class CSMetaDataService implements MetaDataService { + + private static final Logger logger = LoggerFactory.getLogger(CSMetaDataService.class); + + private static CSMetaDataService csMetaDataService; + + private CSMetaDataService() { + + } + + public static CSMetaDataService getInstance() { + if (null == csMetaDataService) { + synchronized (CSMetaDataService.class) { + if (null == csMetaDataService) { + csMetaDataService = new CSMetaDataService(); + } + } + } + return csMetaDataService; + } + + @Override + public Map getAllUpstreamMetaData(String contextIDStr, String nodeName) throws CSErrorException { + if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(nodeName)) { + return null; + } + try { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + return DefaultSearchService.getInstance().searchUpstreamContextMap(contextID, nodeName, Integer.MAX_VALUE, MetaData.class); + } catch (ErrorException e) { + logger.error("Deserialize contextid error. contextID : " + contextIDStr + ", e ", e); + throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Deserialize contextid error. contextID : " + contextIDStr + ", e " + e.getDesc()); + } + } +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSNodeService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSNodeService.java new file mode 100644 index 0000000000..320ded94a3 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSNodeService.java @@ -0,0 +1,23 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.cs.client.service; + +/** + * @author peacewong + * @date 2020/3/21 19:17 + */ +public interface CSNodeService { + + void initNodeCSInfo(String contextIDStr, String ndeName); +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSNodeServiceImpl.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSNodeServiceImpl.java new file mode 100644 index 0000000000..8a6bb00f1e --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSNodeServiceImpl.java @@ -0,0 +1,63 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.cs.client.service; + +import com.webank.wedatasphere.linkis.cs.client.ContextClient; +import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; +import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID; +import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author peacewong + * @date 2020/3/21 19:18 + */ +public class CSNodeServiceImpl implements CSNodeService{ + + private final static Logger logger = LoggerFactory.getLogger(CSNodeServiceImpl.class); + + private SearchService searchService = DefaultSearchService.getInstance(); + + private static CSNodeService csNodeService; + + private CSNodeServiceImpl() { + + } + + public static CSNodeService getInstance() { + if (null == csNodeService) { + synchronized (CSNodeServiceImpl.class) { + if (null == csNodeService) { + csNodeService = new CSNodeServiceImpl(); + } + } + } + return csNodeService; + } + + @Override + public void initNodeCSInfo(String contextIDStr, String ndeName) { + + try { + ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + contextClient.removeAllValueByKeyPrefixAndContextType(contextID, ContextType.METADATA, CSCommonUtils.NODE_PREFIX + ndeName); + } catch (Exception e) { + logger.error("Failed to init node cs Info", e); + } + } +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResourceService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResourceService.java new file mode 100644 index 0000000000..399bbc6af8 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResourceService.java @@ -0,0 +1,75 @@ +package com.webank.wedatasphere.linkis.cs.client.service; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper; +import com.webank.wedatasphere.linkis.cs.common.entity.metadata.CSTable; +import com.webank.wedatasphere.linkis.cs.common.entity.resource.BMLResource; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey; +import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException; +import com.webank.wedatasphere.linkis.cs.common.exception.ErrorCode; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * @Author alexyang + * @Date 2020/3/9 + */ +public class CSResourceService implements ResourceService { + + private static final Logger logger = LoggerFactory.getLogger(CSResourceService.class); + + private static CSResourceService csResourceService; + + private CSResourceService() { + + } + + public static CSResourceService getInstance() { + if (null == csResourceService) { + synchronized (CSResourceService.class) { + if (null == csResourceService) { + csResourceService = new CSResourceService(); + } + } + } + return csResourceService; + } + + @Override + public Map getAllUpstreamBMLResource(String contextIDStr, String nodeName) throws CSErrorException { + if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(nodeName)) { + return null; + } + try { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + return DefaultSearchService.getInstance().searchUpstreamContextMap(contextID, nodeName, Integer.MAX_VALUE, BMLResource.class); + } catch (ErrorException e) { + logger.error("Deserialize contextid error. contextID : " + contextIDStr + ", e ", e); + throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Deserialize contextid error. contextID : " + contextIDStr + ", e " + e.getDesc()); + } + } + + @Override + public List getUpstreamBMLResource(String contextIDStr, String nodeName) throws CSErrorException { + List rsList = new ArrayList<>(); + if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(nodeName)) { + return rsList; + } + try { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + if (null != contextID) { + rsList = DefaultSearchService.getInstance().searchUpstreamContext(contextID, nodeName, Integer.MAX_VALUE, BMLResource.class); + } + return rsList; + } catch (ErrorException e) { + logger.error("Failed to get Resource: " + e.getMessage()); + throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Deserialize contextID error. contextIDStr : ", e); + } + } +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResultDataService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResultDataService.java new file mode 100644 index 0000000000..fd0f528e44 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResultDataService.java @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.cs.client.service; + +import com.webank.wedatasphere.linkis.cs.common.entity.data.CSResultData; +import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException; + +import java.util.List; + +/** + * @author peacewong + * @date 2020/3/13 20:00 + */ +public interface CSResultDataService { + + CSResultData getCSResultData(String contextIDStr, String contextKey)throws CSErrorException; + + void putCSResultData(String contextIDStr, String contextKeyStr, CSResultData csResultData) throws CSErrorException; + + List getUpstreamCSResultData(String contextIDStr, String nodeName) throws CSErrorException; +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResultDataServiceImpl.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResultDataServiceImpl.java new file mode 100644 index 0000000000..fff57b284c --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSResultDataServiceImpl.java @@ -0,0 +1,109 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.cs.client.service; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.cs.client.ContextClient; +import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; +import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper; +import com.webank.wedatasphere.linkis.cs.common.entity.data.CSResultData; +import com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextValue; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue; +import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException; +import com.webank.wedatasphere.linkis.cs.common.exception.ErrorCode; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author peacewong + * @date 2020/3/13 20:23 + */ +public class CSResultDataServiceImpl implements CSResultDataService{ + + private final static Logger logger = LoggerFactory.getLogger(CSResultDataServiceImpl.class); + + private SearchService searchService = DefaultSearchService.getInstance(); + + private static CSResultDataService csResultDataService; + + private CSResultDataServiceImpl() { + + } + + public static CSResultDataService getInstance() { + if (null == csResultDataService) { + synchronized (CSResultDataServiceImpl.class) { + if (null == csResultDataService) { + csResultDataService = new CSResultDataServiceImpl(); + } + } + } + return csResultDataService; + } + + @Override + public CSResultData getCSResultData(String contextIDStr, String contextKeyStr) throws CSErrorException { + if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(contextKeyStr)) { + return null; + } + try { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + ContextKey contextKey = SerializeHelper.deserializeContextKey(contextKeyStr); + return searchService.getContextValue(contextID, contextKey, CSResultData.class); + } catch (ErrorException e) { + logger.error("Deserialize failed, invalid contextId : " + contextIDStr + ", or contextKey : " + contextKeyStr + ", e : " + e.getMessage()); + logger.error("exception ", e); + throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Deserialize failed, invalid contextId : " + contextIDStr + ", or contextKey : " + contextKeyStr + ", e : " + e.getMessage()); + } + } + + @Override + public void putCSResultData(String contextIDStr, String contextKeyStr, CSResultData csResultData) throws CSErrorException { + ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); + try { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + ContextKey contextKey = SerializeHelper.deserializeContextKey(contextKeyStr); + ContextValue contextValue = new CommonContextValue(); + contextValue.setValue(csResultData); + contextClient.update(contextID, contextKey, contextValue); + } catch (ErrorException e) { + logger.error("Deserialize error. e ", e); + throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Deserialize error. e : " + e.getDesc()); + } + } + + @Override + public List getUpstreamCSResultData(String contextIDStr, String nodeName) throws CSErrorException { + List rsList = new ArrayList<>(); + if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(nodeName)) { + return rsList; + } + try { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + if (null != contextID) { + rsList = searchService.searchUpstreamContext(contextID, nodeName, Integer.MAX_VALUE, CSResultData.class); + } + return rsList; + } catch (ErrorException e) { + logger.error("Deserialize contextID error. contextIDStr : " + contextIDStr, e); + throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Deserialize contextID error. contextIDStr : " + contextIDStr + "e : " + e.getDesc()); + } + } +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSTableService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSTableService.java new file mode 100644 index 0000000000..32057fceeb --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSTableService.java @@ -0,0 +1,205 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.cs.client.service; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.cs.client.ContextClient; +import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; +import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; +import com.webank.wedatasphere.linkis.cs.common.entity.metadata.CSTable; +import com.webank.wedatasphere.linkis.cs.common.entity.source.*; +import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException; +import com.webank.wedatasphere.linkis.cs.common.exception.ErrorCode; +import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * @Author alexyang + * @Date 2020/3/9 + */ +public class CSTableService implements TableService { + + private final static Logger logger = LoggerFactory.getLogger(CSTableService.class); + + private SearchService searchService = DefaultSearchService.getInstance(); + + private static CSTableService csTableService; + + private CSTableService() { + + } + + public static CSTableService getInstance() { + if (null == csTableService) { + synchronized (CSTableService.class) { + if (null == csTableService) { + csTableService = new CSTableService(); + } + } + } + return csTableService; + } + + @Override + public CSTable getCSTable(ContextID contextID, ContextKey contextKey) throws CSErrorException { + if (null == contextID || null == contextKey) { + return null; + } + if (contextID instanceof CombinedNodeIDContextID) { + contextID = ((CombinedNodeIDContextID) contextID).getLinkisHaWorkFlowContextID(); + } + CSTable csTable = searchService.getContextValue(contextID, contextKey, CSTable.class); + return csTable; + } + + @Override + public List getUpstreamTables(String contextIDStr, String nodeName) throws CSErrorException { + List rsList = new ArrayList<>(); + if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(nodeName)) { + return rsList; + } + try { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + if (null != contextID) { + if (contextID instanceof CombinedNodeIDContextID) { + contextID = ((CombinedNodeIDContextID) contextID).getLinkisHaWorkFlowContextID(); + } + rsList = searchService.searchUpstreamContext(contextID, nodeName, Integer.MAX_VALUE, CSTable.class); + } + return rsList; + } catch (ErrorException e) { + logger.error("Deserialize contextID error. contextIDStr : " + contextIDStr); + throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "getUpstreamTables error ", e); + } + } + + @Override + public CSTable getUpstreamSuitableTable(String contextIDStr, String nodeName, String keyword) throws CSErrorException { + CSTable csTable = null; + if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(nodeName)) { + return csTable; + } + try { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + if (null != contextID) { + if (contextID instanceof CombinedNodeIDContextID) { + contextID = ((CombinedNodeIDContextID) contextID).getLinkisHaWorkFlowContextID(); + } + csTable = searchService.searchContext(contextID, keyword, nodeName, CSTable.class); + } + } catch (ErrorException e) { + throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "getUpstreamSuitableTable error ", e); + } + return csTable; + } + + @Override + public List searchUpstreamTableKeyValue(String contextIDStr, String nodeName) throws CSErrorException { + try { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + if (contextID instanceof CombinedNodeIDContextID) { + contextID = ((CombinedNodeIDContextID) contextID).getLinkisHaWorkFlowContextID(); + } + return searchService.searchUpstreamKeyValue(contextID, nodeName, Integer.MAX_VALUE, CSTable.class); + } catch (ErrorException e) { + throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Failed to searchUpstreamTableKeyValue ", e); + } + } + + + @Override + public void putCSTable(String contextIDStr, String contextKeyStr, CSTable csTable) throws CSErrorException { + ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); + try { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + ContextKey contextKey = SerializeHelper.deserializeContextKey(contextKeyStr); + ContextValue contextValue = new CommonContextValue(); + // todo check keywords + contextValue.setKeywords(""); + contextValue.setValue(csTable); + if (contextID instanceof CombinedNodeIDContextID) { + contextID = ((CombinedNodeIDContextID) contextID).getLinkisHaWorkFlowContextID(); + } + contextClient.update(contextID, contextKey, contextValue); + } catch (ErrorException e) { + throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "putCSTable error ", e); + } + } + + @Override + public CSTable getCSTable(String contextIDStr, String contextKeyStr) throws CSErrorException { + if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(contextKeyStr)) { + return null; + } + try { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + ContextKey contextKey = SerializeHelper.deserializeContextKey(contextKeyStr); + if (contextID instanceof CombinedNodeIDContextID) { + contextID = ((CombinedNodeIDContextID) contextID).getLinkisHaWorkFlowContextID(); + } + return getCSTable(contextID, contextKey); + } catch (ErrorException e) { + throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "getCSTable error ", e); + } + } + + + @Override + public void registerCSTable(String contextIDStr, String nodeName, String alias, CSTable csTable) throws CSErrorException { + + if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(nodeName)) { + return; + } + String tableName = ""; + if (StringUtils.isNotBlank(alias)) { + tableName = CSCommonUtils.CS_TMP_TABLE_PREFIX + nodeName + "_" + alias; + } else { + for (int i = 1; i < 10; i++) { + String tmpTable = CSCommonUtils.CS_TMP_TABLE_PREFIX + nodeName + "_rs" + i; + try { + ContextKey contextKey = new CommonContextKey(); + contextKey.setContextScope(ContextScope.PUBLIC); + contextKey.setContextType(ContextType.METADATA); + contextKey.setKey(CSCommonUtils.getTableKey(nodeName, tmpTable)); + CSTable oldCsTable = getCSTable(contextIDStr, SerializeHelper.serializeContextKey(contextKey)); + if (null == oldCsTable) { + tableName = tmpTable; + break; + } + } catch (Exception e) { + tableName = tmpTable; + logger.warn("Failed to build tmp tableName", e); + break; + } + } + } + try { + csTable.setName(tableName); + ContextKey contextKey = new CommonContextKey(); + contextKey.setContextScope(ContextScope.PUBLIC); + contextKey.setContextType(ContextType.METADATA); + contextKey.setKey(CSCommonUtils.getTableKey(nodeName, tableName)); + putCSTable(contextIDStr, SerializeHelper.serializeContextKey(contextKey), csTable); + } catch (ErrorException e) { + throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Failed to register cs tmp table ", e); + } + } +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSVariableService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSVariableService.java new file mode 100644 index 0000000000..11d3ee7daa --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSVariableService.java @@ -0,0 +1,94 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.cs.client.service; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.cs.client.ContextClient; +import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; +import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper; +import com.webank.wedatasphere.linkis.cs.common.entity.object.LinkisVariable; +import com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextValue; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue; +import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException; +import com.webank.wedatasphere.linkis.cs.common.exception.ErrorCode; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author peacewong + * @date 2020/3/12 20:29 + */ +public class CSVariableService implements VariableService{ + + private final static Logger logger = LoggerFactory.getLogger(CSVariableService.class); + + private SearchService searchService = DefaultSearchService.getInstance(); + + private static CSVariableService csVariableService; + + private CSVariableService(){ + + } + + @Override + public List getUpstreamVariables(String contextIDStr, String nodeName) throws CSErrorException{ + List rsList = new ArrayList<>(); + if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(nodeName)) { + return rsList; + } + try { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + if (null != contextID) { + rsList = searchService.searchUpstreamContext(contextID, nodeName, Integer.MAX_VALUE, LinkisVariable.class); + } + return rsList; + } catch (Throwable e) { + logger.error("Failed to get variable : " + contextIDStr, e); + // throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Failed to get variable : " + contextIDStr + "e : " + e.getMessage()); + } + return rsList; + } + + @Override + public void putVariable(String contextIDStr, String contextKeyStr, LinkisVariable linkisVariable) throws CSErrorException { + ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); + try { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + ContextKey contextKey = SerializeHelper.deserializeContextKey(contextKeyStr); + ContextValue contextValue = new CommonContextValue(); + contextValue.setValue(linkisVariable); + contextClient.update(contextID, contextKey, contextValue); + } catch (ErrorException e) { + logger.error("Deserialize error. e "); + throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Deserialize error. e : ", e); + } + } + + public static CSVariableService getInstance() { + if (null == csVariableService) { + synchronized (CSVariableService.class) { + if (null == csVariableService) { + csVariableService = new CSVariableService(); + } + } + } + return csVariableService; + } +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSWorkService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSWorkService.java new file mode 100644 index 0000000000..93ded4f310 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSWorkService.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.cs.client.service; + +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.WorkType; +import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException; + +import java.util.List; + +/** + * @Author alexyang + * @Date 2020/3/21 + */ +public interface CSWorkService { + + void initContextServiceInfo(String contextIDStr, WorkType workType) throws CSErrorException; + + void initContextServiceInfo(String contextIDStr, List workTypes) throws CSErrorException; +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSWorkServiceImpl.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSWorkServiceImpl.java new file mode 100644 index 0000000000..7484da9771 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/CSWorkServiceImpl.java @@ -0,0 +1,93 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.cs.client.service; + +import com.webank.wedatasphere.linkis.cs.client.ContextClient; +import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; +import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.WorkType; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID; +import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException; +import com.webank.wedatasphere.linkis.cs.common.exception.ErrorCode; +import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * @Author alexyang + * @Date 2020/3/21 + */ +public class CSWorkServiceImpl implements CSWorkService { + + private static final Logger logger = LoggerFactory.getLogger(CSWorkServiceImpl.class); + private CSWorkServiceImpl() {} + private static CSWorkService csWorkService = null; + + public static CSWorkService getInstance() { + if (null == csWorkService) { + synchronized (CSWorkServiceImpl.class) { + if (null == csWorkService) { + csWorkService = new CSWorkServiceImpl(); + } + } + } + return csWorkService; + } + + + @Override + public void initContextServiceInfo(String contextIDStr, WorkType workType) throws CSErrorException { + List typeList = new ArrayList<>(); + typeList.add(workType); + initContextServiceInfo(contextIDStr, typeList); + } + + @Override + public void initContextServiceInfo(String contextIDStr, List workTypes) throws CSErrorException { + try { + ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + for (WorkType type : workTypes) { + contextClient.removeAllValueByKeyPrefix(contextID, getWorkTypePrefix(type)); + } + } catch (Exception e) { + logger.error("InitContextInfo error. contextIDStr : {}, workTypes : {}" + contextIDStr, CSCommonUtils.gson.toJson(workTypes)); + throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "InitContextInfo error. contextIDStr : " + contextIDStr + ", workTypes : " + CSCommonUtils.gson.toJson(workTypes), e); + } + } + + private String getWorkTypePrefix(WorkType workType) { + String prefix = null; + switch (workType) { + case WORKSPACE: + prefix = CSCommonUtils.WORKSPACE_PREFIX; + break; + case PROJECT: + prefix = CSCommonUtils.PROJECT_PREFIX; + break; + case FLOW: + prefix = CSCommonUtils.FLOW_PREFIX; + break; + case NODE: + prefix = CSCommonUtils.NODE_PREFIX; + break; + default: + logger.error("Invalid workType : {}", workType); + } + return prefix; + } +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/DefaultSearchService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/DefaultSearchService.java new file mode 100644 index 0000000000..85a34f7a92 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/DefaultSearchService.java @@ -0,0 +1,147 @@ +package com.webank.wedatasphere.linkis.cs.client.service; + +import com.google.gson.Gson; +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.cs.client.ContextClient; +import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue; +import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException; +import com.webank.wedatasphere.linkis.cs.common.exception.ErrorCode; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @Author alexyang + * @Date 2020/3/6 + */ +public class DefaultSearchService implements SearchService { + private static final Logger logger = LoggerFactory.getLogger(DefaultSearchService.class); + private static final Gson gson = new Gson(); + private static SearchService searchService = null; + + private DefaultSearchService() { + } + + @Override + public T getContextValue(ContextID contextID, ContextKey contextKey, Class contextValueType) throws CSErrorException { + ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); + ContextValue contextValue = null; + ContextType contextType = contextKey.getContextType(); + try { + contextValue = contextClient.getContextValue(contextID, contextKey); + } catch (Exception e) { + logger.error("Failed to get ContextValue: " + e.getMessage()); + throw new CSErrorException(ErrorCode.GET_CONTEXT_VALUE_ERROR, "Failed to get ContextValue: ", e); + } + + if (null == contextValue || null == contextValue.getValue()) { + return null; + } else if (contextValueType.isInstance(contextValue.getValue())) { + return (T) contextValue.getValue(); + } else { + throw new CSErrorException(ErrorCode.INVALID_CONTEXT_VALUE_TYPE, "Invalid Context Type : " + contextType); + } + } + + @Override + public T searchContext(ContextID contextId, String keyword, String nodeName, Class contextValueType) throws CSErrorException { + if (null == contextId || StringUtils.isBlank(contextId.getContextId()) || StringUtils.isBlank(nodeName) || null == contextValueType) { + logger.error("ContextID or nodeName or contextValueType cannot be null."); + throw new CSErrorException(ErrorCode.INVALID_NULL_STRING, "ContextID or nodeName or contextValueType cannot be null."); + } + ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); + List contains = new ArrayList<>(); + contains.add(keyword); + List contextValueTypes = new ArrayList<>(); + contextValueTypes.add(contextValueType); + List contextKeyValues = null; + try { + contextKeyValues = contextClient.search(contextId, null, null, contains, null, false, nodeName, 1, contextValueTypes); + } catch (ErrorException e) { + logger.error("Search context value error."); + throw new CSErrorException(ErrorCode.GET_CONTEXT_VALUE_ERROR, "Search context value error: ", e); + } + if (CollectionUtils.isEmpty(contextKeyValues) || null == contextKeyValues.get(0).getContextValue()) { + return null; + } + if (contextValueType.isInstance(contextKeyValues.get(0).getContextValue().getValue())) { + return (T) contextKeyValues.get(0).getContextValue().getValue(); + } else { + throw new CSErrorException(ErrorCode.SEARCH_CONTEXT_VALUE_ERROR, "Search value : " + gson.toJson(contextKeyValues.get(0) + + " is not instance of class : " + contextValueType.getName())); + } + } + + @Override + public List searchUpstreamContext(ContextID contextID, String nodeName, int num, Class contextValueType) throws CSErrorException { + Map contextMap = searchUpstreamContextMap(contextID, nodeName, num, contextValueType); + if (null == contextMap || contextMap.size() < 1) { + return null; + } + List retValues = new ArrayList<>(contextMap.size()); + contextMap.entrySet().stream().filter(kv -> null != kv && null != kv.getValue()).map(kv -> kv.getValue()).forEach(retValues::add); + return retValues; + } + + @Override + public Map searchUpstreamContextMap(ContextID contextID, String nodeName, int num, Class contextValueType) throws CSErrorException { + ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); + if (null == contextID || StringUtils.isBlank(contextID.getContextId())) { + return null; + } + List contextValueTypes = new ArrayList<>(); + contextValueTypes.add(contextValueType); + List contextKeyValueList = null; + try { + contextKeyValueList = contextClient.search(contextID, null, null, null, null, true, nodeName, num, contextValueTypes); + } catch (ErrorException e) { + throw new CSErrorException(ErrorCode.GET_CONTEXT_VALUE_ERROR, "searchUpstreamContextMap error ", e); + } + if (CollectionUtils.isEmpty(contextKeyValueList)) { + return null; + } + Map retValues = new HashMap<>(contextKeyValueList.size()); + contextKeyValueList.stream().filter(kv -> null != kv && null != kv.getContextValue()).forEach(kv -> retValues.put(kv.getContextKey(), (T) (kv.getContextValue().getValue()))); + return retValues; + } + + @Override + public List searchUpstreamKeyValue(ContextID contextID, String nodeName, int num, Class contextValueType) throws CSErrorException { + ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); + if (null == contextID || StringUtils.isBlank(contextID.getContextId())) { + return null; + } + List contextValueTypes = new ArrayList<>(); + contextValueTypes.add(contextValueType); + List contextKeyValueList = null; + try { + contextKeyValueList = contextClient.search(contextID, null, null, null, null, true, nodeName, num, contextValueTypes); + } catch (ErrorException e) { + throw new CSErrorException(ErrorCode.GET_CONTEXT_VALUE_ERROR, "searchUpstreamKeyValue error ", e); + } + return contextKeyValueList; + } + + public static SearchService getInstance() { + if (null == searchService) { + synchronized (DefaultSearchService.class) { + if (null == searchService) { + searchService = new DefaultSearchService(); + } + } + } + return searchService; + } +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/LinkisJobDataService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/LinkisJobDataService.java new file mode 100644 index 0000000000..87badea0a1 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/LinkisJobDataService.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.cs.client.service; + +import com.webank.wedatasphere.linkis.cs.common.entity.data.LinkisJobData; +import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException; + +/** + * @author peacewong + * @date 2020/3/13 22:11 + */ +public interface LinkisJobDataService { + + LinkisJobData getLinkisJobData(String contextIDStr, String contextKey)throws CSErrorException; + + void putLinkisJobData(String contextIDStr, String contextKeyStr, LinkisJobData linkisJobData) throws CSErrorException; + + +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/LinkisJobDataServiceImpl.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/LinkisJobDataServiceImpl.java new file mode 100644 index 0000000000..a4b540e94d --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/LinkisJobDataServiceImpl.java @@ -0,0 +1,88 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.cs.client.service; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.cs.client.ContextClient; +import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; +import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper; +import com.webank.wedatasphere.linkis.cs.common.entity.data.LinkisJobData; +import com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextValue; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue; +import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException; +import com.webank.wedatasphere.linkis.cs.common.exception.ErrorCode; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author peacewong + * @date 2020/3/13 22:12 + */ +public class LinkisJobDataServiceImpl implements LinkisJobDataService { + + private final static Logger logger = LoggerFactory.getLogger(LinkisJobDataServiceImpl.class); + + private SearchService searchService = DefaultSearchService.getInstance(); + + private static LinkisJobDataService linkisJobDataService; + + private LinkisJobDataServiceImpl() { + + } + + public static LinkisJobDataService getInstance() { + if (null == linkisJobDataService) { + synchronized (LinkisJobDataServiceImpl.class) { + if (null == linkisJobDataService) { + linkisJobDataService = new LinkisJobDataServiceImpl(); + } + } + } + return linkisJobDataService; + } + + @Override + public LinkisJobData getLinkisJobData(String contextIDStr, String contextKeyStr) throws CSErrorException { + if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(contextKeyStr)) { + return null; + } + try { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + ContextKey contextKey = SerializeHelper.deserializeContextKey(contextKeyStr); + return searchService.getContextValue(contextID, contextKey, LinkisJobData.class); + } catch (ErrorException e) { + logger.error("Deserialize failed, invalid contextId : " + contextIDStr + ", or contextKey : " + contextKeyStr + ", e : " + e.getMessage()); + logger.error("exception ", e); + throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Deserialize failed, invalid contextId : " + contextIDStr + ", or contextKey : " + contextKeyStr + ", e : " + e.getMessage()); + } + } + + @Override + public void putLinkisJobData(String contextIDStr, String contextKeyStr, LinkisJobData linkisJobData) throws CSErrorException { + ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); + try { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + ContextKey contextKey = SerializeHelper.deserializeContextKey(contextKeyStr); + ContextValue contextValue = new CommonContextValue(); + contextValue.setValue(linkisJobData); + contextClient.update(contextID, contextKey, contextValue); + } catch (ErrorException e) { + logger.error("Deserialize error. e ", e); + throw new CSErrorException(ErrorCode.DESERIALIZE_ERROR, "Deserialize error. e : " + e.getDesc()); + } + } +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/MetaDataService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/MetaDataService.java new file mode 100644 index 0000000000..47cee911aa --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/MetaDataService.java @@ -0,0 +1,25 @@ +package com.webank.wedatasphere.linkis.cs.client.service; + +import com.webank.wedatasphere.linkis.common.io.MetaData; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey; +import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException; + +import java.util.Map; + +/** + * @Author alexyang + * @Date 2020/3/9 + */ +public interface MetaDataService { + + /** + * 通过ContextID和NodeName，获取上游的所有Metadata数据 + * @param contextIDStr + * @param nodeName + * @return + * @throws CSErrorException + */ + Map getAllUpstreamMetaData(String contextIDStr, String nodeName) throws CSErrorException; + +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/ResourceService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/ResourceService.java new file mode 100644 index 0000000000..cc5d3d4aca --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/ResourceService.java @@ -0,0 +1,26 @@ +package com.webank.wedatasphere.linkis.cs.client.service; + +import com.webank.wedatasphere.linkis.cs.common.entity.resource.BMLResource; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey; +import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException; + +import java.util.List; +import java.util.Map; + +/** + * @Author alexyang + * @Date 2020/3/9 + */ +public interface ResourceService { + + /** + * 通过ContextID和NodeName，获取上游的所有Resource数据 + * @param contextIDStr + * @param nodeName + * @return + */ + Map getAllUpstreamBMLResource(String contextIDStr, String nodeName) throws CSErrorException; + + List getUpstreamBMLResource(String contextIDStr, String nodeName) throws CSErrorException; +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/SearchService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/SearchService.java new file mode 100644 index 0000000000..3337e2b8f6 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/SearchService.java @@ -0,0 +1,39 @@ +package com.webank.wedatasphere.linkis.cs.client.service; + +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue; +import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException; + +import java.util.List; +import java.util.Map; + +/** + * @Author alexyang + * @Date 2020/3/6 + */ +public interface SearchService { + + T getContextValue(ContextID contextId, ContextKey contextKey, Class contextValueType) throws CSErrorException; + + /** + * + * 返回匹配条件中最合适的一个 + * @param contetID LinkisHAFlowContextID实例 + * @param keyword 包含的关键字 + * @param contextValueType 返回的contextValue必须是该类型的实例 + * @param nodeName 如果nodeName是null，搜寻全部的，如果不为空搜寻上游的 + * @param + * @return + * @throws CSErrorException + */ + T searchContext(ContextID contetID, String keyword, String nodeName, Class contextValueType) throws CSErrorException; + + List searchUpstreamContext(ContextID contextID, String nodeName, int num, Class contextValueType) throws CSErrorException; + + Map searchUpstreamContextMap(ContextID contextID, String nodeName, int num, Class contextValueType) throws CSErrorException; + + List searchUpstreamKeyValue(ContextID contextID, String nodeName, int num, Class contextValueType) throws CSErrorException; + +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/TableService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/TableService.java new file mode 100644 index 0000000000..de4a7b8fb1 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/TableService.java @@ -0,0 +1,59 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.cs.client.service; + +import com.webank.wedatasphere.linkis.cs.common.entity.metadata.CSTable; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue; +import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException; + +import java.util.List; + +/** + * @Author alexyang + * @Date 2020/3/9 + */ +public interface TableService { + + /** + * 通过ContextKey获取一个确定的CSTable + * + * @param contextID + * @param contextKey + * @return + * @throws CSErrorException + */ + CSTable getCSTable(ContextID contextID, ContextKey contextKey) throws CSErrorException; + + /** + * 获取上游节点的所有表 + * + * @param contextIDStr + * @param nodeName + * @return + * @throws CSErrorException + */ + List getUpstreamTables(String contextIDStr, String nodeName) throws CSErrorException; + + void putCSTable(String contextIDStr, String ContextKey, CSTable csTable) throws CSErrorException; + + CSTable getCSTable(String contextIDStr, String contextKey) throws CSErrorException; + + CSTable getUpstreamSuitableTable(String contextIDStr, String nodeName, String keyword) throws CSErrorException; + + List searchUpstreamTableKeyValue(String contextIDStr, String nodeName) throws CSErrorException; + + void registerCSTable(String contextIDStr, String nodeName, String alias, CSTable csTable) throws CSErrorException; +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/VariableService.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/VariableService.java new file mode 100644 index 0000000000..2bb637b511 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/service/VariableService.java @@ -0,0 +1,30 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.cs.client.service; + +import com.webank.wedatasphere.linkis.cs.common.entity.object.LinkisVariable; +import com.webank.wedatasphere.linkis.cs.common.exception.CSErrorException; + +import java.util.List; + +/** + * @author peacewong + * @date 2020/3/12 20:28 + */ +public interface VariableService { + + List getUpstreamVariables(String contextIDStr, String nodeName) throws CSErrorException; + + void putVariable(String contextIDStr, String contextKey, LinkisVariable linkisVariable) throws CSErrorException; +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/ContextServiceUtils.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/ContextServiceUtils.java new file mode 100644 index 0000000000..32a4ba16d0 --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/ContextServiceUtils.java @@ -0,0 +1,192 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.cs.client.utils; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.cs.client.service.DefaultSearchService; +import com.webank.wedatasphere.linkis.cs.client.service.SearchService; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; +import com.webank.wedatasphere.linkis.cs.common.entity.object.CSFlowInfos; +import com.webank.wedatasphere.linkis.cs.common.entity.source.CombinedNodeIDContextID; +import com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextKey; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey; +import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.Properties; + +/** + * @author peacewong + * @date 2020/3/7 22:04 + */ +public class ContextServiceUtils { + + private static final Logger logger = LoggerFactory.getLogger(ContextServiceUtils.class); + + /** + * TODO get instances + */ + private static SearchService commonSearchService = DefaultSearchService.getInstance(); + + public static String getContextIDStrByMap(Map map){ + String contextIDStr = null; + if (null != map){ + Object value = map.get(CSCommonUtils.CONTEXT_ID_STR); + if (null != value){ + contextIDStr = value.toString(); + try { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + //commonSearchService.getContextValue() + if (contextID instanceof CombinedNodeIDContextID) { + contextIDStr = SerializeHelper.serializeContextID(((CombinedNodeIDContextID) contextID).getLinkisHaWorkFlowContextID()); + } + } catch (ErrorException e) { + logger.info("Failed to deserializeContextID", e); + } + } + } + return contextIDStr; + } + + + public static String getNodeNameStrByMap(Map map){ + Object contextIDValue = map.get(CSCommonUtils.CONTEXT_ID_STR); + if (null == contextIDValue) { + return null; + } + + String nodeName = null; + try { + if (null != map) { + Object value = map.get(CSCommonUtils.NODE_NAME_STR); + if (null != value) { + nodeName = value.toString(); + + } + } + if (StringUtils.isBlank(nodeName)) { + nodeName = getNodeNameByCombinedNodeIDContextID(contextIDValue.toString()); + } + } catch (Exception e) { + logger.info("Failed to get nodeName", e); + } + map.put(CSCommonUtils.NODE_NAME_STR, nodeName); + return nodeName; + } + + + + public static String getContextIDStrByProperties(Properties properties){ + String contextIDStr = null; + if (null != properties){ + Object value = properties.get(CSCommonUtils.CONTEXT_ID_STR); + if (null != value) { + contextIDStr = value.toString(); + try { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + //commonSearchService.getContextValue() + if (contextID instanceof CombinedNodeIDContextID) { + contextIDStr = SerializeHelper.serializeContextID(((CombinedNodeIDContextID) contextID).getLinkisHaWorkFlowContextID()); + } + } catch (ErrorException e) { + logger.info("Failed to deserializeContextID", e); + } + } + } + return contextIDStr; + } + + public static String getNodeNameStrByProperties(Properties properties) { + Object contextIDValue = properties.get(CSCommonUtils.CONTEXT_ID_STR); + if (null == contextIDValue) { + return null; + } + String nodeName = null; + + if (null != properties) { + Object value = properties.get(CSCommonUtils.NODE_NAME_STR); + if (null != value) { + nodeName = value.toString(); + } + } + if (StringUtils.isBlank(nodeName)) { + nodeName = getNodeNameByCombinedNodeIDContextID(contextIDValue.toString()); + } + properties.put(CSCommonUtils.NODE_NAME_STR, nodeName); + return nodeName; + } + + + public static String getNodeNameByCombinedNodeIDContextID(String contextIDStr){ + String nodeName = null; + try { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + //commonSearchService.getContextValue() + if (contextID instanceof CombinedNodeIDContextID){ + logger.info("contextID{} is combinedNodeIDContextID", contextID.getContextId()); + String nodeID = ((CombinedNodeIDContextID) contextID).getNodeID(); + return getNodeNameByNodeID(SerializeHelper.serializeContextID(((CombinedNodeIDContextID) contextID).getLinkisHaWorkFlowContextID()), nodeID); + } + } catch (Exception e) { + logger.info("Failed to get nodeName", e); + } + return nodeName; + } + + public static String getNodeNameByNodeID(String contextIDStr, String nodeID) { + if (StringUtils.isBlank(contextIDStr) || StringUtils.isBlank(nodeID)){ + return null; + } + try { + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + ContextKey contextKey = new CommonContextKey(); + contextKey.setContextType(ContextType.OBJECT); + contextKey.setContextScope(ContextScope.PUBLIC); + contextKey.setKey(CSCommonUtils.FLOW_INFOS); + CSFlowInfos csFlowInfos = commonSearchService.getContextValue(contextID, contextKey, CSFlowInfos.class); + if (null != csFlowInfos && null != csFlowInfos.getInfos()) { + Object idName = csFlowInfos.getInfos().get(CSCommonUtils.ID_NODE_NAME); + if (null != idName) { + return ((Map) idName).get(nodeID); + } + } + } catch (ErrorException e) { + logger.info("Failed to get nodeName ", e); + } + return null; + } + + public static String createCombinedNodeIDContextID(String contextIDStr, String nodeID) throws ErrorException { + if (StringUtils.isNotBlank(contextIDStr) && StringUtils.isNotBlank(nodeID)){ + ContextID contextID = SerializeHelper.deserializeContextID(contextIDStr); + if (null != contextID){ + return SerializeHelper.serializeContextID(new CombinedNodeIDContextID(contextID, nodeID)); + } + } + return null; + } + + /* public static String[] getContextIDAndNodeName(Map map){ + String contextIDStr = getContextIDStrByMap(map); + if (StringUtils.isBlank(contextIDStr)) { + return null; + } + + }*/ +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/ExceptionHelper.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/ExceptionHelper.java new file mode 100644 index 0000000000..62da009cae --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/ExceptionHelper.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.linkis.cs.client.utils; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; + +/** + * created by cooperyang on 2020/2/19 + * Description: + */ +public class ExceptionHelper { + public static void throwErrorException(int errCode, String errMsg, Throwable t)throws ErrorException { + ErrorException errorException = new ErrorException(errCode, errMsg); + errorException.initCause(t); + throw errorException; + } +} diff --git a/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/SerializeHelper.java b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/SerializeHelper.java new file mode 100644 index 0000000000..202040bf4f --- /dev/null +++ b/contextservice/cs-client/src/main/java/com/webank/wedatasphere/linkis/cs/client/utils/SerializeHelper.java @@ -0,0 +1,63 @@ +package com.webank.wedatasphere.linkis.cs.client.utils; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextID; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue; +import com.webank.wedatasphere.linkis.cs.common.serialize.helper.ContextSerializationHelper; +import com.webank.wedatasphere.linkis.cs.common.serialize.helper.SerializationHelper; +import com.webank.wedatasphere.linkis.cs.listener.callback.imp.ContextKeyValueBean; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * created by cooperyang on 2020/2/23 + * Description: + */ +public class SerializeHelper { + + + private static final Logger LOGGER = LoggerFactory.getLogger(SerializeHelper.class); + + public static final SerializationHelper SERIALIZE_HELPER = ContextSerializationHelper.getInstance(); + + public static String serializeContextID(ContextID contextID) throws ErrorException{ + return SERIALIZE_HELPER.serialize(contextID); + } + + public static ContextID deserializeContextID(String contextIDStr) throws ErrorException{ + return (ContextID) SERIALIZE_HELPER.deserialize(contextIDStr); + } + + + + public static String serializeContextKey(ContextKey contextKey) throws ErrorException { + return SERIALIZE_HELPER.serialize(contextKey); + } + + + + public static ContextKey deserializeContextKey(String contextKeyStr) throws ErrorException{ + return (ContextKey)SERIALIZE_HELPER.deserialize(contextKeyStr); + } + + public static String serializeContextValue(ContextValue contextValue) throws ErrorException{ + return SERIALIZE_HELPER.serialize(contextValue); + } + + public static ContextValue deserializeContextValue(String contextValueStr) throws ErrorException{ + return (ContextValue)SERIALIZE_HELPER.deserialize(contextValueStr); + } + + public static ContextKeyValueBean deserializeContextKVBean(String contextKVBeanStr) throws ErrorException{ + return (ContextKeyValueBean)SERIALIZE_HELPER.deserialize(contextKVBeanStr); + } + + public static ContextKeyValue deserializeContextKeyValue(String contextKVStr) throws ErrorException{ + return (ContextKeyValue)SERIALIZE_HELPER.deserialize(contextKVStr); + } + + + +} diff --git a/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/exception/ProtocolNotMatchException.scala b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/exception/ProtocolNotMatchException.scala new file mode 100644 index 0000000000..1c22c9115e --- /dev/null +++ b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/exception/ProtocolNotMatchException.scala @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.linkis.cs.client.exception + +import com.webank.wedatasphere.linkis.common.exception.ErrorException + +/** + * created by cooperyang on 2020/2/18 + * Description: + */ +case class ProtocolNotMatchException(errMsg:String) extends ErrorException(70059, errMsg) diff --git a/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/http/ContextAction.scala b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/http/ContextAction.scala new file mode 100644 index 0000000000..0761c4c104 --- /dev/null +++ b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/http/ContextAction.scala @@ -0,0 +1,97 @@ +package com.webank.wedatasphere.linkis.cs.client.http + +import com.webank.wedatasphere.linkis +import com.webank.wedatasphere.linkis.cs.client.utils.{ContextClientConf, ContextClientUtils, ContextServerHttpConf} +import com.webank.wedatasphere.linkis.httpclient.request.{GetAction, POSTAction, UserAction} +import org.apache.commons.text.StringEscapeUtils +/** + * created by cooperyang on 2020/2/11 + * Description: + */ +trait ContextAction extends UserAction{ + + private var user:String = "hadoop" + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user +} + +abstract class ContextGETAction extends GetAction with ContextAction + +abstract class ContextPostAction extends POSTAction with ContextAction { + override def getRequestPayload: String = ContextClientUtils.gson.toJson(getRequestPayloads) +} + +case class ContextCreateAction() extends ContextPostAction{ + + override def getURL: String = ContextServerHttpConf.createContextURL + +} + +case class ContextGetValueAction() extends ContextPostAction{ + override def getURL: String = ContextServerHttpConf.getContextValueURL +} + + +case class ContextUpdateAction() extends ContextPostAction{ + + override def getURL: String = ContextServerHttpConf.updateContextURL + +} + +case class ContextSetKeyValueAction() extends ContextPostAction{ + + override def getURL: String = ContextServerHttpConf.setKeyValueURL + +} + + +case class ContextResetValueAction() extends ContextPostAction{ + + override def getURL: String = ContextServerHttpConf.resetKeyValueURL + +} + + +case class ContextResetIDAction() extends ContextPostAction{ + override def getURL: String = ContextServerHttpConf.resetContextIdURL +} + + +case class ContextRemoveAction(contextId:String, + contextKey:String) extends ContextPostAction with UserAction { + + override def getURL: String = ContextServerHttpConf.removeValueURL + +} + + +case class ContextBindIDAction() extends ContextPostAction{ + override def getURL: String = ContextServerHttpConf.onBindIDURL +} + +case class ContextBindKeyAction() extends ContextPostAction{ + override def getURL: String = ContextServerHttpConf.onBindKeyURL +} + + + + +case class ContextFetchAction(contextId:String) extends ContextGETAction{ + override def getURL: String = ContextServerHttpConf.getContextIDURL +} + +case class ContextHeartBeatAction(client:String) extends ContextPostAction { + override def getURL: String = ContextServerHttpConf.heartBeatURL +} + + +case class ContextSearchContextAction() extends ContextPostAction{ + override def getURL: String = ContextServerHttpConf.searchURL +} + +case class DefaultContextPostAction(url:String) extends ContextPostAction{ + // TODO: 类太多了,放一个default + override def getURL: String = url +} \ No newline at end of file diff --git a/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/http/ContextResult.scala b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/http/ContextResult.scala new file mode 100644 index 0000000000..aaea3807a4 --- /dev/null +++ b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/http/ContextResult.scala @@ -0,0 +1,113 @@ +package com.webank.wedatasphere.linkis.cs.client.http + +import com.webank.wedatasphere.linkis.cs.listener.callback.imp.ContextKeyValueBean +import com.webank.wedatasphere.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import com.webank.wedatasphere.linkis.httpclient.dws.response.DWSResult + +/** + * created by cooperyang on 2020/2/11 + * Description: + */ +abstract class ContextResult extends DWSResult + + +@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/createContextID") +class ContextCreateResult extends ContextResult { + var contextId:String = _ + def setContextId(contextId:String):Unit = this.contextId = contextId + def getContextId:String = this.contextId +} + +@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/resetValue") +class ContextResetResult extends ContextResult { + +} + +@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/setValueByKey") +class ContextUpdateResult extends ContextResult { + +} +@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/getContext") +class ContextGetResult extends ContextResult { + var contextId:String = _ + var contextKeyValues:java.util.Map[String, String] = _ + + def setContextId(contextId:String):Unit = this.contextId = contextId + def getContextId:String = this.contextId + + def setContextKeyValues(kvs:java.util.Map[String,String]):Unit = this.contextKeyValues = kvs + def getContextKeyValues:java.util.Map[String, String] = this.contextKeyValues + +} + + +@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/removeValue") +class ContextRemoveResult extends ContextResult { + +} + + + +@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/heartbeat") +class ContextHeartBeatResult extends ContextResult { + import java.util + var contextKeyValueBeans:util.List[ContextKeyValueBean] = _ + def setContextKeyValueBeans(kvs:util.List[ContextKeyValueBean]):Unit = this.contextKeyValueBeans= kvs + def getContextKeyValueBeans:util.List[ContextKeyValueBean] = this.contextKeyValueBeans; +} + + + + +@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/setValue") +class ContextSetKeyValueResult extends ContextResult { +} + + +@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/removeAllValue") +class ContextResetIDResult extends ContextResult { +} + + +@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/removeValue") +class ContextRemoveValueResult extends ContextResult { +} + +@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/onBindIDListener") +class ContextBindIDResult extends ContextResult { +} + + +@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/onBindKeyListener") +class ContextBindKeyResult extends ContextResult { +} + +@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/searchContextValue") +class ContextSearchResult extends ContextResult { +} + + +@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/getContextValue") +class ContextGetValueResult extends ContextResult { +} + +// TODO: 用来匹配所有的void 不需要处理返回值的result +@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/(createHistory|removeHistory|removeAllValueByKeyPrefix|removeAllValueByKeyPrefixAndContextType)") +class VoidResult extends ContextResult { +} + +@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/(getHistories|searchHistory)") +class ContextHistoriesGetResult extends ContextResult { + var contextHistory:java.util.List[String] = _ + def getContextHistory:java.util.List[String] = this.contextHistory + def setContextHistory(contextHistory:java.util.List[String])= this.contextHistory = contextHistory +} + +@DWSHttpMessageResult("/api/rest_j/v\\d+/contextservice/getHistory") +class ContextHistoryGetResult extends ContextResult { + var contextHistory:String = _ + def getContextHistory:String = this.contextHistory + def setContextHistory(contextHistory:String)= this.contextHistory = contextHistory +} + + diff --git a/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextClientConf.scala b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextClientConf.scala new file mode 100644 index 0000000000..2cc46c4f02 --- /dev/null +++ b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextClientConf.scala @@ -0,0 +1,47 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.cs.client.utils + +import com.webank.wedatasphere.linkis.common.conf.CommonVars + +/** + * created by cooperyang on 2020/2/18 + * Description: + */ +object ContextClientConf { + val LINKIS_WEB_VERSION:CommonVars[String] = CommonVars[String]("wds.linkis.web.version", "v1") + + + val CONTEXT_CLIENT_AUTH_KEY:CommonVars[String] = CommonVars[String]("wds.linkis.context.client.auth.key", "Token-Code") + + val CONTEXT_CLIENT_AUTH_VALUE:CommonVars[String] = CommonVars[String]("wds.linkis.context.client.auth.value", "BML-AUTH") + + + val URL_PREFIX:CommonVars[String] = CommonVars[String]("wds.linkis.cs.url.prefix", "/api/rest_j/v1/contextservice", "cs服务的url前缀") + + val CREATE_CONTEXT_URL:CommonVars[String] = CommonVars[String]("wds.linkis.cs.createcontext.url", "createContextID") + + val SET_VALUE_BY_KEY_URL:CommonVars[String] = CommonVars[String]("wds.linkis.cs.setvaluebykey.url", "setValueByKey") + + val SET_KEYVALUE_URL:CommonVars[String] = CommonVars[String]("wds.linkis.cs.setkeyvalue.url", "setValue") + + val RESET_KEYVALUE_URL:CommonVars[String] = CommonVars[String]("wds.linkis.cs.resetkeyvalue.url", "resetValue") + + val REMOVE_VALUE_URL:CommonVars[String] = CommonVars[String]("wds.linkis.cs.removeValue.url", "removeValue") + + val RESET_CONTEXT_ID_URL:CommonVars[String] = CommonVars[String]("wds.linkis.cs.reset.contextid.url", "removeAllValue") + + val HEART_BEAT_ENABLED:CommonVars[String] = CommonVars[String]("wds.linkis.cs.heartbeat.enabled", "true") + +} diff --git a/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextClientUtils.scala b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextClientUtils.scala new file mode 100644 index 0000000000..6dcd4606cc --- /dev/null +++ b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextClientUtils.scala @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.linkis.cs.client.utils + +import java.lang +import java.lang.reflect.Type + +import com.google.gson.{GsonBuilder, JsonElement, JsonPrimitive, JsonSerializationContext, JsonSerializer} + +/** + * created by cooperyang on 2020/2/23 + * Description: + */ +object ContextClientUtils { + implicit val gson = new GsonBuilder().setPrettyPrinting().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").serializeNulls + .registerTypeAdapter(classOf[java.lang.Double], new JsonSerializer[java.lang.Double] { + override def serialize(t: lang.Double, `type`: Type, jsonSerializationContext: JsonSerializationContext): JsonElement = + if(t == t.longValue()) new JsonPrimitive(t.longValue()) else new JsonPrimitive(t) + }).create +} diff --git a/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextServerHttpConf.scala b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextServerHttpConf.scala new file mode 100644 index 0000000000..b85165bc40 --- /dev/null +++ b/contextservice/cs-client/src/main/scala/com/webank/wedatasphere/linkis/cs/client/utils/ContextServerHttpConf.scala @@ -0,0 +1,65 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.cs.client.utils + +import com.webank.wedatasphere.linkis.common.conf.Configuration + +/** + * created by cooperyang on 2020/2/19 + * Description: + */ +object ContextServerHttpConf { + val gatewayInstance: String = Configuration.getGateWayURL() + val urlPrefix: String = if (ContextClientConf.URL_PREFIX.getValue.endsWith("/")) { + ContextClientConf.URL_PREFIX.getValue.substring(0, ContextClientConf.URL_PREFIX.getValue.length - 1) + } else ContextClientConf.URL_PREFIX.getValue + + val createContextURL:String = urlPrefix + "/" + ContextClientConf.CREATE_CONTEXT_URL.getValue + + val updateContextURL:String = urlPrefix + "/" + ContextClientConf.SET_VALUE_BY_KEY_URL.getValue; + + val setKeyValueURL:String = urlPrefix + "/" + ContextClientConf.SET_KEYVALUE_URL.getValue + + val resetKeyValueURL:String = urlPrefix + "/" + ContextClientConf.RESET_KEYVALUE_URL.getValue + + val removeValueURL:String = urlPrefix + "/" + ContextClientConf.REMOVE_VALUE_URL.getValue + + val resetContextIdURL:String = urlPrefix + "/" + ContextClientConf.RESET_CONTEXT_ID_URL.getValue + + val onBindKeyURL:String = urlPrefix + "/" + "onBindKeyListener" + + val onBindIDURL:String = urlPrefix + "/" + "onBindIDListener" + + val getContextIDURL:String = urlPrefix + "/" + "getContextID" + + val heartBeatURL:String = urlPrefix + "/" + "heartbeat" + + val searchURL:String = urlPrefix + "/" + "searchContextValue" + + val getContextValueURL:String = urlPrefix + "/" + "getContextValue" + + val createContextHistory:String = urlPrefix + "/" + "createHistory" + + val removeContextHistory:String = urlPrefix + "/" + "removeHistory" + + val getContextHistories:String = urlPrefix + "/" + "getHistories" + + val getContextHistory:String = urlPrefix + "/" + "getHistory" + + val searchContextHistory:String = urlPrefix + "/" + "searchHistory" + + val removeAllValueByKeyPrefixAndContextTypeURL: String = urlPrefix + "/" + "removeAllValueByKeyPrefixAndContextType" + + val removeAllValueByKeyPrefixURL: String = urlPrefix + "/" + "removeAllValueByKeyPrefix" +} diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/Test.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/Test.java new file mode 100644 index 0000000000..53638e1cfa --- /dev/null +++ b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/Test.java @@ -0,0 +1,113 @@ +package com.webank.wedatasphere.linkis.cs.client.test; + +import com.google.gson.Gson; +import com.webank.wedatasphere.linkis.common.listener.Event; +import com.webank.wedatasphere.linkis.cs.client.Context; +import com.webank.wedatasphere.linkis.cs.client.ContextClient; +import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; +import com.webank.wedatasphere.linkis.cs.client.listener.ContextClientListener; +import com.webank.wedatasphere.linkis.cs.client.listener.ContextClientListenerBus; +import com.webank.wedatasphere.linkis.cs.client.listener.ContextClientListenerManager; +import com.webank.wedatasphere.linkis.cs.client.listener.ContextKeyListener; +import com.webank.wedatasphere.linkis.cs.client.test.listener.CommonContextKeyListener; +import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; +import com.webank.wedatasphere.linkis.cs.common.entity.object.CSFlowInfos; +import com.webank.wedatasphere.linkis.cs.common.entity.resource.LinkisBMLResource; +import com.webank.wedatasphere.linkis.cs.common.entity.source.*; +import com.webank.wedatasphere.linkis.cs.common.serialize.helper.ContextSerializationHelper; + +import java.util.ArrayList; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * created by cooperyang on 2020/2/24 + * Description: + */ +public class Test { + public static void main(String[] args)throws Exception{ + /*ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); + ContextID contextID = new LinkisHAWorkFlowContextID(); + Context context = contextClient.createContext(contextID); + System.out.println(context.getContextID().getContextId()); + ContextKey contextKey = new CommonContextKey(); + contextKey.setKey("cooper.txt"); + contextKey.setKeywords("xddd"); + contextKey.setContextScope(ContextScope.PUBLIC); + contextKey.setContextType(ContextType.RESOURCE); + ContextValue contextValue = new CommonContextValue(); + LinkisBMLResource resource = new LinkisBMLResource(); + resource.setResourceId("456789"); + resource.setVersion("v00001"); + contextValue.setValue(resource); + ContextKeyValue contextKeyValue = new CommonContextKeyValue(); + contextKeyValue.setContextValue(contextValue); + contextKeyValue.setContextKey(contextKey); + context.setContextKeyAndValue(contextKeyValue); + ContextValue myValue = context.getContextValue(contextKey); + LinkisBMLResource linkisBMLResource = (LinkisBMLResource)myValue.getValue(); + System.out.println(linkisBMLResource.getResourceId()); + ContextClientListenerBus bus = ContextClientListenerManager.getContextClientListenerBus(); + ContextKeyListener listener = new CommonContextKeyListener(); + listener.setContextKey(contextKey); + listener.setContext(context); + context.onBind(contextKey, listener); + bus.addListener(listener); + List contains = new ArrayList<>(); + contains.add("cooper"); + contextClient.search(context.getContextID(), null, null, contains, null); + List list = context.searchContext(null, null, contains, null); + System.out.println("ret list is " + list); + Thread.sleep(5000); + contextClient.close();*/ + testCSFlowInfos(); + + } + + public static void testCSFlowInfos() throws Exception { + ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); + LinkisHAWorkFlowContextID contextID = new LinkisHAWorkFlowContextID(); + contextID.setFlow("test"); + contextID.setProject("test"); + contextID.setWorkSpace("test1"); + contextID.setVersion("v00001"); + Context context = contextClient.createContext(contextID); + System.out.println(context.getContextID().getContextId()); + + ContextSerializationHelper contextSerializationHelper = ContextSerializationHelper.getInstance(); + CommonContextValue contextValue = new CommonContextValue(); + Map infos = new HashMap<>(); + List> edges = new ArrayList<>(); + Map edge = new HashMap<>(); + edge.put("source","sql " ); + edge.put("target","hql"); + edge.put("sourceLocation","bottom"); + edge.put("targetLocation","top"); + edges.add(edge); + infos.put("edges", edges); + infos.put("parent", "flow2"); + Map idNodeName = new HashMap<>(); + idNodeName.put("90a6ee94-4bd6-47d9-a536-f92660c4c051", "sql"); + idNodeName.put("90a6ee94-4bd6-47d9-a536-f92660c4c052", "hql"); + infos.put("id_nodeName", idNodeName); + CSFlowInfos csFlowInfos = new CSFlowInfos(); + csFlowInfos.setInfos(infos); + contextValue.setValue(csFlowInfos); + + ContextKey contextKey = new CommonContextKey(); + contextKey.setKey("flow.infos"); + contextKey.setContextScope(ContextScope.PUBLIC); + contextKey.setContextType(ContextType.RESOURCE); + + ContextKeyValue contextKeyValue = new CommonContextKeyValue(); + contextKeyValue.setContextValue(contextValue); + contextKeyValue.setContextKey(contextKey); + context.setContextKeyAndValue(contextKeyValue); + ContextValue myValue = context.getContextValue(contextKey); + System.out.println(SerializeHelper.serializeContextID(context.getContextID())); + } +} diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ClientTestContextKey.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ClientTestContextKey.java new file mode 100644 index 0000000000..f9edabb3f0 --- /dev/null +++ b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ClientTestContextKey.java @@ -0,0 +1,68 @@ +package com.webank.wedatasphere.linkis.cs.client.test.bean; + +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKey; + +/** + * created by cooperyang on 2020/2/26 + * Description: + */ +public class ClientTestContextKey implements ContextKey { + + + private final String key = "cooperyang.txt"; + + + + + @Override + public String getKey() { + return this.key; + } + + @Override + public void setKey(String key) { + + } + + @Override + public int getType() { + return 0; + } + + @Override + public void setType(int type) { + + } + + @Override + public ContextType getContextType() { + return null; + } + + @Override + public void setContextType(ContextType contextType) { + + } + + @Override + public ContextScope getContextScope() { + return null; + } + + @Override + public void setContextScope(ContextScope contextScope) { + + } + + @Override + public String getKeywords() { + return null; + } + + @Override + public void setKeywords(String keywords) { + + } +} diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ClientTestContextValue.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ClientTestContextValue.java new file mode 100644 index 0000000000..d0f11e0aef --- /dev/null +++ b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ClientTestContextValue.java @@ -0,0 +1,37 @@ +package com.webank.wedatasphere.linkis.cs.client.test.bean; + +import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue; +import com.webank.wedatasphere.linkis.cs.common.entity.source.ValueBean; + +/** + * created by cooperyang on 2020/2/26 + * Description: + */ +public class ClientTestContextValue implements ContextValue { + + + + private Object value; + + private String keywords; + + @Override + public String getKeywords() { + return this.keywords; + } + + @Override + public void setKeywords(String keywords) { + this.keywords = keywords; + } + + @Override + public Object getValue() { + return this.value; + } + + @Override + public void setValue(Object value) { + this.value = value; + } +} diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ResourceValueBean.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ResourceValueBean.java new file mode 100644 index 0000000000..8632b0324b --- /dev/null +++ b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/bean/ResourceValueBean.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.linkis.cs.client.test.bean; + +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; +import com.webank.wedatasphere.linkis.cs.common.entity.resource.Resource; + +/** + * created by cooperyang on 2020/2/26 + * Description: + */ +public class ResourceValueBean implements Resource { + + private String resourceId; + private String version; + + private final ContextType contextType = ContextType.RESOURCE; + + public String getResourceId() { + return resourceId; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + +} diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/listener/CommonContextKeyListener.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/listener/CommonContextKeyListener.java new file mode 100644 index 0000000000..40cb99b11c --- /dev/null +++ b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/listener/CommonContextKeyListener.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.linkis.cs.client.test.listener; + +import com.webank.wedatasphere.linkis.common.listener.Event; +import com.webank.wedatasphere.linkis.cs.client.listener.ContextKeyListener; + +/** + * created by cooperyang on 2020/3/1 + * Description: + */ +public class CommonContextKeyListener extends ContextKeyListener { + + + + + @Override + public void onContextCreated(Event event) { + System.out.println("it is created"); + } + + @Override + public void onContextUpdated(Event event) { + super.onContextUpdated(event); + System.out.println("it is updated"); + } + + @Override + public void onEventError(Event event, Throwable t) { + System.out.println("it is not ok"); + } +} diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/restful/ListenerTest.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/restful/ListenerTest.java new file mode 100644 index 0000000000..e28baeff5f --- /dev/null +++ b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/restful/ListenerTest.java @@ -0,0 +1,8 @@ +package com.webank.wedatasphere.linkis.cs.client.test.restful; + +/** + * created by cooperyang on 2020/3/1 + * Description: + */ +public class ListenerTest { +} diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/restful/RestfulTest.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/restful/RestfulTest.java new file mode 100644 index 0000000000..6e52e0d57d --- /dev/null +++ b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/restful/RestfulTest.java @@ -0,0 +1,92 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.cs.client.test.restful; + +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.cs.client.ContextClient; +import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; +import com.webank.wedatasphere.linkis.cs.common.entity.resource.LinkisBMLResource; +import com.webank.wedatasphere.linkis.cs.common.entity.resource.Resource; +import com.webank.wedatasphere.linkis.cs.common.entity.source.*; + +/** + * Created by v_wbjftang on 2020/2/28. + */ +public class RestfulTest { + /** + * setValueByKey + */ + ContextClient contextClient = null; + + ContextID contextID = null; + + ContextKey contextKey = null; + + ContextValue contextValue = null; + + ContextKeyValue keyValue = null; + + + Resource resource = new LinkisBMLResource(); + + /*@Before*/ + public void init() { + contextClient = ContextClientFactory.getOrCreateContextClient(); + contextID = new LinkisHAWorkFlowContextID(); + contextID.setContextId("84716"); + + contextKey = new CommonContextKey(); + contextKey.setContextScope(ContextScope.FRIENDLY); + contextKey.setContextType(ContextType.ENV); + contextKey.setKey("project1.flow1.node1.key2"); + + contextValue = new CommonContextValue(); + LinkisBMLResource resource = new LinkisBMLResource(); + resource.setResourceId("dfasdfsr2456wertg"); + resource.setVersion("v000002"); + contextValue.setValue(resource); + + keyValue = new CommonContextKeyValue(); + keyValue.setContextKey(contextKey); + keyValue.setContextValue(contextValue); + + resource.setResourceId("edtr44356-34563456"); + resource.setVersion("v000004"); + + + } + + /** + * setValueByKey + * + * @throws ErrorException + */ + /* @org.junit.Test*/ + public void test01() throws ErrorException { + contextClient.update(contextID, contextKey, contextValue); + } + + /** + * reset + * + * @throws ErrorException + */ + /* @org.junit.Test*/ + public void test02() throws ErrorException { + contextClient.reset(contextID, contextKey); + } + +} diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestInfo.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestInfo.java new file mode 100644 index 0000000000..30afc0e251 --- /dev/null +++ b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestInfo.java @@ -0,0 +1,83 @@ +package com.webank.wedatasphere.linkis.cs.client.test.service; + +import com.google.gson.GsonBuilder; +import com.webank.wedatasphere.linkis.cs.client.Context; +import com.webank.wedatasphere.linkis.cs.client.ContextClient; +import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; +//import com.webank.wedatasphere.linkis.cs.client.service.DefaultSearchService; +import com.webank.wedatasphere.linkis.cs.client.service.DefaultSearchService; +import com.webank.wedatasphere.linkis.cs.client.service.SearchService; +import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; +import com.webank.wedatasphere.linkis.cs.common.entity.object.CSFlowInfos; +import com.webank.wedatasphere.linkis.cs.common.entity.resource.BMLResource; +import com.webank.wedatasphere.linkis.cs.common.entity.resource.LinkisBMLResource; +import com.webank.wedatasphere.linkis.cs.common.entity.source.*; +import com.webank.wedatasphere.linkis.cs.common.serialize.helper.ContextSerializationHelper; +import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils; +//import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @Author alexyang + * @Date 2020/3/16 + */ +public class TestInfo { + + + public static void main(String [] args) throws Exception { + + ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); + LinkisHAWorkFlowContextID contextID = new LinkisHAWorkFlowContextID(); + contextID.setFlow("test"); + contextID.setProject("test"); + contextID.setWorkSpace("test1"); + contextID.setVersion("v00001"); + Context context = contextClient.createContext(contextID); + System.out.println(context.getContextID().getContextId()); + + ContextSerializationHelper contextSerializationHelper = ContextSerializationHelper.getInstance(); + CommonContextValue contextValue = new CommonContextValue(); + Map infos = new HashMap<>(); + List> edges = new ArrayList<>(); + Map edge = new HashMap<>(); + edge.put("source","sql " ); + edge.put("target","hql"); + edge.put("sourceLocation","bottom"); + edge.put("targetLocation","top"); + edges.add(edge); + infos.put("edges", edges); + infos.put("parent", "flow2"); + Map idNodeName = new HashMap<>(); + idNodeName.put("90a6ee94-4bd6-47d9-a536-f92660c4c051", "sql"); + idNodeName.put("90a6ee94-4bd6-47d9-a536-f92660c4c052", "hql"); + infos.put("id_nodeName", idNodeName); + CSFlowInfos csFlowInfos = new CSFlowInfos(); + csFlowInfos.setInfos(infos); + contextValue.setValue(csFlowInfos); + + ContextKey contextKey = new CommonContextKey(); + contextKey.setKey("flow.infos"); + contextKey.setContextScope(ContextScope.PUBLIC); + contextKey.setContextType(ContextType.RESOURCE); + + ContextKeyValue contextKeyValue = new CommonContextKeyValue(); + contextKeyValue.setContextValue(contextValue); + contextKeyValue.setContextKey(contextKey); + context.setContextKeyAndValue(contextKeyValue); + ContextValue myValue = context.getContextValue(contextKey); + System.out.println(SerializeHelper.serializeContextID(context.getContextID())); + System.out.println(CSCommonUtils.gson.toJson(myValue)); + + SearchService searchService = DefaultSearchService.getInstance(); + BMLResource bmlResource = searchService.searchContext(contextID, "ql", "sql", LinkisBMLResource.class); + System.out.println(new GsonBuilder().setPrettyPrinting().create().toJson(bmlResource)); + + contextClient.close(); + } +} diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestRemove.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestRemove.java new file mode 100644 index 0000000000..4f296af6c2 --- /dev/null +++ b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestRemove.java @@ -0,0 +1,114 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.cs.client.test.service; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.webank.wedatasphere.linkis.common.exception.ErrorException; +import com.webank.wedatasphere.linkis.cs.client.Context; +import com.webank.wedatasphere.linkis.cs.client.ContextClient; +import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; +import com.webank.wedatasphere.linkis.cs.client.service.CSWorkService; +import com.webank.wedatasphere.linkis.cs.client.service.CSWorkServiceImpl; +import com.webank.wedatasphere.linkis.cs.client.utils.SerializeHelper; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.WorkType; +import com.webank.wedatasphere.linkis.cs.common.entity.resource.LinkisBMLResource; +import com.webank.wedatasphere.linkis.cs.common.entity.source.*; + +import static org.junit.Assert.assertEquals; + +/** + * @Author alexyang + * @Date 2020/3/21 + */ +public class TestRemove { + + private ContextClient contextClient; + private ContextID contextId; + private ContextKey contextKey; + private ContextValue contextValue; + private CSWorkService csWorkService; + private Gson gson; + + public static void main(String [] args) throws Exception { + TestRemove test = new TestRemove(); + test.init(); + test.testGetBeforeSet(); + test.testReset(); + test.testGetAfterSet(); + test.testClose(); + } + + + public void init() throws Exception { + contextClient = ContextClientFactory.getOrCreateContextClient(); + csWorkService = CSWorkServiceImpl.getInstance(); + gson = new GsonBuilder().setPrettyPrinting().create(); + LinkisHAWorkFlowContextID haContextID = new LinkisHAWorkFlowContextID(); + haContextID.setWorkSpace("wp1"); + haContextID.setProject("p1"); + haContextID.setFlow("f1"); + haContextID.setVersion("v1"); + if (null == contextId) { + Context context = contextClient.createContext(haContextID); + contextId = context.getContextID(); + } + + contextKey = new CommonContextKey(); + contextKey.setContextScope(ContextScope.FRIENDLY); + contextKey.setContextType(ContextType.ENV); + contextKey.setKey("project.flow1.node1.key2"); + + contextValue = new CommonContextValue(); + LinkisBMLResource resource = new LinkisBMLResource(); + resource.setResourceId("dfasdfsr2456wertg"); + resource.setVersion("v000002"); + contextValue.setValue(resource); + + ContextKeyValue keyValue = new CommonContextKeyValue(); + keyValue.setContextKey(contextKey); + keyValue.setContextValue(contextValue); + + contextClient.setContextKeyValue(haContextID, keyValue); + + } + + + public void testGetBeforeSet() throws ErrorException { + ContextValue value = contextClient.getContextValue(contextId, contextKey); + LinkisBMLResource resourceOri = (LinkisBMLResource)contextValue.getValue(); + LinkisBMLResource resourceRs = (LinkisBMLResource)value.getValue(); + assertEquals(resourceOri.getResourceId(), resourceRs.getResourceId()); + assertEquals(resourceOri.getVersion(), resourceRs.getVersion()); + System.out.println(gson.toJson(resourceOri)); + System.out.println(gson.toJson(resourceRs)); + } + + public void testReset() throws ErrorException { + csWorkService.initContextServiceInfo(SerializeHelper.serializeContextID(contextId), WorkType.PROJECT); + } + + public void testGetAfterSet() throws ErrorException { + ContextValue value = contextClient.getContextValue(contextId, contextKey); + assertEquals(null, value); + System.out.println(gson.toJson(contextValue)); + System.out.println(gson.toJson(value)); + } + + public void testClose() throws Exception { + contextClient.close(); + } +} diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestSearchService.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestSearchService.java new file mode 100644 index 0000000000..46080a63f1 --- /dev/null +++ b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/service/TestSearchService.java @@ -0,0 +1,112 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.cs.client.test.service; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.webank.wedatasphere.linkis.cs.client.Context; +import com.webank.wedatasphere.linkis.cs.client.ContextClient; +import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; +import com.webank.wedatasphere.linkis.cs.client.service.DefaultSearchService; +import com.webank.wedatasphere.linkis.cs.client.service.SearchService; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; +import com.webank.wedatasphere.linkis.cs.common.entity.object.CSFlowInfos; +import com.webank.wedatasphere.linkis.cs.common.entity.resource.BMLResource; +import com.webank.wedatasphere.linkis.cs.common.entity.resource.LinkisBMLResource; +import com.webank.wedatasphere.linkis.cs.common.entity.source.*; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @Author alexyang + * @Date 2020/3/8 + */ +public class TestSearchService { + + public static void main(String [] args) throws Exception { + + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + SearchService searchService = DefaultSearchService.getInstance(); + ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); + LinkisHAWorkFlowContextID contextID = new LinkisHAWorkFlowContextID(); + { + contextID.setUser("test"); + contextID.setProject("test_client"); + contextID.setFlow("test_client"); + Context context = contextClient.createContext(contextID); + contextID = (LinkisHAWorkFlowContextID) context.getContextID(); + System.out.println(new GsonBuilder().setPrettyPrinting().create().toJson(contextID)); + ContextKey contextKey = new CommonContextKey(); + contextKey.setKey("node.resource.sql.cooper.txt"); + contextKey.setContextType(ContextType.RESOURCE); + contextKey.setContextScope(ContextScope.PUBLIC); +// contextKey.setKeywords("xddd"); + ContextValue contextValue = new CommonContextValue(); + LinkisBMLResource linkisBMLResource = new LinkisBMLResource(); + linkisBMLResource.setVersion("v000001"); + linkisBMLResource.setDownloadedFileName("t.txt"); + linkisBMLResource.setResourceId("adsf"); + contextValue.setValue(linkisBMLResource); + ContextKeyValue contextKeyValue = new CommonContextKeyValue(contextKey, contextValue); + contextClient.setContextKeyValue(contextID, contextKeyValue); + BMLResource resource = searchService.getContextValue(contextID, contextKey, LinkisBMLResource.class); + System.out.println(gson.toJson(resource)); + ContextValue contextValue1 = contextClient.getContextValue(contextID, contextKey); + System.out.println(contextValue1.getValue()); + + } + + { + CommonContextValue contextValue = new CommonContextValue(); + Map infos = new HashMap<>(); + List> edges = new ArrayList<>(); + Map edge = new HashMap<>(); + edge.put("source","90a6ee94-4bd6-47d9-a536-f92660c4c051" ); + edge.put("target","90a6ee94-4bd6-47d9-a536-f92660c4c052"); + edge.put("sourceLocation","bottom"); + edge.put("targetLocation","top"); + edges.add(edge); + infos.put("edges", edges); + infos.put("parent", "flow2"); + Map idNodeName = new HashMap<>(); + idNodeName.put("90a6ee94-4bd6-47d9-a536-f92660c4c051", "sql"); + idNodeName.put("90a6ee94-4bd6-47d9-a536-f92660c4c052", "hql"); + infos.put("id_nodeName", idNodeName); + CSFlowInfos csFlowInfos = new CSFlowInfos(); + csFlowInfos.setInfos(infos); + contextValue.setValue(csFlowInfos); + + ContextKey contextKey = new CommonContextKey(); + contextKey.setKey("flow.infos"); + contextKey.setContextScope(ContextScope.PUBLIC); + contextKey.setContextType(ContextType.OBJECT); + + ContextKeyValue contextKeyValue = new CommonContextKeyValue(); + contextKeyValue.setContextValue(contextValue); + contextKeyValue.setContextKey(contextKey); + contextClient.setContextKeyValue(contextID, contextKeyValue); + Object flowInfoObject = searchService.getContextValue(contextID, contextKey, Object.class); + System.out.println(gson.toJson(flowInfoObject)); + } + + BMLResource bmlResource = searchService.searchContext(contextID, "cooper", "sql", LinkisBMLResource.class); + contextClient.close(); + System.out.println(gson.toJson(bmlResource)); + } + +} \ No newline at end of file diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/test_multiuser/TestChangeContext.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/test_multiuser/TestChangeContext.java new file mode 100644 index 0000000000..08a653f434 --- /dev/null +++ b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/test_multiuser/TestChangeContext.java @@ -0,0 +1,94 @@ +package com.webank.wedatasphere.linkis.cs.client.test.test_multiuser; + +import com.google.gson.Gson; +import com.webank.wedatasphere.linkis.cs.client.Context; +import com.webank.wedatasphere.linkis.cs.client.ContextClient; +import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; +import com.webank.wedatasphere.linkis.cs.common.entity.resource.LinkisBMLResource; +import com.webank.wedatasphere.linkis.cs.common.entity.source.*; +import com.webank.wedatasphere.linkis.cs.common.serialize.helper.ContextSerializationHelper; +import com.webank.wedatasphere.linkis.cs.common.serialize.helper.SerializationHelper; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; + +/** + * @Author alexyang + * @Date 2020/2/28 + */ +public class TestChangeContext { + + public static void main(String [] args) { + + ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); + try { + // 1, read context + File file = new File(TestCreateContext.CONTEXTID_PATH); + if (!file.exists()) { + System.out.println("Error, contextID serialize file : " + TestCreateContext.CONTEXTID_PATH + " invalid."); + return; + } + FileReader fr = new FileReader(file); + BufferedReader br = new BufferedReader(fr); + String contextIDStr = null; + StringBuilder builder = new StringBuilder(""); + java.lang.String tmp = br.readLine(); + while (null != tmp) { + builder.append(tmp); + tmp = br.readLine(); + } + br.close(); + contextIDStr = builder.toString(); + System.out.println("Read contextID : " + contextIDStr); + + SerializationHelper serializationHelper = ContextSerializationHelper.getInstance(); + if (!serializationHelper.accepts(contextIDStr)) { + System.out.println("Invalid contextStr : " + contextIDStr + ", cannot be deserialized"); + return; + } + Object contextObject = serializationHelper.deserialize(contextIDStr); + System.out.println("Deserialized obj : " + new Gson().toJson(contextObject)); + if (!LinkisHAWorkFlowContextID.class.isInstance(contextObject)) { + System.out.println("Invalid contextObject, not LinkisHAWorkFlowContextID instance."); + return; + } + LinkisHAWorkFlowContextID contextID = (LinkisHAWorkFlowContextID) contextObject; + + Context context = contextClient.getContext(contextID); + // 2, update context + ContextKey contextKey = new CommonContextKey(); + contextKey.setKey("testchange.txt"); + contextKey.setKeywords("xddd"); + contextKey.setContextScope(ContextScope.PUBLIC); + contextKey.setContextType(ContextType.RESOURCE); + ContextValue contextValue = new CommonContextValue(); + LinkisBMLResource resource = new LinkisBMLResource(); + resource.setResourceId("456789"); + resource.setVersion("v00001"); + contextValue.setValue(resource); + ContextKeyValue contextKeyValue = new CommonContextKeyValue(); + contextKeyValue.setContextValue(contextValue); + contextKeyValue.setContextKey(contextKey); + context.setContextKeyAndValue(contextKeyValue); + + // 3, get context + ContextValue contextValueResult = context.getContextValue(contextKey); + System.out.println("Got contextValue : " + new Gson().toJson(contextValueResult)); + System.out.println("Original contextValue : " + new Gson().toJson(contextValue)); + + contextClient.close(); + } catch (Exception e) { + if (null != contextClient) { + try { + contextClient.close(); + } catch (Exception e1) { + + } + } + e.printStackTrace(); + } + } +} diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/test_multiuser/TestCreateContext.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/test_multiuser/TestCreateContext.java new file mode 100644 index 0000000000..2c8c7ca28f --- /dev/null +++ b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/test_multiuser/TestCreateContext.java @@ -0,0 +1,91 @@ +package com.webank.wedatasphere.linkis.cs.client.test.test_multiuser; + +import com.google.gson.Gson; +import com.webank.wedatasphere.linkis.cs.client.Context; +import com.webank.wedatasphere.linkis.cs.client.ContextClient; +import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; +import com.webank.wedatasphere.linkis.cs.client.service.DefaultSearchService; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; +import com.webank.wedatasphere.linkis.cs.common.entity.resource.BMLResource; +import com.webank.wedatasphere.linkis.cs.common.entity.resource.LinkisBMLResource; +import com.webank.wedatasphere.linkis.cs.common.entity.source.*; +import com.webank.wedatasphere.linkis.cs.common.serialize.helper.ContextSerializationHelper; +import com.webank.wedatasphere.linkis.cs.common.serialize.helper.SerializationHelper; +import scala.tools.nsc.doc.model.Def; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.util.ArrayList; +import java.util.List; + +/** + * @Author alexyang + * @Date 2020/2/28 + */ +public class TestCreateContext { + + public static final String CONTEXTID_PATH = "out/test-store-contextID.txt"; + + public static void main(String [] args) throws Exception { + + // 1, create contextid + ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); + try { + + Context context = contextClient.createContext("test_client", "test_client", "alex", null); + System.out.println(context.getContextID().getContextId()); + + // 2, save contxtid + ContextID contextID = context.getContextID(); + SerializationHelper serializationHelper = ContextSerializationHelper.getInstance(); + String contextIDStr = null; + if (serializationHelper.accepts(contextID)) { + contextIDStr = serializationHelper.serialize(contextID); + File file = new File(CONTEXTID_PATH); + FileWriter fr = new FileWriter(file); + BufferedWriter bw = new BufferedWriter(fr); + bw.write(contextIDStr); + bw.flush(); + bw.close(); + System.out.println("ContextID saved at : " + file.getAbsolutePath()); + // test deserialize + System.out.println("contextIDStr: " + contextIDStr); + Object obj = serializationHelper.deserialize(contextIDStr); + System.out.println("Deserialize jsonstr : " + new Gson().toJson(obj)); + } else { + System.out.println("ContextID not saved."); + } + + // 3, test search + ContextKey contextKey = new CommonContextKey(); + contextKey.setKey("cooper.txt"); + contextKey.setKeywords("xddd"); + contextKey.setContextScope(ContextScope.PUBLIC); + contextKey.setContextType(ContextType.RESOURCE); + ContextValue contextValue = new CommonContextValue(); + LinkisBMLResource resource = new LinkisBMLResource(); + resource.setResourceId("456789"); + resource.setVersion("v00001"); + contextValue.setValue(resource); + ContextKeyValue contextKeyValue = new CommonContextKeyValue(); + contextKeyValue.setContextValue(contextValue); + contextKeyValue.setContextKey(contextKey); + context.setContextKeyAndValue(contextKeyValue); + ContextValue myValue = context.getContextValue(contextKey); + LinkisBMLResource linkisBMLResource = (LinkisBMLResource)myValue.getValue(); + System.out.println(linkisBMLResource.getResourceId()); + BMLResource rs = DefaultSearchService.getInstance().getContextValue(contextID, contextKey, LinkisBMLResource.class); + System.out.println(new Gson().toJson(rs)); + List contains = new ArrayList<>(); + contains.add("cooper"); + List contextKeyValueList = contextClient.search(context.getContextID(), null, null, contains, null); + System.out.println("ContextKVList : " + new Gson().toJson(contextKeyValueList)); + } catch (Exception e) { + contextClient.close(); + e.printStackTrace(); + } + + } +} From d38f2086158e4a10fe071ddb65b7967c4d7812af Mon Sep 17 00:00:00 2001 From: David hua Date: Wed, 3 Jun 2020 20:51:56 +0800 Subject: [PATCH 5/8] Delete TestCreateContext.java Remove classes for testing --- .../test_multiuser/TestCreateContext.java | 91 ------------------- 1 file changed, 91 deletions(-) delete mode 100644 contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/test_multiuser/TestCreateContext.java diff --git a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/test_multiuser/TestCreateContext.java b/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/test_multiuser/TestCreateContext.java deleted file mode 100644 index 2c8c7ca28f..0000000000 --- a/contextservice/cs-client/src/test/java/com/webank/wedatasphere/linkis/cs/client/test/test_multiuser/TestCreateContext.java +++ /dev/null @@ -1,91 +0,0 @@ -package com.webank.wedatasphere.linkis.cs.client.test.test_multiuser; - -import com.google.gson.Gson; -import com.webank.wedatasphere.linkis.cs.client.Context; -import com.webank.wedatasphere.linkis.cs.client.ContextClient; -import com.webank.wedatasphere.linkis.cs.client.builder.ContextClientFactory; -import com.webank.wedatasphere.linkis.cs.client.service.DefaultSearchService; -import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextScope; -import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.ContextType; -import com.webank.wedatasphere.linkis.cs.common.entity.resource.BMLResource; -import com.webank.wedatasphere.linkis.cs.common.entity.resource.LinkisBMLResource; -import com.webank.wedatasphere.linkis.cs.common.entity.source.*; -import com.webank.wedatasphere.linkis.cs.common.serialize.helper.ContextSerializationHelper; -import com.webank.wedatasphere.linkis.cs.common.serialize.helper.SerializationHelper; -import scala.tools.nsc.doc.model.Def; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.util.ArrayList; -import java.util.List; - -/** - * @Author alexyang - * @Date 2020/2/28 - */ -public class TestCreateContext { - - public static final String CONTEXTID_PATH = "out/test-store-contextID.txt"; - - public static void main(String [] args) throws Exception { - - // 1, create contextid - ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); - try { - - Context context = contextClient.createContext("test_client", "test_client", "alex", null); - System.out.println(context.getContextID().getContextId()); - - // 2, save contxtid - ContextID contextID = context.getContextID(); - SerializationHelper serializationHelper = ContextSerializationHelper.getInstance(); - String contextIDStr = null; - if (serializationHelper.accepts(contextID)) { - contextIDStr = serializationHelper.serialize(contextID); - File file = new File(CONTEXTID_PATH); - FileWriter fr = new FileWriter(file); - BufferedWriter bw = new BufferedWriter(fr); - bw.write(contextIDStr); - bw.flush(); - bw.close(); - System.out.println("ContextID saved at : " + file.getAbsolutePath()); - // test deserialize - System.out.println("contextIDStr: " + contextIDStr); - Object obj = serializationHelper.deserialize(contextIDStr); - System.out.println("Deserialize jsonstr : " + new Gson().toJson(obj)); - } else { - System.out.println("ContextID not saved."); - } - - // 3, test search - ContextKey contextKey = new CommonContextKey(); - contextKey.setKey("cooper.txt"); - contextKey.setKeywords("xddd"); - contextKey.setContextScope(ContextScope.PUBLIC); - contextKey.setContextType(ContextType.RESOURCE); - ContextValue contextValue = new CommonContextValue(); - LinkisBMLResource resource = new LinkisBMLResource(); - resource.setResourceId("456789"); - resource.setVersion("v00001"); - contextValue.setValue(resource); - ContextKeyValue contextKeyValue = new CommonContextKeyValue(); - contextKeyValue.setContextValue(contextValue); - contextKeyValue.setContextKey(contextKey); - context.setContextKeyAndValue(contextKeyValue); - ContextValue myValue = context.getContextValue(contextKey); - LinkisBMLResource linkisBMLResource = (LinkisBMLResource)myValue.getValue(); - System.out.println(linkisBMLResource.getResourceId()); - BMLResource rs = DefaultSearchService.getInstance().getContextValue(contextID, contextKey, LinkisBMLResource.class); - System.out.println(new Gson().toJson(rs)); - List contains = new ArrayList<>(); - contains.add("cooper"); - List contextKeyValueList = contextClient.search(context.getContextID(), null, null, contains, null); - System.out.println("ContextKVList : " + new Gson().toJson(contextKeyValueList)); - } catch (Exception e) { - contextClient.close(); - e.printStackTrace(); - } - - } -} From b372441f77ef499a5d5340f48f8f00e2fd973d24 Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Wed, 3 Jun 2020 21:11:29 +0800 Subject: [PATCH 6/8] bml add a delete interface --- bml/bml-engine-hook/pom.xml | 8 ++--- .../bml/hook/BmlEnginePreExecuteHook.scala | 7 ++-- .../linkis/bml/client/AbstractBmlClient.java | 2 +- .../linkis/bml/client/BmlClient.java | 7 ++++ .../bml/client/impl/HttpBmlClient.scala | 25 ++++++++++++++ .../linkis/bml/conf/BmlConfiguration.scala | 6 ++-- .../linkis/bml/http/HttpConf.scala | 12 +++---- .../linkis/bml/request/BmlPOSTAction.scala | 10 +++--- .../linkis/bml/response/BmlResult.scala | 7 ++-- .../linkis/bml/protocol/BmlProtocol.scala | 3 ++ bml/bmlserver/bin/start-bml.sh | 2 +- .../linkis/bml/common/HdfsResourceHelper.java | 1 - .../linkis/bml/dao/DownloadDao.java | 2 +- .../linkis/bml/dao/VersionDao.java | 8 ++--- .../linkis/bml/restful/BmlRestfulApi.java | 33 ++++--------------- .../linkis/bml/service/TaskService.java | 2 +- .../linkis/bml/service/VersionService.java | 4 +-- .../bml/service/impl/ResourceServiceImpl.java | 4 +-- .../bml/service/impl/TaskServiceImpl.java | 2 +- .../bml/service/impl/VersionServiceImpl.java | 1 - 20 files changed, 79 insertions(+), 67 deletions(-) diff --git a/bml/bml-engine-hook/pom.xml b/bml/bml-engine-hook/pom.xml index 3808605c76..821a0a4077 100644 --- a/bml/bml-engine-hook/pom.xml +++ b/bml/bml-engine-hook/pom.xml @@ -14,10 +14,10 @@ - com.webank.wedatasphere.linkis - linkis-common - provided - + com.webank.wedatasphere.linkis + linkis-common + provided + com.webank.wedatasphere.linkis linkis-bmlclient diff --git a/bml/bml-engine-hook/src/main/scala/com/webank/wedatasphere/linkis/bml/hook/BmlEnginePreExecuteHook.scala b/bml/bml-engine-hook/src/main/scala/com/webank/wedatasphere/linkis/bml/hook/BmlEnginePreExecuteHook.scala index 28f3cc0403..22ed4b4cc7 100644 --- a/bml/bml-engine-hook/src/main/scala/com/webank/wedatasphere/linkis/bml/hook/BmlEnginePreExecuteHook.scala +++ b/bml/bml-engine-hook/src/main/scala/com/webank/wedatasphere/linkis/bml/hook/BmlEnginePreExecuteHook.scala @@ -41,14 +41,12 @@ class BmlEnginePreExecuteHook extends EnginePreExecuteHook with Logging{ val pathType:String = "file://" - override def callPreExecuteHook(engineExecutorContext: EngineExecutorContext, executeRequest: ExecuteRequest): Unit = { - //1.删除工作目录以前的资源文件 - //2.下载资源到当前进程的工作目录 - + override def callPreExecuteHook(engineExecutorContext: EngineExecutorContext, executeRequest: ExecuteRequest, code: String): String = { val workDir = BmlHookUtils.getCurrentWorkDir val jobId = engineExecutorContext.getJobId executeRequest match { case resourceExecuteRequest:ResourceExecuteRequest => val resources = resourceExecuteRequest.resources + if (null == resources) return code resources foreach { case resource:util.Map[String, Object] => val fileName = resource.get(FILE_NAME_STR).toString val resourceId = resource.get(RESOURCE_ID_STR).toString @@ -74,5 +72,6 @@ class BmlEnginePreExecuteHook extends EnginePreExecuteHook with Logging{ } case _ => } + if (StringUtils.isNotBlank(code)) code else executeRequest.code } } diff --git a/bml/bmlclient/src/main/java/com/webank/wedatasphere/linkis/bml/client/AbstractBmlClient.java b/bml/bmlclient/src/main/java/com/webank/wedatasphere/linkis/bml/client/AbstractBmlClient.java index 48ccd27fb3..5b9620f201 100644 --- a/bml/bmlclient/src/main/java/com/webank/wedatasphere/linkis/bml/client/AbstractBmlClient.java +++ b/bml/bmlclient/src/main/java/com/webank/wedatasphere/linkis/bml/client/AbstractBmlClient.java @@ -23,7 +23,7 @@ */ public abstract class AbstractBmlClient implements BmlClient{ protected String user; - protected java.util.Map properties; + protected Map properties; public String getUser() { return user; diff --git a/bml/bmlclient/src/main/java/com/webank/wedatasphere/linkis/bml/client/BmlClient.java b/bml/bmlclient/src/main/java/com/webank/wedatasphere/linkis/bml/client/BmlClient.java index 144592615a..e8d5a55210 100644 --- a/bml/bmlclient/src/main/java/com/webank/wedatasphere/linkis/bml/client/BmlClient.java +++ b/bml/bmlclient/src/main/java/com/webank/wedatasphere/linkis/bml/client/BmlClient.java @@ -91,4 +91,11 @@ public interface BmlClient { public BmlResourceVersionsResponse getVersions(String user, String resourceId); + /** + * + */ + public BmlDeleteResponse deleteResource(String user, String resourceId, String version); + + public BmlDeleteResponse deleteResource(String user, String resourceId); + } diff --git a/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/client/impl/HttpBmlClient.scala b/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/client/impl/HttpBmlClient.scala index 76d3dc7ba6..e7d63672a4 100644 --- a/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/client/impl/HttpBmlClient.scala +++ b/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/client/impl/HttpBmlClient.scala @@ -290,6 +290,31 @@ class HttpBmlClient extends AbstractBmlClient{ } + /** + * + */ + override def deleteResource(user: String, resourceId: String, version: String): BmlDeleteResponse = { + null + } + + override def deleteResource(user: String, resourceId: String): BmlDeleteResponse = { + val deleteAction = BmlDeleteAction(resourceId) + deleteAction.getParameters.put("resourceId", resourceId) + val result = dwsClient.execute(deleteAction) + result match { + case bmlDeleteResult: BmlDeleteResult => val isSuccess= if (bmlDeleteResult.getStatus == 0) true else false + if (isSuccess){ + BmlDeleteResponse(isSuccess) + }else{ + logger.error(s"user $user update resource failed, status code is ${bmlDeleteResult.getStatusCode}") + BmlDeleteResponse(isSuccess) + } + case r:BmlResult => logger.error(s"result type ${r.getResultType} not match BmlResourceDownloadResult") + throw POSTResultNotMatchException() + case _ => throw POSTResultNotMatchException() + } + } + //todo 现在是为了通过编译 private def getInputStream(str: String):InputStream = { null diff --git a/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/conf/BmlConfiguration.scala b/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/conf/BmlConfiguration.scala index 2bb0ce5455..cc2e6d9990 100644 --- a/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/conf/BmlConfiguration.scala +++ b/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/conf/BmlConfiguration.scala @@ -22,11 +22,7 @@ import com.webank.wedatasphere.linkis.common.conf.CommonVars * Description: */ object BmlConfiguration { - val GATEWAY_IP:CommonVars[String] = - CommonVars[String]("wds.linkis.gateway.ip", "ip", "DWS gateway的ip地址") - val GATEWAY_PORT:CommonVars[Int] = - CommonVars[Int]("wds.linkis.gateway.port", 1000, "DWS gateway的端口") val DWS_VERSION:CommonVars[String] = CommonVars[String]("wds.linkis.bml.dws.version", "v1") @@ -47,6 +43,8 @@ object BmlConfiguration { val DOWNLOAD_URL:CommonVars[String] = CommonVars[String]("wds.linkis.bml.download.url", "download") + val DELETE_URL:CommonVars[String] = CommonVars[String]("wds.linkis.bml.delete.url", "delete") + val GET_VERSIONS_URL:CommonVars[String] = CommonVars[String]("wds.linkis.bml.getVersions.url", "getVersions") val GET_BASIC_URL:CommonVars[String] = CommonVars[String]("wds.linkis.bml.getBasic.url","getBasic") diff --git a/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/http/HttpConf.scala b/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/http/HttpConf.scala index 57029704bf..4d335e3e79 100644 --- a/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/http/HttpConf.scala +++ b/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/http/HttpConf.scala @@ -16,22 +16,22 @@ package com.webank.wedatasphere.linkis.bml.http import com.webank.wedatasphere.linkis.bml.conf.BmlConfiguration +import com.webank.wedatasphere.linkis.common.conf.Configuration /** * created by cooperyang on 2019/5/15 * Description: */ object HttpConf { - val ip:String = BmlConfiguration.GATEWAY_IP.getValue - val port:Int = BmlConfiguration.GATEWAY_PORT.getValue - val schema:String = "http://" - val gatewayInstance:String = schema + ip + ":" + port - val urlPrefix:String = if (BmlConfiguration.URL_PREFIX.getValue.endsWith("/")) { + + val gatewayInstance: String = Configuration.getGateWayURL() + val urlPrefix: String = if (BmlConfiguration.URL_PREFIX.getValue.endsWith("/")) { BmlConfiguration.URL_PREFIX.getValue.substring(0, BmlConfiguration.URL_PREFIX.getValue.length - 1) - }else BmlConfiguration.URL_PREFIX.getValue + } else BmlConfiguration.URL_PREFIX.getValue val uploadURL:String = urlPrefix + "/" + BmlConfiguration.UPLOAD_URL.getValue val downloadURL:String = urlPrefix + "/" + BmlConfiguration.DOWNLOAD_URL.getValue + val deleteURL:String = urlPrefix + "/" + BmlConfiguration.DELETE_URL val updateVersionURL:String = urlPrefix + "/" + BmlConfiguration.UPDATE_VERSION_URL.getValue val relateHdfsURL:String = gatewayInstance + urlPrefix + "/" + BmlConfiguration.RELATE_HDFS.getValue val relateStorageURL:String = gatewayInstance + urlPrefix + "/" + BmlConfiguration.RELATE_STORAGE.getValue diff --git a/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/request/BmlPOSTAction.scala b/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/request/BmlPOSTAction.scala index db40ec9f40..be915bea2c 100644 --- a/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/request/BmlPOSTAction.scala +++ b/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/request/BmlPOSTAction.scala @@ -19,11 +19,7 @@ import java.io.{File, InputStream} import java.util import com.webank.wedatasphere.linkis.bml.http.HttpConf - -import scala.collection.JavaConversions._ -import com.webank.wedatasphere.linkis.common.io.FsPath import com.webank.wedatasphere.linkis.httpclient.request._ -import com.webank.wedatasphere.linkis.storage.FSFactory /** * created by cooperyang on 2019/5/23 @@ -173,5 +169,11 @@ case class BmlGetBasicAction(resourceId:String) extends BmlGETAction with UserAc +case class BmlDeleteAction(resourceId:String) extends BmlPOSTAction { + override def getRequestPayload: String = "" + + override def getURL: String = HttpConf.deleteURL +} + diff --git a/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/response/BmlResult.scala b/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/response/BmlResult.scala index b43ca48529..a63943bab0 100644 --- a/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/response/BmlResult.scala +++ b/bml/bmlclient/src/main/scala/com/webank/wedatasphere/linkis/bml/response/BmlResult.scala @@ -21,8 +21,6 @@ import com.webank.wedatasphere.linkis.bml.protocol.ResourceVersions import com.webank.wedatasphere.linkis.httpclient.dws.annotation.DWSHttpMessageResult import com.webank.wedatasphere.linkis.httpclient.dws.response.DWSResult -import scala.beans.BeanProperty - /** * created by cooperyang on 2019/5/23 * Description: @@ -106,3 +104,8 @@ class BmlGetBasicResult extends BmlResult{ def getDownloadedFileName:String = this.downloadedFileName def setDownloadedFileName(downloadedFileName:String):Unit = this.downloadedFileName = downloadedFileName } + +@DWSHttpMessageResult("/api/rest_j/v\\d+/bml/delete") +class BmlDeleteResult extends BmlResult{ + +} \ No newline at end of file diff --git a/bml/bmlcommon/src/main/scala/com/webank/wedatasphere/linkis/bml/protocol/BmlProtocol.scala b/bml/bmlcommon/src/main/scala/com/webank/wedatasphere/linkis/bml/protocol/BmlProtocol.scala index 25465ce39e..c8f29e4086 100644 --- a/bml/bmlcommon/src/main/scala/com/webank/wedatasphere/linkis/bml/protocol/BmlProtocol.scala +++ b/bml/bmlcommon/src/main/scala/com/webank/wedatasphere/linkis/bml/protocol/BmlProtocol.scala @@ -74,3 +74,6 @@ case class ResourceVersions(resourceId:String, case class BmlResourceVersionsResponse(isSuccess:Boolean, resourceId:String, resourceVersions: ResourceVersions) extends BmlResponse(isSuccess) + + +case class BmlDeleteResponse(isSuccess:Boolean) extends BmlResponse(isSuccess) \ No newline at end of file diff --git a/bml/bmlserver/bin/start-bml.sh b/bml/bmlserver/bin/start-bml.sh index 56cab6df00..80cc775a4a 100644 --- a/bml/bmlserver/bin/start-bml.sh +++ b/bml/bmlserver/bin/start-bml.sh @@ -26,7 +26,7 @@ if [[ -f "${SERVER_PID}" ]]; then fi fi -nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & +nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & pid=$! if [[ -z "${pid}" ]]; then echo "server $SERVER_NAME start failed!" diff --git a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/common/HdfsResourceHelper.java b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/common/HdfsResourceHelper.java index 1b3a32e145..35072a6f3c 100644 --- a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/common/HdfsResourceHelper.java +++ b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/common/HdfsResourceHelper.java @@ -20,7 +20,6 @@ import com.webank.wedatasphere.linkis.common.io.FsPath; import com.webank.wedatasphere.linkis.storage.FSFactory; import com.webank.wedatasphere.linkis.storage.utils.FileSystemUtils; - import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.IOUtils; diff --git a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/dao/DownloadDao.java b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/dao/DownloadDao.java index d28673e961..eb2c8671f8 100644 --- a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/dao/DownloadDao.java +++ b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/dao/DownloadDao.java @@ -25,7 +25,7 @@ */ public interface DownloadDao { - void insertDownloadModel(@Param("downloadModel")DownloadModel downloadModel); + void insertDownloadModel(@Param("downloadModel") DownloadModel downloadModel); } diff --git a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/dao/VersionDao.java b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/dao/VersionDao.java index be5e947b65..2c23ab2a61 100644 --- a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/dao/VersionDao.java +++ b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/dao/VersionDao.java @@ -55,7 +55,7 @@ public interface VersionDao { @Select("select start_byte from linkis_resources_version where resource_id = #{resourceId} and version = #{version}") - long getStartByteForResource(@Param("resourceId") String resourceId, @Param("version")String version); + long getStartByteForResource(@Param("resourceId") String resourceId, @Param("version") String version); @@ -75,14 +75,14 @@ ResourceVersion findResourceVersion(@Param("resourceId") String resourceId, @Select("select enable_flag from `linkis_resources_version` where resource_id = #{resourceId} and version = #{version}") - int selectResourceVersionEnbleFlag(@Param("resourceId") String resourceId, @Param("version")String version); + int selectResourceVersionEnbleFlag(@Param("resourceId") String resourceId, @Param("version") String version); /** * 将resourceId对应的所有版本的enable_flag设为0，这样就不能继续访问该资源的任意版本 * @param resourceId resourceId */ @Update("update `linkis_resources_version` set enable_flag = 0 where resource_id = #{resourceId}") - void deleteResource(@Param("resourceId")String resourceId); + void deleteResource(@Param("resourceId") String resourceId); void batchDeleteResources(@Param("resourceIds") List resourceIds); @@ -92,6 +92,6 @@ ResourceVersion findResourceVersion(@Param("resourceId") String resourceId, - List selectVersionByPage(@Param("resourceId")String resourceId); + List selectVersionByPage(@Param("resourceId") String resourceId); } diff --git a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/restful/BmlRestfulApi.java b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/restful/BmlRestfulApi.java index 072d377d9c..412bfeb813 100644 --- a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/restful/BmlRestfulApi.java +++ b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/restful/BmlRestfulApi.java @@ -15,17 +15,9 @@ */ package com.webank.wedatasphere.linkis.bml.restful; -import com.webank.wedatasphere.linkis.bml.Entity.DownloadModel; -import com.webank.wedatasphere.linkis.bml.Entity.Resource; -import com.webank.wedatasphere.linkis.bml.Entity.ResourceTask; -import com.webank.wedatasphere.linkis.bml.Entity.ResourceVersion; -import com.webank.wedatasphere.linkis.bml.Entity.Version; -import com.webank.wedatasphere.linkis.bml.common.Constant; -import com.webank.wedatasphere.linkis.bml.service.BmlService; -import com.webank.wedatasphere.linkis.bml.service.DownloadService; -import com.webank.wedatasphere.linkis.bml.service.ResourceService; -import com.webank.wedatasphere.linkis.bml.service.TaskService; -import com.webank.wedatasphere.linkis.bml.service.VersionService; +import com.webank.wedatasphere.linkis.bml.Entity.*; +import com.webank.wedatasphere.linkis.bml.common.*; +import com.webank.wedatasphere.linkis.bml.service.*; import com.webank.wedatasphere.linkis.bml.threading.TaskState; import com.webank.wedatasphere.linkis.bml.util.HttpRequestHelper; import com.webank.wedatasphere.linkis.bml.vo.ResourceBasicVO; @@ -33,8 +25,6 @@ import com.webank.wedatasphere.linkis.bml.vo.ResourceVersionsVO; import com.webank.wedatasphere.linkis.common.exception.ErrorException; import com.webank.wedatasphere.linkis.server.Message; -import com.webank.wedatasphere.linkis.bml.common.*; - import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.codehaus.jackson.JsonNode; @@ -45,25 +35,14 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import javax.ws.rs.Consumes; -import javax.ws.rs.GET; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; +import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.io.IOException; +import java.util.*; /** * created by cooperyang on 2019/5/14 diff --git a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/TaskService.java b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/TaskService.java index 5aed763d09..cad6695cdc 100644 --- a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/TaskService.java +++ b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/TaskService.java @@ -53,7 +53,7 @@ ResourceTask createUpdateTask(String resourceId, String user, FormDataMultiPart * @param updateTime 操作时间 * @param errMsg 异常信息 */ - void updateState2Failed( long taskId, String state, Date updateTime, String errMsg); + void updateState2Failed(long taskId, String state, Date updateTime, String errMsg); ResourceTask createDeleteVersionTask(String resourceId, String version, String user, String ip); diff --git a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/VersionService.java b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/VersionService.java index 7b218709fe..332a8b53ed 100644 --- a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/VersionService.java +++ b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/VersionService.java @@ -38,7 +38,7 @@ public interface VersionService { List getAllResourcesViaSystem(String system, String user); //分页查询VResourcesViaSystem - public List selectResourcesViaSystemByPage(int currentPage, int pageSize,String system, String user); + public List selectResourcesViaSystemByPage(int currentPage, int pageSize, String system, String user); void deleteResourceVersion(String resourceId, String version); @@ -55,7 +55,7 @@ public interface VersionService { List getVersions(String resourceId); //分页查询Version - List selectVersionByPage(int currentPage, int pageSize,String resourceId); + List selectVersionByPage(int currentPage, int pageSize, String resourceId); diff --git a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/ResourceServiceImpl.java b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/ResourceServiceImpl.java index 06379f9196..fe0a4868b0 100644 --- a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/ResourceServiceImpl.java +++ b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/ResourceServiceImpl.java @@ -24,7 +24,6 @@ import com.webank.wedatasphere.linkis.bml.dao.VersionDao; import com.webank.wedatasphere.linkis.bml.service.ResourceService; import com.webank.wedatasphere.linkis.common.exception.ErrorException; - import org.apache.commons.lang.StringUtils; import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; @@ -35,13 +34,12 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import javax.validation.constraints.NotNull; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; -import javax.validation.constraints.NotNull; - /** * Created by cooperyang on 2019/5/17. */ diff --git a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/TaskServiceImpl.java b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/TaskServiceImpl.java index cc7a3dc0ef..830b6c084a 100644 --- a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/TaskServiceImpl.java +++ b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/TaskServiceImpl.java @@ -18,6 +18,7 @@ import com.webank.wedatasphere.linkis.bml.Entity.ResourceTask; import com.webank.wedatasphere.linkis.bml.Entity.Version; import com.webank.wedatasphere.linkis.bml.common.Constant; +import com.webank.wedatasphere.linkis.bml.common.UpdateResourceException; import com.webank.wedatasphere.linkis.bml.dao.ResourceDao; import com.webank.wedatasphere.linkis.bml.dao.TaskDao; import com.webank.wedatasphere.linkis.bml.dao.VersionDao; @@ -25,7 +26,6 @@ import com.webank.wedatasphere.linkis.bml.service.TaskService; import com.webank.wedatasphere.linkis.bml.service.VersionService; import com.webank.wedatasphere.linkis.bml.threading.TaskState; -import com.webank.wedatasphere.linkis.bml.common.*; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.glassfish.jersey.media.multipart.FormDataMultiPart; diff --git a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/VersionServiceImpl.java b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/VersionServiceImpl.java index 9d0142e57b..1003fa1fc1 100644 --- a/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/VersionServiceImpl.java +++ b/bml/bmlserver/src/main/java/com/webank/wedatasphere/linkis/bml/service/impl/VersionServiceImpl.java @@ -28,7 +28,6 @@ import com.webank.wedatasphere.linkis.common.io.Fs; import com.webank.wedatasphere.linkis.common.io.FsPath; import com.webank.wedatasphere.linkis.storage.FSFactory; - import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.glassfish.jersey.media.multipart.FormDataBodyPart; From e2718dfffd0e361bb358a498e23604f3c5ab4cf0 Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Wed, 3 Jun 2020 22:03:08 +0800 Subject: [PATCH 7/8] update ujes module to adjust context service --- .../ujes/client/response/JobInfoResult.scala | 10 +- .../client/response/JobProgressResult.scala | 1 - .../hive/HiveEngineSpringConfiguration.scala | 2 +- .../linkis/engine/hive/common/HiveUtils.scala | 3 +- .../hive/executor/HiveEngineExecutor.scala | 2 +- .../hive/hook/UseDatabaseEngineHook.scala | 2 - .../hive/progress/HiveProgressHelper.scala | 1 - .../hive/process/HiveQLProcessBuilder.scala | 5 +- .../hive/entrance/bin/start-hive-entrance.sh | 2 +- .../jdbc/entrance/bin/start-jdbc-entrance.sh | 2 +- ujes/definedEngines/jdbc/entrance/pom.xml | 4 +- .../conf/executer/ConnectionManager.java | 4 +- .../conf/JDBCSpringConfiguration.scala | 1 - .../engine/pipeline/OutputStreamCache.scala | 2 +- .../engine/pipeline/PipeEngineJob.scala | 2 +- .../pipeline/PipelineEngineParser.scala | 2 +- .../pipeline/constant/PipeLineConstant.scala | 2 +- .../pipeline/conversions/FsConvertions.scala | 37 ++++ .../pipeline/executor/CSVExecutor.scala | 2 +- .../pipeline/executor/CopyExecutor.scala | 2 +- .../pipeline/executor/ExcelExecutor.scala | 2 +- .../PipeLineDefaultEngineCreator.scala | 1 - .../PipeLineProcessEngineBuilder.scala | 1 - .../entrance/bin/start-pipeline-entrance.sh | 2 +- .../linkis/engine/ExecuteResponse.scala | 1 - .../engine/HDFSResultSetOutputStream.scala | 1 + .../PythonEngineSpringConfiguration.scala | 2 +- .../linkis/engine/PythonSession.scala | 23 ++- .../executors/PythonEngineExecutor.scala | 6 +- .../python/enginemanager/pom.xml | 5 +- .../PythonEngineSpringConfiguration.scala | 2 - .../entrance/bin/start-python-entrance.sh | 2 +- .../shell/executor/ShellEngineExecutor.scala | 7 +- .../executor/ShellEngineExecutorFactory.scala | 2 +- .../ShellEngineManagerConfiguration.scala | 5 +- .../entrance/bin/start-shell-entrance.sh | 2 +- .../Interpreter/ProcessInterpreter.scala | 5 +- .../Interpreter/PythonInterpreter.scala | 13 +- .../configuration/SparkConfiguration.scala | 3 + ...SparkEngineServerSpringConfiguration.scala | 73 ++++---- .../linkis/engine/cs/CSSparkHelper.scala | 34 ++++ .../engine/cs/CSSparkPreExecutionHook.scala | 55 ++++++ .../linkis/engine/cs/CSTableParser.scala | 89 ++++++++++ .../engine/execute/SparkEngineJob.scala | 1 + .../executors/SparkEngineExecutor.scala | 18 +- .../engine/executors/SparkExecutor.scala | 5 +- .../executors/SparkPythonExecutor.scala | 11 +- .../factory/SparkEngineExecutorFactory.scala | 11 +- .../linkis/engine/imexport/CsvRelation.scala | 2 +- .../engine/mdq/MDQPostExecutionHook.scala | 3 +- .../engine/mdq/MDQPreExecutionHook.scala | 2 +- .../spark/common/LineBufferedStream.scala | 2 +- .../engine/spark/common/LogContainer.scala | 2 +- .../engine/spark/utils/EngineUtils.scala | 2 +- .../datasources/csv/DolphinToSpark.scala | 5 +- .../configuration/SparkConfiguration.scala | 2 - ...parkEngineManagerSpringConfiguration.scala | 1 - .../process/SparkEngineCreator.scala | 5 +- .../process/SparkSubmitProcessBuilder.scala | 7 +- .../entrance/job/SparkEntranceJob.scala | 1 - .../TiSparkSpringConfiguration.scala | 5 - .../tispark/engine/hook/TiSparkHook.scala | 7 - ujes/engine/pom.xml | 5 + .../linkis/engine/log/LogCache.java | 2 - .../linkis/engine/log/MountLogCache.java | 4 +- .../linkis/engine/log/RPCAppender.java | 8 - .../linkis/engine/log/SendAppender.java | 68 ++++--- .../linkis/engine/EngineReceiver.scala | 4 +- .../linkis/engine/EngineServer.scala | 1 - .../EngineServerSpringConfiguration.scala | 1 - .../linkis/engine/LockManager.scala | 1 - .../engine/PropertiesExecuteRequest.scala | 22 +++ .../engine/ResourceExecuteRequest.scala | 2 +- .../engine/conf/EngineConfiguration.scala | 6 + .../engine/cs/CSEnginePreExecuteHook.scala | 49 +++++ .../linkis/engine/cs/CSResourceParser.scala | 73 ++++++++ .../linkis/engine/cs/CSTableRegister.scala | 92 ++++++++++ .../linkis/engine/execute/CodeParser.scala | 10 +- .../engine/execute/EngineExecutor.scala | 20 +-- .../execute/EngineExecutorContext.scala | 59 ++++-- .../execute/EngineExecutorManager.scala | 3 +- .../linkis/engine/execute/EngineJob.scala | 22 ++- .../hook/CodeGeneratorEngineHook.scala | 1 - .../execute/hook/ReleaseEngineHook.scala | 3 +- .../execute/hook/UdfLoadEngineHook.scala | 4 +- .../extension/EnginePreExecuteHook.scala | 2 +- .../engine/lock/EngineTimedLockManager.scala | 4 - .../linkis/engine/log/LogHelper.scala | 2 +- .../execute/SparkCombinedCodeParserTest.scala | 5 - .../lock/EngineTimedLockManagerTest.scala | 1 - .../linkis/engine/log/ScalaLoggingTest.scala | 1 - .../enginemanager/AbstractEngineCreator.scala | 2 +- .../enginemanager/AbstractEngineManager.scala | 6 +- .../enginemanager/conf/EnvConfiguration.scala | 2 +- .../exception/EMRetryException.scala | 10 ++ .../hook/JarLoaderEngineHook.scala | 13 +- .../impl/EngineManagerImpl.scala | 2 +- .../EngineManagerSpringConfiguration.scala | 1 - ujes/entrance/pom.xml | 13 ++ ...EntranceExecutorManagerBeanAnnotation.java | 5 +- .../conf/EntranceSpringConfiguration.java | 12 +- .../EntranceIllegalParamException.java | 2 - .../entrance/job/EntranceExecutionJob.java | 1 - .../parser/AbstractEntranceParser.java | 1 + .../persistence/QueryPersistenceEngine.java | 5 +- .../persistence/QueryPersistenceManager.java | 30 +++- .../entrance/restful/EntranceRestfulApi.java | 44 +++-- .../linkis/entrance/EntranceServer.scala | 1 - .../entrance/EntranceWebSocketService.scala | 30 +++- .../entrance/conf/EntranceConfiguration.scala | 25 ++- .../linkis/entrance/cs/CSEntranceHelper.scala | 168 ++++++++++++++++++ .../exception/CacheNotReadyException.scala | 2 +- .../exception/JobHistoryFailedException.scala | 9 + .../execute/EntranceExecutorManager.scala | 12 +- .../linkis/entrance/execute/EntranceJob.scala | 10 ++ .../entrance/execute/EntranceReceiver.scala | 3 +- .../interceptor/EntranceInterceptor.scala | 2 +- .../exception/VarSubstitutionException.scala | 2 +- .../impl/CSEntranceInterceptor.scala | 42 +++++ .../interceptor/impl/CommentInterceptor.scala | 50 ++---- .../impl/CustomVariableUtils.scala | 134 +++++--------- .../entrance/interceptor/impl/Explain.scala | 27 ++- .../impl/PythonCodeCheckInterceptor.scala | 2 +- .../impl/SQLLimitEntranceInterceptor.scala | 1 - .../impl/ScalaCodeInterceptor.scala | 1 - .../impl/SparkCodeCheckInterceptor.scala | 4 +- .../linkis/entrance/log/CacheLogWriter.scala | 7 +- .../entrance/log/ErrorCodeManager.scala | 12 -- .../parser/CommonEntranceParser.scala | 16 +- .../persistence/EntranceResultSetEngine.scala | 2 - .../persistence/PersistenceEngine.scala | 1 - .../restful/EntranceRestfulRemote.scala | 5 +- .../scheduler/EntranceGroupFactory.scala | 3 +- .../scheduler/EntranceSchedulerContext.scala | 2 - .../entrance/utils/JobHistoryHelper.scala | 87 +++++++++ ujes/entranceclient/pom.xml | 6 + ujes/jdbc/pom.xml | 15 +- 137 files changed, 1334 insertions(+), 487 deletions(-) create mode 100644 ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/conversions/FsConvertions.scala create mode 100644 ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSSparkHelper.scala create mode 100644 ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSSparkPreExecutionHook.scala create mode 100644 ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSTableParser.scala create mode 100644 ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PropertiesExecuteRequest.scala create mode 100644 ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSEnginePreExecuteHook.scala create mode 100644 ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSResourceParser.scala create mode 100644 ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSTableRegister.scala create mode 100644 ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/exception/EMRetryException.scala create mode 100644 ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/cs/CSEntranceHelper.scala create mode 100644 ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/JobHistoryFailedException.scala create mode 100644 ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CSEntranceInterceptor.scala create mode 100644 ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/utils/JobHistoryHelper.scala diff --git a/ujes/client/src/main/scala/com/webank/wedatasphere/linkis/ujes/client/response/JobInfoResult.scala b/ujes/client/src/main/scala/com/webank/wedatasphere/linkis/ujes/client/response/JobInfoResult.scala index 5c721803cf..61c84b19df 100644 --- a/ujes/client/src/main/scala/com/webank/wedatasphere/linkis/ujes/client/response/JobInfoResult.scala +++ b/ujes/client/src/main/scala/com/webank/wedatasphere/linkis/ujes/client/response/JobInfoResult.scala @@ -23,6 +23,7 @@ package com.webank.wedatasphere.linkis.ujes.client.response import java.util import java.util.Date +import com.webank.wedatasphere.linkis.common.utils.Utils import com.webank.wedatasphere.linkis.httpclient.dws.annotation.DWSHttpMessageResult import com.webank.wedatasphere.linkis.httpclient.dws.response.DWSResult import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask @@ -48,9 +49,16 @@ class JobInfoResult extends DWSResult with UserAction with Status { val updatedTime = task.get("updatedTime").asInstanceOf[Long] task.remove("createdTime") task.remove("updatedTime") - BeanUtils.populate(requestPersistTask, task) + task.remove("engineStartTime") + Utils.tryCatch{ + BeanUtils.populate(requestPersistTask, task.asInstanceOf[util.Map[String, _]]) + }{ + case e:Exception => error("copy failed", e) + } + requestPersistTask.setStatus(task.get("status").asInstanceOf[String]) requestPersistTask.setCreatedTime(new Date(createdTime)) requestPersistTask.setUpdatedTime(new Date(updatedTime)) + requestPersistTask.setEngineStartTime(new Date(updatedTime)) } def getTask = task diff --git a/ujes/client/src/main/scala/com/webank/wedatasphere/linkis/ujes/client/response/JobProgressResult.scala b/ujes/client/src/main/scala/com/webank/wedatasphere/linkis/ujes/client/response/JobProgressResult.scala index 4b9b4d7846..b6988eb729 100644 --- a/ujes/client/src/main/scala/com/webank/wedatasphere/linkis/ujes/client/response/JobProgressResult.scala +++ b/ujes/client/src/main/scala/com/webank/wedatasphere/linkis/ujes/client/response/JobProgressResult.scala @@ -24,7 +24,6 @@ import java.util.{List, Map} import com.webank.wedatasphere.linkis.httpclient.dws.annotation.DWSHttpMessageResult import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo - import org.json4s._ import org.json4s.jackson.Serialization._ diff --git a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/HiveEngineSpringConfiguration.scala b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/HiveEngineSpringConfiguration.scala index a7e57f34bf..61455f49a4 100644 --- a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/HiveEngineSpringConfiguration.scala +++ b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/HiveEngineSpringConfiguration.scala @@ -18,7 +18,7 @@ package com.webank.wedatasphere.linkis.engine.hive import com.webank.wedatasphere.linkis.engine.execute.hook._ import com.webank.wedatasphere.linkis.engine.execute.{CodeParser, EngineHook, SQLCodeParser} -import com.webank.wedatasphere.linkis.engine.hive.hook.{HiveAddJarsEngineHook, UseDatabaseEngineHook} +import com.webank.wedatasphere.linkis.engine.hive.hook.HiveAddJarsEngineHook import org.slf4j.LoggerFactory import org.springframework.context.annotation.{Bean, Configuration} diff --git a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/common/HiveUtils.scala b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/common/HiveUtils.scala index b3aba20105..0faf90f831 100644 --- a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/common/HiveUtils.scala +++ b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/common/HiveUtils.scala @@ -19,14 +19,13 @@ package com.webank.wedatasphere.linkis.engine.hive.common import java.io.File import java.nio.file.Paths +import com.webank.wedatasphere.linkis.common.conf.{Configuration => CommonConfiguration} import com.webank.wedatasphere.linkis.engine.hive.exception.HadoopConfSetFailedException - import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.hadoop.hive.conf import org.apache.hadoop.hive.conf.HiveConf import org.apache.hadoop.hive.ql.Driver -import com.webank.wedatasphere.linkis.common.conf.{Configuration => CommonConfiguration} /** * created by cooperyang on 2018/11/22 diff --git a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/executor/HiveEngineExecutor.scala b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/executor/HiveEngineExecutor.scala index c0ace13d28..b12cc4ffc1 100644 --- a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/executor/HiveEngineExecutor.scala +++ b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/executor/HiveEngineExecutor.scala @@ -36,10 +36,10 @@ import org.apache.commons.io.IOUtils import org.apache.commons.lang.StringUtils import org.apache.hadoop.hive.conf.HiveConf import org.apache.hadoop.hive.metastore.api.{FieldSchema, Schema} +import org.apache.hadoop.hive.ql.Driver import org.apache.hadoop.hive.ql.exec.mr.HadoopJobExecHelper import org.apache.hadoop.hive.ql.processors.{CommandProcessor, CommandProcessorFactory, CommandProcessorResponse} import org.apache.hadoop.hive.ql.session.SessionState -import org.apache.hadoop.hive.ql.Driver import org.apache.hadoop.mapred.RunningJob import org.apache.hadoop.security.UserGroupInformation import org.slf4j.LoggerFactory diff --git a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/hook/UseDatabaseEngineHook.scala b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/hook/UseDatabaseEngineHook.scala index 5bade51956..728925a805 100644 --- a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/hook/UseDatabaseEngineHook.scala +++ b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/hook/UseDatabaseEngineHook.scala @@ -22,8 +22,6 @@ import com.webank.wedatasphere.linkis.server.JMap import org.apache.commons.lang.StringUtils import org.slf4j.{Logger, LoggerFactory} -import scala.collection.mutable.ArrayBuffer - /** * created by cooperyang on 2019/4/22 * Description: diff --git a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/progress/HiveProgressHelper.scala b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/progress/HiveProgressHelper.scala index 047f2c069c..cf1188b717 100644 --- a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/progress/HiveProgressHelper.scala +++ b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/progress/HiveProgressHelper.scala @@ -16,7 +16,6 @@ package com.webank.wedatasphere.linkis.engine.hive.progress -import java.util import java.util.concurrent.LinkedBlockingQueue import com.webank.wedatasphere.linkis.engine.hive.log.HiveProgress diff --git a/ujes/definedEngines/hive/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/hive/process/HiveQLProcessBuilder.scala b/ujes/definedEngines/hive/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/hive/process/HiveQLProcessBuilder.scala index 1a8a358c56..897c9dc753 100644 --- a/ujes/definedEngines/hive/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/hive/process/HiveQLProcessBuilder.scala +++ b/ujes/definedEngines/hive/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/hive/process/HiveQLProcessBuilder.scala @@ -19,15 +19,14 @@ package com.webank.wedatasphere.linkis.enginemanager.hive.process import java.nio.file.Paths import com.webank.wedatasphere.linkis.common.conf.Configuration -import com.webank.wedatasphere.linkis.enginemanager.{AbstractEngineCreator, EngineResource} -import com.webank.wedatasphere.linkis.enginemanager.conf.EnvConfiguration.{DEFAULT_JAVA_OPTS, ENGINE_CLIENT_MEMORY, JAVA_HOME, engineGCLogPath} +import com.webank.wedatasphere.linkis.enginemanager.conf.EnvConfiguration.{DEFAULT_JAVA_OPTS, JAVA_HOME, engineGCLogPath} import com.webank.wedatasphere.linkis.enginemanager.hive.conf.HiveEngineConfiguration import com.webank.wedatasphere.linkis.enginemanager.impl.UserEngineResource import com.webank.wedatasphere.linkis.enginemanager.process.JavaProcessEngineBuilder +import com.webank.wedatasphere.linkis.enginemanager.{AbstractEngineCreator, EngineResource} import com.webank.wedatasphere.linkis.protocol.engine.RequestEngine import org.apache.commons.lang.StringUtils import org.slf4j.LoggerFactory -import org.springframework.stereotype.Component import scala.collection.mutable.ArrayBuffer diff --git a/ujes/definedEngines/hive/entrance/bin/start-hive-entrance.sh b/ujes/definedEngines/hive/entrance/bin/start-hive-entrance.sh index 56cab6df00..80cc775a4a 100644 --- a/ujes/definedEngines/hive/entrance/bin/start-hive-entrance.sh +++ b/ujes/definedEngines/hive/entrance/bin/start-hive-entrance.sh @@ -26,7 +26,7 @@ if [[ -f "${SERVER_PID}" ]]; then fi fi -nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & +nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & pid=$! if [[ -z "${pid}" ]]; then echo "server $SERVER_NAME start failed!" diff --git a/ujes/definedEngines/jdbc/entrance/bin/start-jdbc-entrance.sh b/ujes/definedEngines/jdbc/entrance/bin/start-jdbc-entrance.sh index 56cab6df00..80cc775a4a 100644 --- a/ujes/definedEngines/jdbc/entrance/bin/start-jdbc-entrance.sh +++ b/ujes/definedEngines/jdbc/entrance/bin/start-jdbc-entrance.sh @@ -26,7 +26,7 @@ if [[ -f "${SERVER_PID}" ]]; then fi fi -nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & +nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & pid=$! if [[ -z "${pid}" ]]; then echo "server $SERVER_NAME start failed!" diff --git a/ujes/definedEngines/jdbc/entrance/pom.xml b/ujes/definedEngines/jdbc/entrance/pom.xml index 0f214b667c..bba9386513 100644 --- a/ujes/definedEngines/jdbc/entrance/pom.xml +++ b/ujes/definedEngines/jdbc/entrance/pom.xml @@ -77,7 +77,7 @@ org.apache.thrift libthrift - 0.9.3 + 0.9.2 pom @@ -139,4 +139,4 @@ ${project.artifactId}-${project.version} - + \ No newline at end of file diff --git a/ujes/definedEngines/jdbc/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/executer/ConnectionManager.java b/ujes/definedEngines/jdbc/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/executer/ConnectionManager.java index 793f4c54d9..c3e3772049 100644 --- a/ujes/definedEngines/jdbc/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/executer/ConnectionManager.java +++ b/ujes/definedEngines/jdbc/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/executer/ConnectionManager.java @@ -18,6 +18,8 @@ import org.apache.commons.dbcp.BasicDataSource; import org.apache.commons.dbcp.BasicDataSourceFactory; import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.sql.DataSource; import java.sql.Connection; @@ -27,8 +29,6 @@ import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class ConnectionManager { diff --git a/ujes/definedEngines/jdbc/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/JDBCSpringConfiguration.scala b/ujes/definedEngines/jdbc/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/JDBCSpringConfiguration.scala index c42ba79486..50ccfe781a 100644 --- a/ujes/definedEngines/jdbc/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/JDBCSpringConfiguration.scala +++ b/ujes/definedEngines/jdbc/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/JDBCSpringConfiguration.scala @@ -23,7 +23,6 @@ import com.webank.wedatasphere.linkis.entrance.parser.JDBCEntranceParser import com.webank.wedatasphere.linkis.scheduler.queue.GroupFactory import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.context.annotation.Configuration @Configuration diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/OutputStreamCache.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/OutputStreamCache.scala index 70c4a7a633..c07a472885 100644 --- a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/OutputStreamCache.scala +++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/OutputStreamCache.scala @@ -18,7 +18,7 @@ package com.webank.wedatasphere.linkis.engine.pipeline import java.io.OutputStream /** - * created by patinousward on 2019/12/9 + * created by cooperyang on 2019/12/9 * Description: */ object OutputStreamCache { diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipeEngineJob.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipeEngineJob.scala index ce9bdf1a08..76d51cddf8 100644 --- a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipeEngineJob.scala +++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipeEngineJob.scala @@ -18,7 +18,7 @@ package com.webank.wedatasphere.linkis.engine.pipeline import com.webank.wedatasphere.linkis.engine.execute.CommonEngineJob /** - * created by patinousward on 2019/12/9 + * created by cooperyang on 2019/12/9 * Description: */ class PipeEngineJob extends CommonEngineJob { diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipelineEngineParser.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipelineEngineParser.scala index 1d6cf258a1..fb6db27c41 100644 --- a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipelineEngineParser.scala +++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/PipelineEngineParser.scala @@ -21,7 +21,7 @@ import com.webank.wedatasphere.linkis.scheduler.queue.Job import org.springframework.stereotype.Component /** - * created by patinousward on 2019/12/9 + * created by cooperyang on 2019/12/9 * Description: */ @Component("engineParser") diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/constant/PipeLineConstant.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/constant/PipeLineConstant.scala index f06bfe935d..80cd7036a4 100644 --- a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/constant/PipeLineConstant.scala +++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/constant/PipeLineConstant.scala @@ -17,7 +17,7 @@ package com.webank.wedatasphere.linkis.engine.pipeline.constant /** - * Created by patinousward on 2018/11/16. + * Created by johnnwang on 2018/11/16. */ object PipeLineConstant { val DEFAULTC_HARSET = "utf-8" diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/conversions/FsConvertions.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/conversions/FsConvertions.scala new file mode 100644 index 0000000000..9858a60252 --- /dev/null +++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/conversions/FsConvertions.scala @@ -0,0 +1,37 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.engine.pipeline.conversions + +import java.io.Closeable + +import com.webank.wedatasphere.linkis.common.io.Fs + +/** + * Created by johnnwang on 2020/2/4. + */ +object FsConvertions { + + implicit def fsToFsClosable(fs: Fs): Closeable = { + new FsClosable(fs) + } +} + +class FsClosable(fs: Fs) extends Closeable { + override def close(): Unit = { + fs.close() + } +} diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/CSVExecutor.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/CSVExecutor.scala index 4f9c594005..18aaad5684 100644 --- a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/CSVExecutor.scala +++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/CSVExecutor.scala @@ -22,7 +22,7 @@ import com.webank.wedatasphere.linkis.common.io.FsPath import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext import com.webank.wedatasphere.linkis.engine.pipeline.OutputStreamCache import com.webank.wedatasphere.linkis.engine.pipeline.constant.PipeLineConstant._ -import com.webank.wedatasphere.linkis.engine.pipeline.conversions.FsConversions._ +import com.webank.wedatasphere.linkis.engine.pipeline.conversions.FsConvertions._ import com.webank.wedatasphere.linkis.engine.pipeline.exception.PipeLineErrorException import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteResponse import com.webank.wedatasphere.linkis.server._ diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/CopyExecutor.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/CopyExecutor.scala index c6e321af56..1a5afdbc1a 100644 --- a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/CopyExecutor.scala +++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/CopyExecutor.scala @@ -20,7 +20,7 @@ import com.webank.wedatasphere.linkis.common.io.FsPath import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext import com.webank.wedatasphere.linkis.engine.pipeline.OutputStreamCache import com.webank.wedatasphere.linkis.engine.pipeline.constant.PipeLineConstant._ -import com.webank.wedatasphere.linkis.engine.pipeline.conversions.FsConversions._ +import com.webank.wedatasphere.linkis.engine.pipeline.conversions.FsConvertions._ import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteResponse import com.webank.wedatasphere.linkis.server._ import com.webank.wedatasphere.linkis.storage.FSFactory diff --git a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/ExcelExecutor.scala b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/ExcelExecutor.scala index 5adaca3114..dde2f103e9 100644 --- a/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/ExcelExecutor.scala +++ b/ujes/definedEngines/pipeline/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/pipeline/executor/ExcelExecutor.scala @@ -22,7 +22,7 @@ import com.webank.wedatasphere.linkis.common.io.FsPath import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext import com.webank.wedatasphere.linkis.engine.pipeline.OutputStreamCache import com.webank.wedatasphere.linkis.engine.pipeline.constant.PipeLineConstant._ -import com.webank.wedatasphere.linkis.engine.pipeline.conversions.FsConversions._ +import com.webank.wedatasphere.linkis.engine.pipeline.conversions.FsConvertions._ import com.webank.wedatasphere.linkis.engine.pipeline.exception.PipeLineErrorException import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteResponse import com.webank.wedatasphere.linkis.storage.FSFactory diff --git a/ujes/definedEngines/pipeline/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/pipeline/PipeLineDefaultEngineCreator.scala b/ujes/definedEngines/pipeline/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/pipeline/PipeLineDefaultEngineCreator.scala index cb510de065..cce8def86b 100644 --- a/ujes/definedEngines/pipeline/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/pipeline/PipeLineDefaultEngineCreator.scala +++ b/ujes/definedEngines/pipeline/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/pipeline/PipeLineDefaultEngineCreator.scala @@ -18,7 +18,6 @@ package com.webank.wedatasphere.linkis.enginemanager.pipeline import com.webank.wedatasphere.linkis.enginemanager.AbstractEngineCreator import com.webank.wedatasphere.linkis.enginemanager.process.ProcessEngineBuilder -import org.springframework.stereotype.Component /** * Created by johnnwang on 2018/11/15. diff --git a/ujes/definedEngines/pipeline/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/pipeline/PipeLineProcessEngineBuilder.scala b/ujes/definedEngines/pipeline/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/pipeline/PipeLineProcessEngineBuilder.scala index dea14b136a..8381456d79 100644 --- a/ujes/definedEngines/pipeline/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/pipeline/PipeLineProcessEngineBuilder.scala +++ b/ujes/definedEngines/pipeline/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/pipeline/PipeLineProcessEngineBuilder.scala @@ -16,7 +16,6 @@ package com.webank.wedatasphere.linkis.enginemanager.pipeline -import com.webank.wedatasphere.linkis.common.utils.ClassUtils import com.webank.wedatasphere.linkis.enginemanager.conf.EngineManagerConfiguration import com.webank.wedatasphere.linkis.enginemanager.pipeline.conf.PipeLineConfiguration import com.webank.wedatasphere.linkis.enginemanager.process.JavaProcessEngineBuilder diff --git a/ujes/definedEngines/pipeline/entrance/bin/start-pipeline-entrance.sh b/ujes/definedEngines/pipeline/entrance/bin/start-pipeline-entrance.sh index 56cab6df00..80cc775a4a 100644 --- a/ujes/definedEngines/pipeline/entrance/bin/start-pipeline-entrance.sh +++ b/ujes/definedEngines/pipeline/entrance/bin/start-pipeline-entrance.sh @@ -26,7 +26,7 @@ if [[ -f "${SERVER_PID}" ]]; then fi fi -nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & +nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & pid=$! if [[ -z "${pid}" ]]; then echo "server $SERVER_NAME start failed!" diff --git a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/ExecuteResponse.scala b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/ExecuteResponse.scala index 4bb9bce44e..8768216985 100644 --- a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/ExecuteResponse.scala +++ b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/ExecuteResponse.scala @@ -16,7 +16,6 @@ package com.webank.wedatasphere.linkis.engine - import com.webank.wedatasphere.linkis.engine.exception.QueryFailedException import org.json4s.JValue import org.json4s.JsonAST.JString diff --git a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/HDFSResultSetOutputStream.scala b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/HDFSResultSetOutputStream.scala index ed8b1798cf..7712d3e18b 100644 --- a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/HDFSResultSetOutputStream.scala +++ b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/HDFSResultSetOutputStream.scala @@ -15,6 +15,7 @@ */ package com.webank.wedatasphere.linkis.engine + /** * Created by allenlliu on 2019/4/8. */ diff --git a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PythonEngineSpringConfiguration.scala b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PythonEngineSpringConfiguration.scala index 9b919b0f3f..49942708db 100644 --- a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PythonEngineSpringConfiguration.scala +++ b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PythonEngineSpringConfiguration.scala @@ -16,8 +16,8 @@ package com.webank.wedatasphere.linkis.engine -import com.webank.wedatasphere.linkis.engine.execute.{CodeParser, EngineHook, PythonCodeParser, SQLCodeParser} import com.webank.wedatasphere.linkis.engine.execute.hook.{MaxExecuteNumEngineHook, ReleaseEngineHook} +import com.webank.wedatasphere.linkis.engine.execute.{CodeParser, EngineHook, PythonCodeParser} import org.slf4j.LoggerFactory import org.springframework.context.annotation.Bean import org.springframework.stereotype.Component diff --git a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PythonSession.scala b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PythonSession.scala index 86c7763d7d..af707d6f0a 100644 --- a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PythonSession.scala +++ b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PythonSession.scala @@ -23,30 +23,27 @@ package com.webank.wedatasphere.linkis.engine import java.io.{File, FileFilter, FileOutputStream, InputStream} import java.net.ServerSocket import java.nio.file.Files - -import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} -import com.webank.wedatasphere.linkis.engine.exception.{ExecuteException, PythonExecuteError, QueryFailedException} -import com.webank.wedatasphere.linkis.engine.util.PythonConfiguration._ -import com.webank.wedatasphere.linkis.engine.util._ -import org.apache.commons.exec.CommandLine -import org.apache.commons.lang.StringUtils -import py4j.GatewayServer import java.util.{List => JList} +import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} import com.webank.wedatasphere.linkis.engine.conf.PythonEngineConfiguration - -import scala.collection.JavaConverters._ +import com.webank.wedatasphere.linkis.engine.exception.{ExecuteException, PythonExecuteError} import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext import com.webank.wedatasphere.linkis.engine.rs.RsOutputStream -import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord} +import com.webank.wedatasphere.linkis.engine.util.PythonConfiguration._ +import com.webank.wedatasphere.linkis.engine.util._ import com.webank.wedatasphere.linkis.storage.domain._ -import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetWriter} import com.webank.wedatasphere.linkis.storage.resultset.table.{TableMetaData, TableRecord} +import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetWriter} +import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord} +import org.apache.commons.exec.CommandLine import org.apache.commons.io.IOUtils +import org.apache.commons.lang.StringUtils import org.scalactic.Pass +import py4j.GatewayServer import scala.collection.JavaConversions._ -import scala.collection.JavaConverters +import scala.collection.JavaConverters._ import scala.collection.mutable.{ArrayBuffer, StringBuilder} import scala.concurrent.duration.Duration import scala.concurrent.{Await, Future, Promise} diff --git a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/PythonEngineExecutor.scala b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/PythonEngineExecutor.scala index ead201416a..895d20f9de 100644 --- a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/PythonEngineExecutor.scala +++ b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/PythonEngineExecutor.scala @@ -16,15 +16,13 @@ package com.webank.wedatasphere.linkis.engine.executors -import java.util.UUID - -import com.webank.wedatasphere.linkis.common.utils.{ByteTimeUtils, Logging} +import com.webank.wedatasphere.linkis.common.utils.Logging import com.webank.wedatasphere.linkis.engine.PythonSession import com.webank.wedatasphere.linkis.engine.exception.EngineException import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineExecutorContext} import com.webank.wedatasphere.linkis.engine.rs.RsOutputStream import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo -import com.webank.wedatasphere.linkis.resourcemanager.{LoadInstanceResource, LoadResource, Resource} +import com.webank.wedatasphere.linkis.resourcemanager.{LoadInstanceResource, Resource} import com.webank.wedatasphere.linkis.rpc.Sender import com.webank.wedatasphere.linkis.scheduler.executer._ import org.apache.commons.io.IOUtils diff --git a/ujes/definedEngines/python/enginemanager/pom.xml b/ujes/definedEngines/python/enginemanager/pom.xml index b81d0f3cad..cac5209654 100644 --- a/ujes/definedEngines/python/enginemanager/pom.xml +++ b/ujes/definedEngines/python/enginemanager/pom.xml @@ -32,6 +32,9 @@ + + 0.9.4 + @@ -125,4 +128,4 @@ - + \ No newline at end of file diff --git a/ujes/definedEngines/python/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/PythonEngineSpringConfiguration.scala b/ujes/definedEngines/python/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/PythonEngineSpringConfiguration.scala index a8904db157..ea57d29321 100644 --- a/ujes/definedEngines/python/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/PythonEngineSpringConfiguration.scala +++ b/ujes/definedEngines/python/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/PythonEngineSpringConfiguration.scala @@ -16,9 +16,7 @@ package com.webank.wedatasphere.linkis.enginemanager.process -import com.webank.wedatasphere.linkis.enginemanager.EngineHook import com.webank.wedatasphere.linkis.enginemanager.conf.EnvConfiguration -import com.webank.wedatasphere.linkis.enginemanager.hook.{ConsoleConfigurationEngineHook, JarUdfEngineHook, PyFunctionEngineHook, PyUdfEngineHook} import com.webank.wedatasphere.linkis.resourcemanager.domain.ModuleInfo import com.webank.wedatasphere.linkis.resourcemanager.{LoadInstanceResource, ResourceRequestPolicy} import com.webank.wedatasphere.linkis.rpc.Sender diff --git a/ujes/definedEngines/python/entrance/bin/start-python-entrance.sh b/ujes/definedEngines/python/entrance/bin/start-python-entrance.sh index 56cab6df00..80cc775a4a 100644 --- a/ujes/definedEngines/python/entrance/bin/start-python-entrance.sh +++ b/ujes/definedEngines/python/entrance/bin/start-python-entrance.sh @@ -26,7 +26,7 @@ if [[ -f "${SERVER_PID}" ]]; then fi fi -nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & +nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & pid=$! if [[ -z "${pid}" ]]; then echo "server $SERVER_NAME start failed!" diff --git a/ujes/definedEngines/shell/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/shell/executor/ShellEngineExecutor.scala b/ujes/definedEngines/shell/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/shell/executor/ShellEngineExecutor.scala index 3847a908bb..322c81bd35 100644 --- a/ujes/definedEngines/shell/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/shell/executor/ShellEngineExecutor.scala +++ b/ujes/definedEngines/shell/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/shell/executor/ShellEngineExecutor.scala @@ -3,15 +3,14 @@ package com.webank.wedatasphere.linkis.engine.shell.executor import java.io.{BufferedReader, InputStreamReader} import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineExecutorContext} +import com.webank.wedatasphere.linkis.engine.shell.conf.ShellEngineConfiguration +import com.webank.wedatasphere.linkis.engine.shell.exception.ShellCodeErrorException import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo import com.webank.wedatasphere.linkis.resourcemanager.{LoadInstanceResource, Resource} import com.webank.wedatasphere.linkis.rpc.Sender import com.webank.wedatasphere.linkis.scheduler.executer._ -import com.webank.wedatasphere.linkis.engine.shell.conf.ShellEngineConfiguration -import com.webank.wedatasphere.linkis.engine.shell.exception.ShellCodeErrorException -import org.apache.commons.lang.StringUtils -import com.webank.wedatasphere.linkis.engine.execute.EngineExecutor import org.apache.commons.io.IOUtils +import org.apache.commons.lang.StringUtils /** * created by cooperyang on 2019/5/14 diff --git a/ujes/definedEngines/shell/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/shell/executor/ShellEngineExecutorFactory.scala b/ujes/definedEngines/shell/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/shell/executor/ShellEngineExecutorFactory.scala index ef929032cc..81dd094f6d 100644 --- a/ujes/definedEngines/shell/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/shell/executor/ShellEngineExecutorFactory.scala +++ b/ujes/definedEngines/shell/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/shell/executor/ShellEngineExecutorFactory.scala @@ -2,8 +2,8 @@ package com.webank.wedatasphere.linkis.engine.shell.executor import com.webank.wedatasphere.linkis.common.utils.Logging import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineExecutorFactory} -import com.webank.wedatasphere.linkis.server.JMap import com.webank.wedatasphere.linkis.engine.shell.exception.NoCorrectUserException +import com.webank.wedatasphere.linkis.server.JMap import org.apache.commons.lang.StringUtils import org.springframework.stereotype.Component diff --git a/ujes/definedEngines/shell/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/shell/enginemanager/conf/ShellEngineManagerConfiguration.scala b/ujes/definedEngines/shell/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/shell/enginemanager/conf/ShellEngineManagerConfiguration.scala index 4f95bb7c8c..de397dd291 100644 --- a/ujes/definedEngines/shell/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/shell/enginemanager/conf/ShellEngineManagerConfiguration.scala +++ b/ujes/definedEngines/shell/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/shell/enginemanager/conf/ShellEngineManagerConfiguration.scala @@ -3,11 +3,10 @@ package com.webank.wedatasphere.linkis.shell.enginemanager.conf import com.webank.wedatasphere.linkis.common.utils.Logging import com.webank.wedatasphere.linkis.enginemanager.EngineHook import com.webank.wedatasphere.linkis.enginemanager.conf.EnvConfiguration -import com.webank.wedatasphere.linkis.enginemanager.hook.{ConsoleConfigurationEngineHook, JarLoaderEngineHook} -import com.webank.wedatasphere.linkis.resourcemanager.{LoadInstanceResource, ResourceRequestPolicy} +import com.webank.wedatasphere.linkis.enginemanager.hook.ConsoleConfigurationEngineHook import com.webank.wedatasphere.linkis.resourcemanager.domain.ModuleInfo +import com.webank.wedatasphere.linkis.resourcemanager.{LoadInstanceResource, ResourceRequestPolicy} import com.webank.wedatasphere.linkis.rpc.Sender -import org.slf4j.{Logger, LoggerFactory} import org.springframework.context.annotation.{Bean, Configuration} /** diff --git a/ujes/definedEngines/shell/entrance/bin/start-shell-entrance.sh b/ujes/definedEngines/shell/entrance/bin/start-shell-entrance.sh index 56cab6df00..80cc775a4a 100644 --- a/ujes/definedEngines/shell/entrance/bin/start-shell-entrance.sh +++ b/ujes/definedEngines/shell/entrance/bin/start-shell-entrance.sh @@ -26,7 +26,7 @@ if [[ -f "${SERVER_PID}" ]]; then fi fi -nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & +nohup java $SERVER_JAVA_OPTS -cp ../module/lib/*:$HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & pid=$! if [[ -z "${pid}" ]]; then echo "server $SERVER_NAME start failed!" diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/Interpreter/ProcessInterpreter.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/Interpreter/ProcessInterpreter.scala index fc8e763a3d..e015878772 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/Interpreter/ProcessInterpreter.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/Interpreter/ProcessInterpreter.scala @@ -20,14 +20,13 @@ import java.io.{BufferedReader, InputStreamReader, PrintWriter} import java.util.concurrent.TimeUnit import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} -import com.webank.wedatasphere.linkis.engine.spark.common.{LineBufferedStream, Starting, State} +import com.webank.wedatasphere.linkis.engine.spark.common.{LineBufferedStream, Starting, State, _} import com.webank.wedatasphere.linkis.scheduler.executer.{ErrorExecuteResponse, ExecuteResponse, SuccessExecuteResponse} import org.apache.commons.io.IOUtils import org.json4s._ -import scala.concurrent.{Await, ExecutionContext, Future} import scala.concurrent.duration.Duration -import com.webank.wedatasphere.linkis.engine.spark.common._ +import scala.concurrent.{Await, ExecutionContext, Future} /** * Created by allenlliu on 2018/11/19. diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/Interpreter/PythonInterpreter.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/Interpreter/PythonInterpreter.scala index 17bf4907ee..638a89abb4 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/Interpreter/PythonInterpreter.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/Interpreter/PythonInterpreter.scala @@ -15,24 +15,25 @@ */ package com.webank.wedatasphere.linkis.engine.Interpreter -import org.apache.spark.{SparkContext, SparkException} -import org.json4s.jackson.JsonMethods._ -import org.json4s.jackson.Serialization.write -import org.json4s.{DefaultFormats, JValue} -import py4j.GatewayServer + import java.io._ import java.nio.file.Files import com.webank.wedatasphere.linkis.common.conf.CommonVars import com.webank.wedatasphere.linkis.common.io.FsPath import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} -import com.webank.wedatasphere.linkis.engine.configuration.{SparkConfiguration} +import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration import com.webank.wedatasphere.linkis.engine.spark.common.LineBufferedStream import com.webank.wedatasphere.linkis.engine.spark.utils.EngineUtils import com.webank.wedatasphere.linkis.storage.FSFactory import org.apache.commons.io.IOUtils import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.{SparkContext, SparkException} +import org.json4s.jackson.JsonMethods._ +import org.json4s.jackson.Serialization.write +import org.json4s.{DefaultFormats, JValue} +import py4j.GatewayServer import scala.collection.JavaConversions._ import scala.collection.mutable.ArrayBuffer diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkConfiguration.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkConfiguration.scala index c6d4936f4d..7326e9998e 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkConfiguration.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkConfiguration.scala @@ -37,4 +37,7 @@ object SparkConfiguration { val DOLPHIN_LIMIT_LEN = CommonVars("wds.linkis.dolphin.limit.len",5000) val MDQ_APPLICATION_NAME = CommonVars("wds.linkis.mdq.application.name", "cloud-datasource") val SHOW_DF_MAX_RES = CommonVars("wds.linkis.show.df.max.res",Int.MaxValue) + + + val SPARK_PRE_EXECUTION_HOOKS = CommonVars("wds.linkis.spark.pre.execution.hooks", "com.webank.wedatasphere.linkis.engine.cs.CSSparkPreExecutionHook") } diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkEngineServerSpringConfiguration.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkEngineServerSpringConfiguration.scala index 06786dd708..92ed884846 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkEngineServerSpringConfiguration.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkEngineServerSpringConfiguration.scala @@ -1,37 +1,36 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.webank.wedatasphere.linkis.engine.configuration - -import com.webank.wedatasphere.linkis.engine.condition.EngineHooksCondition -import com.webank.wedatasphere.linkis.engine.execute.hook._ -import com.webank.wedatasphere.linkis.engine.execute.{CodeParser, EngineHook, SparkCombinedCodeParser} -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean -import org.springframework.context.annotation.{Bean, Conditional, Configuration} - -/** - * Created by allenlliu on 2018/12/3. - */ -@Configuration -class SparkEngineServerSpringConfiguration { - @Bean(Array("codeParser")) - def createCodeParser(): CodeParser = new SparkCombinedCodeParser() - - - @Bean(Array("engineHooks")) - @Conditional(Array(classOf[EngineHooksCondition])) - def createEngineHooks(): Array[EngineHook] = Array(new ReleaseEngineHook, new MaxExecuteNumEngineHook, new JarUdfEngineHook, new PyUdfEngineHook, new ScalaUdfEngineHook, new PyFunctionEngineHook, new ScalaFunctionEngineHook) -} +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.engine.configuration + +import com.webank.wedatasphere.linkis.engine.condition.EngineHooksCondition +import com.webank.wedatasphere.linkis.engine.execute.hook._ +import com.webank.wedatasphere.linkis.engine.execute.{CodeParser, EngineHook, SparkCombinedCodeParser} +import org.springframework.context.annotation.{Bean, Conditional, Configuration} + +/** + * Created by allenlliu on 2018/12/3. + */ +@Configuration +class SparkEngineServerSpringConfiguration { + @Bean(Array("codeParser")) + def createCodeParser(): CodeParser = new SparkCombinedCodeParser() + + + @Bean(Array("engineHooks")) + @Conditional(Array(classOf[EngineHooksCondition])) + def createEngineHooks(): Array[EngineHook] = Array(new ReleaseEngineHook, new MaxExecuteNumEngineHook, new JarUdfEngineHook, new PyUdfEngineHook, new ScalaUdfEngineHook, new PyFunctionEngineHook, new ScalaFunctionEngineHook) +} diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSSparkHelper.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSSparkHelper.scala new file mode 100644 index 0000000000..287e1c9c80 --- /dev/null +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSSparkHelper.scala @@ -0,0 +1,34 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.engine.cs + +import com.webank.wedatasphere.linkis.cs.client.utils.ContextServiceUtils +import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils +import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext +import org.apache.spark.SparkContext + +/** + * @author peacewong + * @date 2020/4/19 16:39 + */ +object CSSparkHelper { + + def setContextIDInfoToSparkConf(engineExecutorContext: EngineExecutorContext, sparkContext: SparkContext): Unit = { + val contextIDValueStr = ContextServiceUtils.getContextIDStrByMap(engineExecutorContext.getProperties) + val nodeNameStr = ContextServiceUtils.getNodeNameStrByMap(engineExecutorContext.getProperties) + sparkContext.setLocalProperty(CSCommonUtils.CONTEXT_ID_STR, contextIDValueStr) + sparkContext.setLocalProperty(CSCommonUtils.NODE_NAME_STR, nodeNameStr) + } + +} diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSSparkPreExecutionHook.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSSparkPreExecutionHook.scala new file mode 100644 index 0000000000..b57584359f --- /dev/null +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSSparkPreExecutionHook.scala @@ -0,0 +1,55 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.engine.cs + +import com.webank.wedatasphere.linkis.common.utils.Logging +import com.webank.wedatasphere.linkis.cs.client.utils.ContextServiceUtils +import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext +import com.webank.wedatasphere.linkis.engine.extension.SparkPreExecutionHook +import javax.annotation.PostConstruct +import org.springframework.stereotype.Component + +/** + * @author peacewong + * @date 2020/3/6 0:06 + */ +@Component +class CSSparkPreExecutionHook extends SparkPreExecutionHook with Logging{ + + @PostConstruct + def init(): Unit ={ + SparkPreExecutionHook.register(this) + } + + private val csTableParser = new CSTableParser + + override def hookName: String = "CSSparkPreExecutionHook" + + override def callPreExecutionHook(engineExecutorContext: EngineExecutorContext, code: String): String = { + + var parsedCode = code + val contextIDValueStr = ContextServiceUtils.getContextIDStrByMap(engineExecutorContext.getProperties) + val nodeNameStr = ContextServiceUtils.getNodeNameStrByMap(engineExecutorContext.getProperties) + info(s"Start to call CSSparkPreExecutionHook,contextID is $contextIDValueStr, nodeNameStr is $nodeNameStr") + parsedCode = try { + csTableParser.parse(engineExecutorContext, parsedCode, contextIDValueStr, nodeNameStr) + } catch { + case t: Throwable => + info("Failed to parser cs table", t) + parsedCode + } + info(s"Finished to call CSSparkPreExecutionHook,contextID is $contextIDValueStr, nodeNameStr is $nodeNameStr") + parsedCode + } +} diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSTableParser.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSTableParser.scala new file mode 100644 index 0000000000..00580ef98f --- /dev/null +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSTableParser.scala @@ -0,0 +1,89 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.engine.cs + +import java.util.regex.Pattern + +import com.webank.wedatasphere.linkis.common.utils.Logging +import com.webank.wedatasphere.linkis.cs.client.service.CSTableService +import com.webank.wedatasphere.linkis.cs.common.entity.metadata.CSTable +import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils +import com.webank.wedatasphere.linkis.engine.exception.ExecuteError +import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext +import org.apache.commons.lang.StringUtils +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.execution.datasources.csv.DolphinToSpark + +import scala.collection.mutable.ArrayBuffer + +/** + * @author peacewong + * @date 2020/3/6 9:57 + */ +class CSTableParser extends Logging { + + private val pb: Pattern = Pattern.compile(CSCommonUtils.CS_TMP_TABLE_PREFIX + "[^\\s\";'()]+[$\\s]{0,1}", Pattern.CASE_INSENSITIVE) + + private val DB = "default." + + + private def getCSTempTable(code: String): Array[String] = { + val bmlResourceNames = new ArrayBuffer[String]() + val mb = pb.matcher(code) + while (mb.find) bmlResourceNames.append(mb.group.trim) + bmlResourceNames.toArray + } + + /** + * 1. code parse cs_tamp + * 2. getCSTable + * 3. registerTable：暂时用dopphin，后续修改为拼接sql语句 + * + * @param engineExecutorContext + * @param code + * @param contextIDValueStr + * @param nodeNameStr + * @return + */ + def parse(engineExecutorContext: EngineExecutorContext, code: String, contextIDValueStr: String, nodeNameStr: String): String = { + val csTempTables = getCSTempTable(code) + val parsedTables = new ArrayBuffer[String]() + csTempTables.foreach{ csTempTable => + val table = getCSTable(csTempTable, contextIDValueStr, nodeNameStr) + if (null == table){ + throw new ExecuteError(40007,s"The csTable that name is $csTempTable not found in cs") + } + registerTempTable(table) + parsedTables.append(csTempTable) + } + StringUtils.replaceEach(code, csTempTables,parsedTables.toArray) + } + + /** + * TODO peaceWong From cs to get csTable + * Exact Match + * @param csTempTable + * @return + */ + def getCSTable(csTempTable:String, contextIDValueStr: String, nodeNameStr: String):CSTable = { + CSTableService.getInstance().getUpstreamSuitableTable(contextIDValueStr, nodeNameStr, csTempTable) + } + + def registerTempTable(csTable: CSTable):Unit = { + val spark = SparkSession.builder().enableHiveSupport().getOrCreate() + info(s"Start to create tempView to sparkSession viewName(${csTable.getName}) location(${csTable.getLocation})") + DolphinToSpark.createTempView(spark, csTable.getName, csTable.getLocation, true) + info(s"Finished to create tempView to sparkSession viewName(${csTable.getName}) location(${csTable.getLocation})") + } +} diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/SparkEngineJob.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/SparkEngineJob.scala index 37c69d0202..60c4f6f236 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/SparkEngineJob.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/SparkEngineJob.scala @@ -15,6 +15,7 @@ */ package com.webank.wedatasphere.linkis.engine.execute + import com.webank.wedatasphere.linkis.protocol.engine.RequestTask import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteRequest, JobExecuteRequest, RunTypeExecuteRequest} diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkEngineExecutor.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkEngineExecutor.scala index afe3dfd5dc..a1c11ccfb5 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkEngineExecutor.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkEngineExecutor.scala @@ -22,8 +22,9 @@ import java.util.concurrent.atomic.AtomicLong import com.webank.wedatasphere.linkis.common.utils.{ByteTimeUtils, Logging, Utils} import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration +import com.webank.wedatasphere.linkis.engine.cs.CSSparkHelper import com.webank.wedatasphere.linkis.engine.exception.{NoSupportEngineException, SparkEngineException} -import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineExecutorContext} +import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineExecutorContext, SparkCombinedCodeParser} import com.webank.wedatasphere.linkis.engine.extension.{SparkPostExecutionHook, SparkPreExecutionHook} import com.webank.wedatasphere.linkis.engine.spark.common._ import com.webank.wedatasphere.linkis.engine.spark.utils.{EngineUtils, JobProgressUtil} @@ -64,6 +65,7 @@ class SparkEngineExecutor(val sc: SparkContext, id: Long, outputPrintLimit: Int, } } }, 1000, 3000, TimeUnit.MILLISECONDS) + setCodeParser(new SparkCombinedCodeParser) super.init() } @@ -225,15 +227,26 @@ object SQLSession extends Logging { // get field names //logger.info("SCHEMA BEGIN") import java.util + + import scala.collection.JavaConversions._ val colSet = new util.HashSet[String]() val schema = dataFrame.schema var columnsSet:StructType = null schema foreach (s => colSet.add(s.name)) if (colSet.size() < schema.size){ val arr:ArrayBuffer[StructField] = new ArrayBuffer[StructField]() + val tmpSet = new util.HashSet[StructField]() dataFrame.queryExecution.analyzed.output foreach { attri => val tempAttri = StructField(attri.qualifiedName, attri.dataType, attri.nullable, attri.metadata) - arr += tempAttri + tmpSet += tempAttri + } + if (tmpSet.size() < schema.size){ + dataFrame.queryExecution.analyzed.output foreach { + attri => val tempAttri = StructField(attri.toString(), attri.dataType, attri.nullable, attri.metadata) + arr += tempAttri + } + }else{ + tmpSet.foreach(arr += _) } columnsSet = StructType(arr.toArray) }else{ @@ -242,6 +255,7 @@ object SQLSession extends Logging { //val columnsSet = dataFrame.schema val columns = columnsSet.map(c => Column(c.name, DataType.toDataType(c.dataType.typeName.toLowerCase), c.getComment().orNull)).toArray[Column] + columns.foreach(c => info(s"c is ${c.columnName}, comment is ${c.comment}")) if (columns == null || columns.isEmpty) return val metaData = new TableMetaData(columns) val writer = if (StringUtils.isNotBlank(alias)) diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkExecutor.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkExecutor.scala index 1a9c565dc9..97832b645e 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkExecutor.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkExecutor.scala @@ -19,12 +19,9 @@ package com.webank.wedatasphere.linkis.engine.executors import java.io.IOException import com.webank.wedatasphere.linkis.common.utils.Logging -import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineExecutorContext} +import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext import com.webank.wedatasphere.linkis.engine.spark.common.Kind -import com.webank.wedatasphere.linkis.rpc.Sender import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteResponse -import org.apache.spark.SparkContext -import org.apache.spark.sql.SQLContext /** * Created by allenlliu on 2019/4/8. diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkPythonExecutor.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkPythonExecutor.scala index 65c5b43c8a..66fd9292f0 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkPythonExecutor.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkPythonExecutor.scala @@ -23,29 +23,28 @@ import java.util.concurrent.atomic.AtomicLong import com.webank.wedatasphere.linkis.common.conf.CommonVars import com.webank.wedatasphere.linkis.common.utils.Utils import com.webank.wedatasphere.linkis.engine.Interpreter.PythonInterpreter._ -import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration._ import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration +import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration._ import com.webank.wedatasphere.linkis.engine.exception.ExecuteError import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext import com.webank.wedatasphere.linkis.engine.imexport.CsvRelation import com.webank.wedatasphere.linkis.engine.rs.RsOutputStream import com.webank.wedatasphere.linkis.engine.spark.common.PySpark import com.webank.wedatasphere.linkis.engine.spark.utils.EngineUtils -import com.webank.wedatasphere.linkis.scheduler.executer.{ErrorExecuteResponse, ExecuteResponse, SuccessExecuteResponse} +import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteResponse, SuccessExecuteResponse} import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetWriter} -import org.apache.commons.io.IOUtils import org.apache.commons.exec.CommandLine +import org.apache.commons.io.IOUtils import org.apache.commons.lang.StringUtils +import org.apache.spark.SparkContext import org.apache.spark.api.java.JavaSparkContext -import org.apache.spark.rdd.RDD import org.apache.spark.sql.{DataFrame, SQLContext, SparkSession} -import org.apache.spark.{SparkContext, SparkException} import py4j.GatewayServer import scala.collection.JavaConversions._ import scala.collection.mutable.StringBuilder -import scala.concurrent.duration.Duration import scala.concurrent._ +import scala.concurrent.duration.Duration /** * Created by allenlliu on 2018/11/19. diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/factory/SparkEngineExecutorFactory.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/factory/SparkEngineExecutorFactory.scala index f744236451..02a89bd44f 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/factory/SparkEngineExecutorFactory.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/factory/SparkEngineExecutorFactory.scala @@ -19,24 +19,19 @@ package com.webank.wedatasphere.linkis.engine.factory import java.io.File import java.lang.reflect.Constructor +import com.webank.wedatasphere.linkis.common.conf.{CommonVars, TimeType} import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} +import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration._ import com.webank.wedatasphere.linkis.engine.exception.SparkSessionNullException import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorFactory import com.webank.wedatasphere.linkis.engine.executors.{SparkEngineExecutor, SparkPythonExecutor, SparkScalaExecutor, SparkSqlExecutor} -import com.webank.wedatasphere.linkis.engine.spark.utils.EngineUtils import com.webank.wedatasphere.linkis.server.JMap import org.apache.commons.lang.StringUtils import org.apache.spark.sql.{SQLContext, SparkSession} +import org.apache.spark.util.SparkUtils import org.apache.spark.{SparkConf, SparkContext} import org.springframework.stereotype.Component -import com.google.common.base.Joiner -import com.webank.wedatasphere.linkis.common.conf.{CommonVars, TimeType} -import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration -import org.apache.spark.util.SparkUtils - -import scala.collection.mutable.ArrayBuffer -import scala.collection.JavaConversions._ /** * Created by allenlliu on 2019/4/8. */ diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/imexport/CsvRelation.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/imexport/CsvRelation.scala index 31046e8690..cb9e4eaa25 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/imexport/CsvRelation.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/imexport/CsvRelation.scala @@ -22,7 +22,7 @@ import java.sql.{Date, Timestamp} import java.text.SimpleDateFormat import java.util.Locale -import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} +import com.webank.wedatasphere.linkis.common.utils.Logging import com.webank.wedatasphere.linkis.engine.imexport.util.ImExportUtils import org.apache.hadoop.fs.Path import org.apache.hadoop.io.{IOUtils, LongWritable, Text} diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/mdq/MDQPostExecutionHook.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/mdq/MDQPostExecutionHook.scala index bace449970..d04574f345 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/mdq/MDQPostExecutionHook.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/mdq/MDQPostExecutionHook.scala @@ -15,8 +15,6 @@ */ package com.webank.wedatasphere.linkis.engine.mdq -import javax.annotation.PostConstruct - import com.webank.wedatasphere.linkis.common.utils.Logging import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext @@ -26,6 +24,7 @@ import com.webank.wedatasphere.linkis.protocol.mdq.{DDLCompleteResponse, DDLExec import com.webank.wedatasphere.linkis.rpc.Sender import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteResponse, SuccessExecuteResponse} import com.webank.wedatasphere.linkis.storage.utils.StorageUtils +import javax.annotation.PostConstruct import org.apache.commons.lang.StringUtils import org.springframework.stereotype.Component diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/mdq/MDQPreExecutionHook.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/mdq/MDQPreExecutionHook.scala index e5504fc5fa..de552bc0fe 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/mdq/MDQPreExecutionHook.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/mdq/MDQPreExecutionHook.scala @@ -16,7 +16,6 @@ package com.webank.wedatasphere.linkis.engine.mdq import java.util -import javax.annotation.PostConstruct import com.webank.wedatasphere.linkis.common.utils.Logging import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration @@ -27,6 +26,7 @@ import com.webank.wedatasphere.linkis.engine.spark.common.SparkKind import com.webank.wedatasphere.linkis.protocol.mdq.{DDLRequest, DDLResponse} import com.webank.wedatasphere.linkis.rpc.Sender import com.webank.wedatasphere.linkis.storage.utils.StorageUtils +import javax.annotation.PostConstruct import org.apache.commons.lang.StringUtils import org.springframework.stereotype.Component diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/common/LineBufferedStream.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/common/LineBufferedStream.scala index f2b6a85c58..00f8781913 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/common/LineBufferedStream.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/common/LineBufferedStream.scala @@ -21,7 +21,7 @@ import java.util.concurrent.locks.ReentrantLock import com.webank.wedatasphere.linkis.common.conf.CommonVars import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} -import com.webank.wedatasphere.linkis.engine.configuration.{SparkConfiguration} +import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration import org.apache.commons.io.IOUtils import scala.concurrent.duration.Duration diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/common/LogContainer.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/common/LogContainer.scala index 4e1d8ec1cd..a4ab282e92 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/common/LogContainer.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/common/LogContainer.scala @@ -17,8 +17,8 @@ package com.webank.wedatasphere.linkis.engine.spark.common import scala.collection.Iterable -import scala.collection.mutable.ArrayBuffer import scala.collection.JavaConversions._ +import scala.collection.mutable.ArrayBuffer /** * Created by allenlliu on 2018/11/19. diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/utils/EngineUtils.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/utils/EngineUtils.scala index 4c4e3331c8..758473fb3f 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/utils/EngineUtils.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/utils/EngineUtils.scala @@ -27,9 +27,9 @@ import com.webank.wedatasphere.linkis.common.utils.Utils import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration._ import com.webank.wedatasphere.linkis.engine.spark.common.LineBufferedProcess import com.webank.wedatasphere.linkis.rpc.Sender -import com.webank.wedatasphere.linkis.storage.{FSFactory, LineMetaData} import com.webank.wedatasphere.linkis.storage.resultset.ResultSetReader import com.webank.wedatasphere.linkis.storage.utils.StorageUtils +import com.webank.wedatasphere.linkis.storage.{FSFactory, LineMetaData} import scala.util.Random diff --git a/ujes/definedEngines/spark/engine/src/main/scala/org/apache/spark/sql/execution/datasources/csv/DolphinToSpark.scala b/ujes/definedEngines/spark/engine/src/main/scala/org/apache/spark/sql/execution/datasources/csv/DolphinToSpark.scala index 7db344a6c1..3393bc9ee6 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/org/apache/spark/sql/execution/datasources/csv/DolphinToSpark.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/org/apache/spark/sql/execution/datasources/csv/DolphinToSpark.scala @@ -16,7 +16,6 @@ package org.apache.spark.sql.execution.datasources.csv - import java.util import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration @@ -60,7 +59,7 @@ object DolphinToSpark { def toSparkType(dataType:wds.DataType):DataType = dataType match { case wds.NullType => NullType - //case dwc.StringType | dwc.CharType | dwc.VarcharType | dwc.StructType | dwc.ListType | dwc.ArrayType | dwc.MapType => StringType + //case wds.StringType | wds.CharType | wds.VarcharType | wds.StructType | wds.ListType | wds.ArrayType | wds.MapType => StringType case wds.BooleanType => BooleanType case wds.ShortIntType => ShortType case wds.IntType => IntegerType @@ -69,7 +68,7 @@ object DolphinToSpark { case wds.DoubleType => DoubleType case wds.DecimalType => DecimalType(bigDecimalPrecision,bigDecimalScale) case wds.DateType => DateType - case wds.TimestampType => TimestampType + //case wds.TimestampType => TimestampType case wds.BinaryType => BinaryType case _ => StringType } diff --git a/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/configuration/SparkConfiguration.scala b/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/configuration/SparkConfiguration.scala index 320cec61fd..1f6a28940c 100644 --- a/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/configuration/SparkConfiguration.scala +++ b/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/configuration/SparkConfiguration.scala @@ -16,12 +16,10 @@ package com.webank.wedatasphere.linkis.enginemanager.configuration - import com.webank.wedatasphere.linkis.common.conf.{CommonVars, Configuration} import com.webank.wedatasphere.linkis.common.utils.{ClassUtils, Logging} import com.webank.wedatasphere.linkis.engine.factory.SparkEngineExecutorFactory import com.webank.wedatasphere.linkis.enginemanager.AbstractEngineCreator -import com.webank.wedatasphere.linkis.enginemanager.process.{JavaProcessEngineBuilder, SparkSubmitProcessBuilder} import scala.collection.mutable.ArrayBuffer /** diff --git a/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/configuration/SparkEngineManagerSpringConfiguration.scala b/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/configuration/SparkEngineManagerSpringConfiguration.scala index af085136a9..cb2bb5af3a 100644 --- a/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/configuration/SparkEngineManagerSpringConfiguration.scala +++ b/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/configuration/SparkEngineManagerSpringConfiguration.scala @@ -22,7 +22,6 @@ import com.webank.wedatasphere.linkis.enginemanager.hook._ import com.webank.wedatasphere.linkis.resourcemanager.domain.ModuleInfo import com.webank.wedatasphere.linkis.resourcemanager.{DriverAndYarnResource, LoadInstanceResource, ResourceRequestPolicy} import com.webank.wedatasphere.linkis.rpc.Sender -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.context.annotation.{Bean, Configuration} /** diff --git a/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/SparkEngineCreator.scala b/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/SparkEngineCreator.scala index 1f338e8623..5aaef2f19c 100644 --- a/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/SparkEngineCreator.scala +++ b/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/SparkEngineCreator.scala @@ -18,11 +18,8 @@ package com.webank.wedatasphere.linkis.enginemanager.process import com.webank.wedatasphere.linkis.common.conf.DWCArgumentsParser import com.webank.wedatasphere.linkis.common.utils.Logging -import com.webank.wedatasphere.linkis.enginemanager.conf.EngineManagerConfiguration +import com.webank.wedatasphere.linkis.enginemanager.AbstractEngineCreator import com.webank.wedatasphere.linkis.enginemanager.impl.UserTimeoutEngineResource -import com.webank.wedatasphere.linkis.enginemanager.{AbstractEngineCreator, Engine, EngineManagerReceiver, EngineResource} -import com.webank.wedatasphere.linkis.protocol.engine.{EngineCallback, RequestEngine} -import com.webank.wedatasphere.linkis.rpc.Sender import org.springframework.stereotype.Component /** diff --git a/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/SparkSubmitProcessBuilder.scala b/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/SparkSubmitProcessBuilder.scala index 99bac971da..ae678d1327 100644 --- a/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/SparkSubmitProcessBuilder.scala +++ b/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/SparkSubmitProcessBuilder.scala @@ -19,7 +19,7 @@ package com.webank.wedatasphere.linkis.enginemanager.process import java.lang.ProcessBuilder.Redirect -import com.webank.wedatasphere.linkis.common.utils.{ByteTimeUtils, Logging} +import com.webank.wedatasphere.linkis.common.utils.Logging import com.webank.wedatasphere.linkis.enginemanager.EngineResource import com.webank.wedatasphere.linkis.enginemanager.conf.EnvConfiguration._ import com.webank.wedatasphere.linkis.enginemanager.configuration.SparkConfiguration @@ -29,7 +29,6 @@ import com.webank.wedatasphere.linkis.enginemanager.impl.UserEngineResource import com.webank.wedatasphere.linkis.enginemanager.process.SparkSubmitProcessBuilder.{AbsolutePath, Path, RelativePath} import com.webank.wedatasphere.linkis.protocol.engine.RequestEngine import com.webank.wedatasphere.linkis.resourcemanager.DriverAndYarnResource -import org.apache.commons.lang.StringUtils import scala.collection.mutable.ArrayBuffer @@ -135,7 +134,7 @@ class SparkSubmitProcessBuilder extends ProcessEngineBuilder with Logging { this.deployMode("client") val driverJavaSet = "\"-Dwds.linkis.configuration=linkis-engine.properties " + SparkConfiguration.getJavaRemotePort + "\"" this.conf(SPARK_DRIVER_EXTRA_JAVA_OPTIONS.key, driverJavaSet) - this.name(properties.getOrDefault("appName", "sparksqltest")) + this.name(properties.getOrDefault("appName", "linkis")) this.className(properties.getOrDefault("className", "com.webank.wedatasphere.linkis.engine.DataWorkCloudEngineApplication")) properties.getOrDefault("archives", "").toString.split(",").map(RelativePath).foreach(this.archive) this.driverCores(DWC_SPARK_DRIVER_CORES) @@ -153,7 +152,7 @@ class SparkSubmitProcessBuilder extends ProcessEngineBuilder with Logging { this.driverClassPath(SPARK_DRIVER_CLASSPATH.getValue) this.redirectOutput(Redirect.PIPE) this.redirectErrorStream(true) - this.env("spark.app.name", properties.getOrDefault("appName", "dwc" + request.creator)) + this.env("spark.app.name", properties.getOrDefault("appName", "linkis" + request.creator)) } diff --git a/ujes/definedEngines/spark/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/job/SparkEntranceJob.scala b/ujes/definedEngines/spark/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/job/SparkEntranceJob.scala index 500cb40f4b..c522d64ca7 100644 --- a/ujes/definedEngines/spark/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/job/SparkEntranceJob.scala +++ b/ujes/definedEngines/spark/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/job/SparkEntranceJob.scala @@ -16,7 +16,6 @@ package com.webank.wedatasphere.linkis.entrance.job - /** * Created by allenlliu on 2018/12/5. */ diff --git a/ujes/definedEngines/tispark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/configuration/TiSparkSpringConfiguration.scala b/ujes/definedEngines/tispark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/configuration/TiSparkSpringConfiguration.scala index 75ab5400b9..664308d18b 100644 --- a/ujes/definedEngines/tispark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/configuration/TiSparkSpringConfiguration.scala +++ b/ujes/definedEngines/tispark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/configuration/TiSparkSpringConfiguration.scala @@ -16,11 +16,6 @@ package com.webank.wedatasphere.linkis.tispark.engine.configuration -import com.webank.wedatasphere.linkis.engine.execute.EngineHook -import com.webank.wedatasphere.linkis.engine.execute.hook._ -import com.webank.wedatasphere.linkis.tispark.engine.hook.{TiSparkHook, UserDataBaseHook} -import org.springframework.context.annotation.{Bean, Configuration} - /** * Created by johnnwang on 2019/6/26. */ diff --git a/ujes/definedEngines/tispark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/hook/TiSparkHook.scala b/ujes/definedEngines/tispark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/hook/TiSparkHook.scala index d1eafba8e3..7c8177bde1 100644 --- a/ujes/definedEngines/tispark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/hook/TiSparkHook.scala +++ b/ujes/definedEngines/tispark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/hook/TiSparkHook.scala @@ -16,13 +16,6 @@ package com.webank.wedatasphere.linkis.tispark.engine.hook -import com.webank.wedatasphere.linkis.common.utils.Logging -import com.webank.wedatasphere.linkis.engine.exception.EngineErrorException -import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineHook} -import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteRequest, RunTypeExecuteRequest} -import com.webank.wedatasphere.linkis.server.JMap -import org.apache.commons.lang.StringUtils - /** * Created by johnnwang on 2019/6/26. */ diff --git a/ujes/engine/pom.xml b/ujes/engine/pom.xml index 9fa05dbcbb..3b52059131 100644 --- a/ujes/engine/pom.xml +++ b/ujes/engine/pom.xml @@ -80,6 +80,11 @@ test + + com.webank.wedatasphere.linkis + linkis-cs-ujes-client + ${linkis.version} + diff --git a/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/LogCache.java b/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/LogCache.java index adcc6cd609..63b67fba23 100644 --- a/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/LogCache.java +++ b/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/LogCache.java @@ -17,8 +17,6 @@ package com.webank.wedatasphere.linkis.engine.log; -import org.apache.logging.log4j.core.LogEvent; - import java.util.List; /** diff --git a/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/MountLogCache.java b/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/MountLogCache.java index fb897025e6..5512ce525d 100644 --- a/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/MountLogCache.java +++ b/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/MountLogCache.java @@ -18,8 +18,6 @@ import com.webank.wedatasphere.linkis.engine.conf.EngineConfiguration$; import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.core.LogEvent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -121,7 +119,7 @@ public List getLog(int num) { } @Override - public List getRemain() { + public synchronized List getRemain() { return logs.getRemain(); } diff --git a/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/RPCAppender.java b/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/RPCAppender.java index d418ba4293..e7482f6b7f 100644 --- a/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/RPCAppender.java +++ b/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/RPCAppender.java @@ -22,18 +22,10 @@ */ package com.webank.wedatasphere.linkis.engine.log; -import com.webank.wedatasphere.linkis.engine.EngineReceiver; import com.webank.wedatasphere.linkis.rpc.Sender; import com.webank.wedatasphere.linkis.scheduler.listener.LogListener; -import com.webank.wedatasphere.linkis.scheduler.queue.Job; -import com.webank.wedatasphere.linkis.server.Message; import org.apache.log4j.AppenderSkeleton; import org.apache.log4j.spi.LoggingEvent; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; - -import java.util.HashMap; -import java.util.Map; public class RPCAppender extends AppenderSkeleton { diff --git a/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/SendAppender.java b/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/SendAppender.java index 3b47153d4b..2fe1c88922 100644 --- a/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/SendAppender.java +++ b/ujes/engine/src/main/java/com/webank/wedatasphere/linkis/engine/log/SendAppender.java @@ -20,6 +20,7 @@ import com.webank.wedatasphere.linkis.engine.conf.EngineConfiguration; import com.webank.wedatasphere.linkis.engine.conf.EngineConfiguration$; import com.webank.wedatasphere.linkis.scheduler.listener.LogListener; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.Filter; import org.apache.logging.log4j.core.Layout; import org.apache.logging.log4j.core.LogEvent; @@ -51,25 +52,33 @@ public class SendAppender extends AbstractAppender { private LogCache logCache; private static final Logger logger = LoggerFactory.getLogger(SendAppender.class); + private static final String IGNORE_WORDS = EngineConfiguration.ENGINE_IGNORE_WORDS().getValue(); + + private static final String[] IGNORE_WORD_ARR = IGNORE_WORDS.split(","); + + private static final String PASS_WORDS = EngineConfiguration.ENGINE_PASS_WORDS().getValue(); + + private static final String[] PASS_WORDS_ARR = PASS_WORDS.split(","); + class SendThread implements Runnable{ @Override public void run() { - if (logListener == null){ + if (logListener == null){ //ignore - }else{ - if (logCache == null){ - logger.warn("logCache is null"); - return; - } - List logs = logCache.getRemain(); - if (logs.size() > 0){ - StringBuilder sb = new StringBuilder(); - for(String log : logs){ - sb.append(log); - } - logListener.onLogUpdate(null, sb.toString()); - } - } + }else{ + if (logCache == null){ + logger.warn("logCache is null"); + return; + } + List logs = logCache.getRemain(); + if (logs.size() > 0){ + StringBuilder sb = new StringBuilder(); + for(String log : logs){ + sb.append(log).append("\n"); + } + logListener.onLogUpdate(null, sb.toString()); + } + } } } @@ -77,8 +86,7 @@ public void run() { public SendAppender(final String name, final Filter filter, final Layout layout, final boolean ignoreExceptions) { super(name, filter, layout, ignoreExceptions); - //todo enjoyyin 500 to be made configurable ide number（500要做成可配置ide数字） - //this.logCache = new MountLogCache((Integer) EngineConfiguration.ENGINE_LOG_CACHE_NUM().getValue()); + //todo cooperyang 500要做成可配置ide数字 this.logCache = LogHelper.logCache(); SendThread thread = new SendThread(); Utils.defaultScheduler().scheduleAtFixedRate(thread, 10, (Integer)EngineConfiguration$.MODULE$.ENGINE_LOG_SEND_TIME_INTERVAL().getValue(), TimeUnit.MILLISECONDS); @@ -88,16 +96,34 @@ public static void setLogListener(LogListener ll){ logListener = ll; } -// public static void setLogCache(LogCache lc){ -// logCache = lc; -// } + @Override public void append(LogEvent event) { if (logListener == null) { return; } - logCache.cacheLog(new String(getLayout().toByteArray(event))); + String logStr = new String(getLayout().toByteArray(event)); + if (event.getLevel().intLevel() == Level.INFO.intLevel()){ + boolean flag = false; + for(String ignoreLog : IGNORE_WORD_ARR){ + if (logStr.contains(ignoreLog)){ + flag = true; + break; + } + } + for(String word : PASS_WORDS_ARR){ + if(logStr.contains(word)){ + flag = false; + break; + } + } + if (!flag) { + logCache.cacheLog(logStr); + } + }else{ + logCache.cacheLog(logStr); + } } @PluginFactory diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineReceiver.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineReceiver.scala index 255ac59746..006bad7bfe 100644 --- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineReceiver.scala +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineReceiver.scala @@ -17,7 +17,7 @@ package com.webank.wedatasphere.linkis.engine import java.lang.management.ManagementFactory -import java.util.concurrent.{Future, ScheduledFuture, TimeUnit} +import java.util.concurrent.{Future, TimeUnit} import com.webank.wedatasphere.linkis.common.ServiceInstance import com.webank.wedatasphere.linkis.common.conf.DWCArgumentsParser @@ -28,7 +28,7 @@ import com.webank.wedatasphere.linkis.engine.conf.EngineConfiguration.{ENGINE_PU import com.webank.wedatasphere.linkis.engine.exception.{EngineErrorException, JobNotExistsException} import com.webank.wedatasphere.linkis.engine.execute.scheduler.EngineGroupFactory import com.webank.wedatasphere.linkis.engine.execute.{CommonEngineJob, _} -import com.webank.wedatasphere.linkis.engine.log.{LogHelper, MountLogCache, SendAppender} +import com.webank.wedatasphere.linkis.engine.log.{LogHelper, SendAppender} import com.webank.wedatasphere.linkis.protocol.UserWithCreator import com.webank.wedatasphere.linkis.protocol.engine._ import com.webank.wedatasphere.linkis.resourcemanager.UserResultResource diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineServer.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineServer.scala index 0ab3454c52..d5ffb82cf5 100644 --- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineServer.scala +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineServer.scala @@ -19,7 +19,6 @@ package com.webank.wedatasphere.linkis.engine import java.io.Closeable import com.webank.wedatasphere.linkis.common.utils.Logging -import javax.annotation.PreDestroy import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineServerSpringConfiguration.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineServerSpringConfiguration.scala index bec668c87e..f873e83d41 100644 --- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineServerSpringConfiguration.scala +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/EngineServerSpringConfiguration.scala @@ -26,7 +26,6 @@ import com.webank.wedatasphere.linkis.protocol.engine.RequestTask import com.webank.wedatasphere.linkis.rpc.Sender import com.webank.wedatasphere.linkis.scheduler.SchedulerContext import com.webank.wedatasphere.linkis.scheduler.queue.Job -import org.springframework.beans.factory.annotation.Qualifier import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.context.annotation.{Bean, Configuration} diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/LockManager.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/LockManager.scala index 42b2c244be..69e59b7524 100644 --- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/LockManager.scala +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/LockManager.scala @@ -17,7 +17,6 @@ package com.webank.wedatasphere.linkis.engine import com.webank.wedatasphere.linkis.scheduler.SchedulerContext -import com.webank.wedatasphere.linkis.scheduler.executer.Executor /** * Created by enjoyyin on 2018/9/3. diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PropertiesExecuteRequest.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PropertiesExecuteRequest.scala new file mode 100644 index 0000000000..790e037891 --- /dev/null +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PropertiesExecuteRequest.scala @@ -0,0 +1,22 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.engine + +/** + * @author peacewong + * @date 2020/3/5 17:28 + */ +trait PropertiesExecuteRequest { + val properties: java.util.Map[String, Object] +} diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/ResourceExecuteRequest.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/ResourceExecuteRequest.scala index 52d2998222..9c1dfd6623 100644 --- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/ResourceExecuteRequest.scala +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/ResourceExecuteRequest.scala @@ -5,5 +5,5 @@ package com.webank.wedatasphere.linkis.engine * Description: */ trait ResourceExecuteRequest { - val resources:java.util.List[Object] + def resources:java.util.List[Object] } diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/conf/EngineConfiguration.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/conf/EngineConfiguration.scala index 3de772fc8e..25e5e7f64c 100644 --- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/conf/EngineConfiguration.scala +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/conf/EngineConfiguration.scala @@ -60,4 +60,10 @@ object EngineConfiguration { val ENGINE_PUSH_PROGRESS_TO_ENTRANCE = CommonVars("wds.linkis.engine.push.progress.enable", true) val ENGINE_PRE_EXECUTE_HOOK_CLASSES = CommonVars("wds.linkis.engine.pre.hook.class", "com.webank.wedatasphere.linkis.bml.hook.BmlEnginePreExecuteHook") + + val ENGINE_IGNORE_WORDS = CommonVars("wds.linkis.engine.ignore.words", "org.apache.spark.deploy.yarn.Client") + + val ENGINE_PASS_WORDS = CommonVars("wds.linkis.engine.pass.words", "org.apache.hadoop.hive.ql.exec.Task") + + val ENGINE_TUNING_DX_PERIOD = CommonVars("wds.linkis.engine.tuning.dx.period", 1000*60*5) } diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSEnginePreExecuteHook.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSEnginePreExecuteHook.scala new file mode 100644 index 0000000000..ac08532172 --- /dev/null +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSEnginePreExecuteHook.scala @@ -0,0 +1,49 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.engine.cs + +import com.webank.wedatasphere.linkis.common.utils.Logging +import com.webank.wedatasphere.linkis.cs.client.utils.ContextServiceUtils +import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils +import com.webank.wedatasphere.linkis.engine.PropertiesExecuteRequest +import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext +import com.webank.wedatasphere.linkis.engine.extension.EnginePreExecuteHook +import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteRequest + +/** + * @author peacewong + * @date 2020/3/5 17:13 + */ +class CSEnginePreExecuteHook extends EnginePreExecuteHook with Logging { + + private val csResourceParser: CSResourceParser = new CSResourceParser + + override val hookName: String = "ContextServicePreHook" + + + override def callPreExecuteHook(engineExecutorContext: EngineExecutorContext, executeRequest: ExecuteRequest, code: String): String = executeRequest match { + case propertiesExecuteRequest: PropertiesExecuteRequest => + var parsedCode = code + val contextIDValueStr = ContextServiceUtils.getContextIDStrByMap(propertiesExecuteRequest.properties) + val nodeNameStr = ContextServiceUtils.getNodeNameStrByMap(propertiesExecuteRequest.properties) + engineExecutorContext.addProperty(CSCommonUtils.CONTEXT_ID_STR, contextIDValueStr) + engineExecutorContext.addProperty(CSCommonUtils.NODE_NAME_STR, nodeNameStr) + info(s"Start to call cs engine pre hook,contextID is $contextIDValueStr, nodeNameStr is $nodeNameStr") + parsedCode = csResourceParser.parse(propertiesExecuteRequest, parsedCode, contextIDValueStr, nodeNameStr) + + info(s"Finished to call cs engine pre hook,contextID is $contextIDValueStr, nodeNameStr is $nodeNameStr") + parsedCode + case _ => code + } +} diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSResourceParser.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSResourceParser.scala new file mode 100644 index 0000000000..df09dfc170 --- /dev/null +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSResourceParser.scala @@ -0,0 +1,73 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.engine.cs + + +import java.util +import java.util.regex.Pattern + +import com.webank.wedatasphere.linkis.cs.client.service.CSResourceService +import com.webank.wedatasphere.linkis.engine.PropertiesExecuteRequest +import org.apache.commons.lang.StringUtils + +import scala.collection.JavaConversions._ +import scala.collection.mutable.ArrayBuffer + +/** + * @author peacewong + * @date 2020/3/5 19:42 + */ +class CSResourceParser { + + private val pb = Pattern.compile("cs://[^\\s\"]+[$\\s]{0,1}", Pattern.CASE_INSENSITIVE) + + private val PREFIX = "cs://" + + private def getPreFixResourceNames(code: String): Array[String] = { + val bmlResourceNames = new ArrayBuffer[String]() + val mb = pb.matcher(code) + while (mb.find) bmlResourceNames.append(mb.group.trim) + bmlResourceNames.toArray + } + + def parse(executeRequest: PropertiesExecuteRequest, code: String, contextIDValueStr: String, nodeNameStr: String): String = { + + //TODO getBMLResource peaceWong + val bmlResourceList = CSResourceService.getInstance().getUpstreamBMLResource(contextIDValueStr, nodeNameStr) + + val parsedResources = new util.ArrayList[util.Map[String, Object]]() + val preFixResourceNames = getPreFixResourceNames(code) + + val preFixNames = new ArrayBuffer[String]() + val parsedNames = new ArrayBuffer[String]() + preFixResourceNames.foreach { preFixResourceName => + val resourceName = preFixResourceName.replace(PREFIX, "").trim + val bmlResourceOption = bmlResourceList.find(_.getDownloadedFileName.equals(resourceName)) + if (bmlResourceOption.isDefined) { + val bmlResource = bmlResourceOption.get + val map = new util.HashMap[String, Object]() + map.put("resourceId", bmlResource.getResourceId) + map.put("version", bmlResource.getVersion) + map.put("fileName", resourceName) + parsedResources.add(map) + preFixNames.append(preFixResourceName) + parsedNames.append(resourceName) + } + + } + executeRequest.properties.put("resources", parsedResources) + StringUtils.replaceEach(code, preFixNames.toArray, parsedNames.toArray) + } + +} diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSTableRegister.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSTableRegister.scala new file mode 100644 index 0000000000..e6c4dadc87 --- /dev/null +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/cs/CSTableRegister.scala @@ -0,0 +1,92 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.engine.cs + +import java.util.Date + +import com.webank.wedatasphere.linkis.common.io.resultset.ResultSetWriter +import com.webank.wedatasphere.linkis.common.io.{MetaData, Record} +import com.webank.wedatasphere.linkis.common.utils.Logging +import com.webank.wedatasphere.linkis.cs.client.service.CSTableService +import com.webank.wedatasphere.linkis.cs.client.utils.{ContextServiceUtils, SerializeHelper} +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.{ContextScope, ContextType} +import com.webank.wedatasphere.linkis.cs.common.entity.metadata.{CSColumn, CSTable} +import com.webank.wedatasphere.linkis.cs.common.entity.source.CommonContextKey +import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils +import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorContext +import com.webank.wedatasphere.linkis.storage.domain.Column +import com.webank.wedatasphere.linkis.storage.utils.StorageUtils +import org.apache.commons.lang.StringUtils + +/** + * @author peacewong + * @date 2020/3/13 16:29 + */ +object CSTableRegister extends Logging{ + + def registerTempTable(engineExecutorContext: EngineExecutorContext, + writer: ResultSetWriter[_ <: MetaData, _ <: Record], alias: String, columns: Array[Column]): Unit = { + + val contextIDValueStr = ContextServiceUtils.getContextIDStrByMap(engineExecutorContext.getProperties) + val nodeNameStr = ContextServiceUtils.getNodeNameStrByMap(engineExecutorContext.getProperties) + + if (StringUtils.isNotBlank(contextIDValueStr) && StringUtils.isNotBlank(nodeNameStr)) { + info(s"Start to register TempTable nodeName:$nodeNameStr") + writer.flush() + val tableName = if (StringUtils.isNotBlank(alias)) s"${CSCommonUtils.CS_TMP_TABLE_PREFIX}${nodeNameStr}_${alias}" else { + var i = 1; + var rsName: String = null; + while (StringUtils.isEmpty(rsName)) { + val tmpTable = s"${CSCommonUtils.CS_TMP_TABLE_PREFIX}${nodeNameStr}_rs${i}" + i = i + 1 + val contextKey = new CommonContextKey + contextKey.setContextScope(ContextScope.FRIENDLY) + contextKey.setContextType(ContextType.METADATA) + contextKey.setKey(CSCommonUtils.getTableKey(nodeNameStr, tmpTable)) + val table = CSTableService.getInstance().getCSTable(contextIDValueStr, SerializeHelper.serializeContextKey(contextKey)) + if (null == table) { + rsName = tmpTable + } + } + rsName + } + val csTable = new CSTable + csTable.setName(tableName) + csTable.setAlias(alias) + csTable.setAvailable(true) + csTable.setComment("cs temp table") + csTable.setCreateTime(new Date()) + csTable.setCreator(StorageUtils.getJvmUser) + csTable.setExternalUse(true) + csTable.setImport(false) + csTable.setLocation(writer.toString) + csTable.setPartitionTable(false) + csTable.setView(true) + val csColumns = columns.map { column => + val csColumn = new CSColumn + csColumn.setName(column.columnName) + csColumn.setType(column.dataType.typeName) + csColumn.setComment(column.comment) + csColumn + } + csTable.setColumns(csColumns) + val contextKey = new CommonContextKey + contextKey.setContextScope(ContextScope.PUBLIC) + contextKey.setContextType(ContextType.METADATA) + contextKey.setKey(CSCommonUtils.getTableKey(nodeNameStr, tableName)) + CSTableService.getInstance().putCSTable(contextIDValueStr, SerializeHelper.serializeContextKey(contextKey), csTable) + info(s"Finished to register TempTable nodeName:$nodeNameStr") + } + } +} diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/CodeParser.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/CodeParser.scala index 3652e31e3f..e60b6e27a7 100644 --- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/CodeParser.scala +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/CodeParser.scala @@ -23,7 +23,6 @@ import com.webank.wedatasphere.linkis.engine.execute.CodeType.CodeType import org.apache.commons.lang.StringUtils import org.slf4j.{Logger, LoggerFactory} -import scala.collection.immutable.HashSet import scala.collection.mutable import scala.collection.mutable.ArrayBuffer @@ -101,8 +100,11 @@ class PythonCodeParser extends SingleCodeParser { statementBuffer.append(l) recordBrackets(bracketStack, l) case l if quotationMarks => statementBuffer.append(l) - recordBrackets(bracketStack, l) + //shanhuang 用于修复python的引号问题 + //recordBrackets(bracketStack, l) case l if notDoc && l.startsWith("#") => + case l if StringUtils.isNotBlank(statementBuffer.last) && statementBuffer.last.endsWith("""\""") => + statementBuffer.append(l) case l if notDoc && l.startsWith(" ") => statementBuffer.append(l) recordBrackets(bracketStack, l.trim) @@ -180,13 +182,13 @@ class SQLCodeParser extends SingleCodeParser { if (StringUtils.contains(code, separator)) { StringUtils.split(code, ";").foreach{ case s if StringUtils.isBlank(s) => - case s if isSelectCmdNoLimit(s) => appendStatement(s + " limit " + defaultLimit); + case s if isSelectCmdNoLimit(s) => appendStatement(s); case s => appendStatement(s); } } else { code match { case s if StringUtils.isBlank(s) => - case s if isSelectCmdNoLimit(s) => appendStatement(s + " limit " + defaultLimit); + case s if isSelectCmdNoLimit(s) => appendStatement(s); case s => appendStatement(s); } } diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutor.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutor.scala index 95c4107c0e..83b7f18f6c 100644 --- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutor.scala +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutor.scala @@ -16,10 +16,8 @@ package com.webank.wedatasphere.linkis.engine.execute -import com.webank.wedatasphere.linkis.common.log.LogUtils import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} import com.webank.wedatasphere.linkis.engine.conf.EngineConfiguration -import com.webank.wedatasphere.linkis.engine.exception.EngineErrorException import com.webank.wedatasphere.linkis.engine.extension.EnginePreExecuteHook import com.webank.wedatasphere.linkis.resourcemanager.Resource import com.webank.wedatasphere.linkis.scheduler.executer._ @@ -49,14 +47,14 @@ abstract class EngineExecutor(outputPrintLimit: Int, isSupportParallelism: Boole val hooks = new ArrayBuffer[EnginePreExecuteHook]() EngineConfiguration.ENGINE_PRE_EXECUTE_HOOK_CLASSES.getValue.split(",") foreach { hookStr => Utils.tryCatch{ - val clazz = Class.forName(hookStr) + val clazz = Class.forName(hookStr.trim) val obj = clazz.newInstance() obj match { case hook:EnginePreExecuteHook => hooks += hook case _ => logger.warn(s"obj is not a engineHook obj is ${obj.getClass}") } }{ - case e:Exception => logger.error("failed to load class", e) + case e:Exception => logger.error(s"failed to load class ${hookStr}") } } hooks.toArray @@ -117,20 +115,22 @@ abstract class EngineExecutor(outputPrintLimit: Int, isSupportParallelism: Boole else if(isSupportParallelism) whenAvailable(f) else ensureIdle(f) ensureOp { val engineExecutorContext = createEngineExecutorContext(executeRequest) + var hookedCode = executeRequest.code; Utils.tryCatch{ enginePreExecuteHooks foreach { hook => logger.info(s"${hook.hookName} begins to do a hook") - hook.callPreExecuteHook(engineExecutorContext, executeRequest) + hookedCode = hook.callPreExecuteHook(engineExecutorContext, executeRequest, hookedCode) logger.info(s"${hook.hookName} ends to do a hook") } }{ - case e:Exception => logger.info("failed to do with hook") + case e:Throwable => logger.info("failed to do with hook", e) } + info(s"hooked after code:$hookedCode") var response: ExecuteResponse = null val incomplete = new StringBuilder - val codes = Utils.tryCatch(codeParser.map(_.parse(executeRequest.code, engineExecutorContext)).getOrElse(Array(executeRequest.code))){ - e => warn("Your code failed to commit one line at a time, and is now ready to execute as a full commit(您的代码在进行一行一行代码提交时失败，现在准备按照全部提交的方式进行执行)",e) - Array(executeRequest.code) + val codes = Utils.tryCatch(codeParser.map(_.parse(hookedCode, engineExecutorContext)).getOrElse(Array(hookedCode))){ + e => info("Your code will be submitted in overall mode") + Array(hookedCode) } engineExecutorContext.setTotalParagraph(codes.length) codes.indices.foreach { index => @@ -145,7 +145,7 @@ abstract class EngineExecutor(outputPrintLimit: Int, isSupportParallelism: Boole //engineExecutorContext.appendStdout(getName + ">> " + incomplete.toString().trim + " complete ") response match { case e: ErrorExecuteResponse => - error(s"execute code $code failed!", e.t) + error(s"execute code failed!", e.t) return response case SuccessExecuteResponse() => engineExecutorContext.appendStdout("\n") diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutorContext.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutorContext.scala index 2e9a43c5dc..f4e87dd1bf 100644 --- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutorContext.scala +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutorContext.scala @@ -23,11 +23,14 @@ import java.util.concurrent.atomic.AtomicInteger import com.webank.wedatasphere.linkis.common.io.resultset.{ResultSet, ResultSetWriter} import com.webank.wedatasphere.linkis.common.io.{FsPath, MetaData, Record} import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} +import com.webank.wedatasphere.linkis.cs.client.utils.ContextServiceUtils +import com.webank.wedatasphere.linkis.cs.storage.CSTableResultSetWriter import com.webank.wedatasphere.linkis.engine.conf.EngineConfiguration.{ENGINE_RESULT_SET_MAX_CACHE, ENGINE_RESULT_SET_STORE_PATH} import com.webank.wedatasphere.linkis.engine.exception.EngineErrorException import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo import com.webank.wedatasphere.linkis.rpc.Sender import com.webank.wedatasphere.linkis.scheduler.executer.{AliasOutputExecuteResponse, OutputExecuteResponse} +import com.webank.wedatasphere.linkis.storage.resultset.table.TableResultSet import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetWriter} import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord} import org.apache.commons.io.IOUtils @@ -36,7 +39,7 @@ import org.apache.commons.lang.time.DateFormatUtils import scala.collection.mutable.ArrayBuffer -class EngineExecutorContext(engineExecutor: EngineExecutor) extends Logging{ +class EngineExecutorContext(engineExecutor: EngineExecutor) extends Logging { private val resultSetFactory = ResultSetFactory.getInstance private val resultSetWriters = ArrayBuffer[ResultSetWriter[_ <: MetaData, _ <: Record]]() @@ -48,17 +51,21 @@ class EngineExecutorContext(engineExecutor: EngineExecutor) extends Logging{ private val aliasNum = new AtomicInteger(0) protected var storePath: Option[String] = None - private val properties:java.util.Map[String,Object] = new util.HashMap[String, Object]() + private val properties: java.util.Map[String, Object] = new util.HashMap[String, Object]() private var totalParagraph = 0 private var currentParagraph = 0 def kill(): Unit = interrupted = true + def isKilled: Boolean = interrupted def getTotalParagraph: Int = totalParagraph + def setTotalParagraph(totalParagraph: Int): Unit = this.totalParagraph = totalParagraph + def getCurrentParagraph: Int = currentParagraph + def setCurrentParagraph(currentParagraph: Int): Unit = this.currentParagraph = currentParagraph def pushProgress(progress: Float, progressInfo: Array[JobProgressInfo]): Unit = @@ -66,10 +73,10 @@ class EngineExecutorContext(engineExecutor: EngineExecutor) extends Logging{ def sendResultSet(resultSetWriter: ResultSetWriter[_ <: MetaData, _ <: Record]): Unit = { val fileName = new File(resultSetWriter.toFSPath.getPath).getName - val index = if(fileName.indexOf(".") < 0) fileName.length else fileName.indexOf(".") - val alias = if(fileName.startsWith("_")) fileName.substring(1, index) else fileName.substring(0, fileName.indexOf("_")) -// resultSetWriter.flush() - Utils.tryFinally(sendResultSet(resultSetWriter.toString(), alias)){ + val index = if (fileName.indexOf(".") < 0) fileName.length else fileName.indexOf(".") + val alias = if (fileName.startsWith("_")) fileName.substring(1, index) else fileName.substring(0, fileName.indexOf("_")) + // resultSetWriter.flush() + Utils.tryFinally(sendResultSet(resultSetWriter.toString(), alias)) { IOUtils.closeQuietly(resultSetWriter) resultSetWriters synchronized resultSetWriters -= resultSetWriter } @@ -78,8 +85,8 @@ class EngineExecutorContext(engineExecutor: EngineExecutor) extends Logging{ def sendResultSet(output: String): Unit = sendResultSet(output, "_" + aliasNum.getAndIncrement()) def appendTextResultSet(output: String): Unit = { - if(defaultResultSetWriter == null) aliasNum synchronized { - if(defaultResultSetWriter == null) { + if (defaultResultSetWriter == null) aliasNum synchronized { + if (defaultResultSetWriter == null) { defaultResultSetWriter = createDefaultResultSetWriter(ResultSetFactory.TEXT_TYPE) defaultResultSetWriter.addMetaData(new LineMetaData()) resultSetWriters += defaultResultSetWriter @@ -91,12 +98,12 @@ class EngineExecutorContext(engineExecutor: EngineExecutor) extends Logging{ private def sendResultSet(output: String, alias: String): Unit = { if (StringUtils.isEmpty(output)) return if (resultSetFactory.isResultSetPath(output)) - engineExecutor.getResultSetListener.foreach{ l => + engineExecutor.getResultSetListener.foreach { l => jobId.foreach(l.onResultSetCreated(_, output, alias)) resultSize += 1 } else if (resultSetFactory.isResultSet(output)) - engineExecutor.getResultSetListener.foreach{ l => + engineExecutor.getResultSetListener.foreach { l => jobId.foreach(l.onResultSetCreated(_, output, alias)) resultSize += 1 } @@ -104,7 +111,9 @@ class EngineExecutorContext(engineExecutor: EngineExecutor) extends Logging{ } def setJobId(jobId: String) = this.jobId = Option(jobId) + def getJobId = jobId + def setStorePath(storePath: String) = this.storePath = Option(storePath) def sendResultSet(outputExecuteResponse: OutputExecuteResponse): Unit = outputExecuteResponse match { @@ -112,13 +121,13 @@ class EngineExecutorContext(engineExecutor: EngineExecutor) extends Logging{ case output: OutputExecuteResponse => sendResultSet(output.getOutput, "_" + aliasNum.getAndIncrement()) } - def getProperties:java.util.Map[String, Object] = properties + def getProperties: java.util.Map[String, Object] = properties - def addProperty(key:String, value:String):Unit = properties.put(key, value) + def addProperty(key: String, value: String): Unit = properties.put(key, value) protected def getDefaultStorePath: String = { val path = ENGINE_RESULT_SET_STORE_PATH.getValue - (if(path.endsWith("/")) path else path + "/") + "user" + "/" + + (if (path.endsWith("/")) path else path + "/") + "user" + "/" + DateFormatUtils.format(System.currentTimeMillis(), "yyyyMMdd") + "/" + Sender.getThisServiceInstance.getApplicationName + "/" + System.nanoTime } @@ -139,16 +148,34 @@ class EngineExecutorContext(engineExecutor: EngineExecutor) extends Logging{ def createResultSetWriter(resultSet: ResultSet[_ <: MetaData, _ <: Record], alias: String): ResultSetWriter[_ <: MetaData, _ <: Record] = { val filePath = storePath.getOrElse(getDefaultStorePath) - val fileName = if(StringUtils.isEmpty(alias)) "_" + aliasNum.getAndIncrement() else alias + "_" + aliasNum.getAndIncrement() + val fileName = if (StringUtils.isEmpty(alias)) "_" + aliasNum.getAndIncrement() else alias + "_" + aliasNum.getAndIncrement() val resultSetPath = resultSet.getResultSetPath(new FsPath(filePath), fileName) - val resultSetWriter = ResultSetWriter.getResultSetWriter(resultSet, ENGINE_RESULT_SET_MAX_CACHE.getValue.toLong, resultSetPath) + //update by peaceWong 20200402 + val resultSetWriter = resultSet match { + case result: TableResultSet => + val contextIDStr = ContextServiceUtils.getContextIDStrByMap(getProperties) + val nodeName = ContextServiceUtils.getNodeNameStrByMap(getProperties) + if (StringUtils.isNotBlank(contextIDStr) && StringUtils.isNotBlank(nodeName)) { + new CSTableResultSetWriter(result, ENGINE_RESULT_SET_MAX_CACHE.getValue.toLong, resultSetPath, contextIDStr, nodeName, alias) + } else { + ResultSetWriter.getResultSetWriter(resultSet, ENGINE_RESULT_SET_MAX_CACHE.getValue.toLong, resultSetPath) + } + case _ => ResultSetWriter.getResultSetWriter(resultSet, ENGINE_RESULT_SET_MAX_CACHE.getValue.toLong, resultSetPath) + } + //update by peaceWong 20200402 end resultSetWriters synchronized resultSetWriters += resultSetWriter resultSetWriter } - def appendStdout(log: String): Unit = if(!engineExecutor.isEngineInitialized) + def appendStdout(log: String): Unit = if (!engineExecutor.isEngineInitialized) engineExecutor.info(log) else engineExecutor.getLogListener.foreach(ll => jobId.foreach(ll.onLogUpdate(_, log))) + def sendProgress(progress: Float, progressInfos: Array[JobProgressInfo]): Unit = { + if (engineExecutor.isEngineInitialized) { + engineExecutor.getJobProgressListener.foreach(ll => jobId.foreach(ll.onProgressUpdate(_, progress, progressInfos))) + } + } + def close(): Unit = { resultSetWriters.toArray.foreach(sendResultSet) engineExecutor.getResultSetListener.foreach(l => jobId.foreach(l.onResultSizeCreated(_, resultSize))) diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutorManager.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutorManager.scala index 38712a497b..6dac410e85 100644 --- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutorManager.scala +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineExecutorManager.scala @@ -58,11 +58,12 @@ abstract class EngineExecutorManager extends ExecutorManager with Logging { if(executor == null) synchronized { if(executor == null) { var options: JMap[String, String] = DWCArgumentsParser.getDWCOptionMap + //TODO getUDF peaceWong getEngineHooks.foreach(hook => options = hook.beforeCreateEngine(options)) executor = getOrCreateEngineExecutorFactory().createExecutor(options) + executor.setCodeParser(getOrCreateCodeParser()) executor.init() executor.setLogListener(jobLogListener) - executor.setCodeParser(getOrCreateCodeParser()) executor.setResultSetListener(resultSetListener) //TODO Consider adding timeout（考虑加上超时时间） getEngineHooks.foreach(_.afterCreatedEngine(executor)) diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineJob.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineJob.scala index ac093c4ea2..89f1a6f84f 100644 --- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineJob.scala +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/EngineJob.scala @@ -16,14 +16,17 @@ package com.webank.wedatasphere.linkis.engine.execute +import com.webank.wedatasphere.linkis.engine.{PropertiesExecuteRequest, ResourceExecuteRequest} import com.webank.wedatasphere.linkis.protocol.engine.RequestTask import com.webank.wedatasphere.linkis.scheduler.executer.{ErrorExecuteResponse, ExecuteRequest, JobExecuteRequest, RunTypeExecuteRequest} import com.webank.wedatasphere.linkis.scheduler.queue.{Job, JobInfo} - +import java.util /** * Created by enjoyyin on 2018/9/25. */ -abstract class EngineJob extends Job with SenderContainer +abstract class EngineJob extends Job with SenderContainer{ + val resourcesStr:String = "resources" +} class CommonEngineJob extends EngineJob with SyncSenderContainer { protected var request: RequestTask = _ @@ -36,13 +39,24 @@ class CommonEngineJob extends EngineJob with SyncSenderContainer { override def isJobSupportRetry: Boolean = false override protected def jobToExecuteRequest: ExecuteRequest = { + if (request.getProperties.containsKey("runType") && request.getProperties.containsKey(RequestTask.RESULT_SET_STORE_PATH)) - return new ExecuteRequest with JobExecuteRequest with StorePathExecuteRequest with RunTypeExecuteRequest{ + return new ExecuteRequest with JobExecuteRequest with StorePathExecuteRequest + with RunTypeExecuteRequest with ResourceExecuteRequest with PropertiesExecuteRequest { override val code: String = request.getCode override val jobId: String = CommonEngineJob.this.getId override val storePath: String = request.getProperties.get(RequestTask.RESULT_SET_STORE_PATH).toString - override val runType: String = request.getProperties.get("runType").toString + override val runType: String = if (request.getProperties.get("runType") != null) { + request.getProperties.get("runType").toString + } else "sql" + override def resources: util.List[Object] = properties.get(resourcesStr) match { + case rs:util.List[Object] => rs + case _ => logger.warn(s"${CommonEngineJob.this.getId} resources type is not correct") + null + } + override val properties: util.Map[String, Object] = request.getProperties } + if(request.getProperties.containsKey(RequestTask.RESULT_SET_STORE_PATH)) new ExecuteRequest with JobExecuteRequest with StorePathExecuteRequest { override val code: String = request.getCode override val jobId: String = CommonEngineJob.this.getId diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/CodeGeneratorEngineHook.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/CodeGeneratorEngineHook.scala index 974627565e..9af88a8caf 100644 --- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/CodeGeneratorEngineHook.scala +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/CodeGeneratorEngineHook.scala @@ -20,7 +20,6 @@ import java.io.File import com.webank.wedatasphere.linkis.common.utils.Logging import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineHook} -import com.webank.wedatasphere.linkis.protocol.engine.RequestEngine import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteRequest, RunTypeExecuteRequest} import com.webank.wedatasphere.linkis.server.JMap import org.apache.commons.io.FileUtils diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/ReleaseEngineHook.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/ReleaseEngineHook.scala index 6b4cfa79e8..10fb13ae7d 100644 --- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/ReleaseEngineHook.scala +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/ReleaseEngineHook.scala @@ -18,13 +18,12 @@ package com.webank.wedatasphere.linkis.engine.execute.hook import java.util.concurrent.TimeUnit -import com.webank.wedatasphere.linkis.common.utils.{ByteTimeUtils, Logging, Utils} +import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} import com.webank.wedatasphere.linkis.engine.conf.EngineConfiguration import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineHook} import com.webank.wedatasphere.linkis.protocol.engine.EngineState.Idle import com.webank.wedatasphere.linkis.scheduler.executer.ExecutorState import com.webank.wedatasphere.linkis.server.JMap -import org.apache.commons.lang.StringUtils /** * Created by enjoyyin on 2018/9/27. diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/UdfLoadEngineHook.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/UdfLoadEngineHook.scala index 4a1d2d24c3..2af1999d55 100644 --- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/UdfLoadEngineHook.scala +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/execute/hook/UdfLoadEngineHook.scala @@ -21,7 +21,6 @@ import java.io.File import com.webank.wedatasphere.linkis.common.utils.Logging import com.webank.wedatasphere.linkis.engine.conf.EngineConfiguration._ import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineHook} -import com.webank.wedatasphere.linkis.protocol.engine.RequestEngine import com.webank.wedatasphere.linkis.rpc.Sender import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteRequest, RunTypeExecuteRequest} import com.webank.wedatasphere.linkis.server.JMap @@ -30,10 +29,9 @@ import com.webank.wedatasphere.linkis.udf.entity.{UDFInfo, UDFTree} import org.apache.commons.collections.CollectionUtils import org.apache.commons.io.FileUtils import org.apache.commons.lang.StringUtils -import org.codehaus.jackson.map.ObjectMapper -import scala.collection.mutable import scala.collection.JavaConversions._ +import scala.collection.mutable import scala.collection.mutable.ArrayBuffer abstract class UdfLoadEngineHook extends EngineHook with Logging{ self => diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/extension/EnginePreExecuteHook.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/extension/EnginePreExecuteHook.scala index 1bbbfcbd34..ae656051b2 100644 --- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/extension/EnginePreExecuteHook.scala +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/extension/EnginePreExecuteHook.scala @@ -9,5 +9,5 @@ import com.webank.wedatasphere.linkis.scheduler.executer.ExecuteRequest */ trait EnginePreExecuteHook { val hookName:String - def callPreExecuteHook(engineExecutorContext:EngineExecutorContext, executeRequest: ExecuteRequest) + def callPreExecuteHook(engineExecutorContext:EngineExecutorContext, executeRequest: ExecuteRequest, code: String): String } \ No newline at end of file diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/lock/EngineTimedLockManager.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/lock/EngineTimedLockManager.scala index 8eba4156a4..9550c4208f 100644 --- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/lock/EngineTimedLockManager.scala +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/lock/EngineTimedLockManager.scala @@ -16,16 +16,12 @@ package com.webank.wedatasphere.linkis.engine.lock -import java.util.concurrent.{ConcurrentHashMap, ScheduledThreadPoolExecutor, TimeUnit} - import com.webank.wedatasphere.linkis.common.utils.Logging import com.webank.wedatasphere.linkis.engine.LockManager import com.webank.wedatasphere.linkis.engine.execute.EngineExecutorManager import com.webank.wedatasphere.linkis.scheduler.SchedulerContext import com.webank.wedatasphere.linkis.scheduler.executer.ExecutorState -import scala.collection.JavaConversions._ - class EngineTimedLockManager(schedulerContext: SchedulerContext) extends LockManager(schedulerContext) with Logging{ var executorLock: EngineTimedLock = null diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/log/LogHelper.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/log/LogHelper.scala index a4eae5a857..5ba7c13329 100644 --- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/log/LogHelper.scala +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/log/LogHelper.scala @@ -36,7 +36,7 @@ object LogHelper { def setLogListener(logListener: LogListener):Unit = this.logListener = logListener def pushAllRemainLogs():Unit = { - logger.info(s"start to push all remain logs, and size is ${logCache.getRemain.size()}") + logger.info(s"start to push all remain logs") Thread.sleep(30) //logCache.synchronized{ if (logListener == null){ diff --git a/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/execute/SparkCombinedCodeParserTest.scala b/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/execute/SparkCombinedCodeParserTest.scala index 1c838389e6..4ce6c62d7e 100644 --- a/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/execute/SparkCombinedCodeParserTest.scala +++ b/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/execute/SparkCombinedCodeParserTest.scala @@ -16,11 +16,6 @@ package com.webank.wedatasphere.linkis.engine.execute -import java.io.File - -import com.google.common.io.Resources -import org.apache.commons.io.FileUtils - //object SparkCombinedCodeParserTest { // def main(args: Array[String]): Unit = { // val parser = new SparkCombinedCodeParser diff --git a/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/lock/EngineTimedLockManagerTest.scala b/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/lock/EngineTimedLockManagerTest.scala index 5c5e371f52..893b5d497b 100644 --- a/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/lock/EngineTimedLockManagerTest.scala +++ b/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/lock/EngineTimedLockManagerTest.scala @@ -17,7 +17,6 @@ package com.webank.wedatasphere.linkis.engine.lock import com.webank.wedatasphere.linkis.common.listener.ListenerEventBus -import com.webank.wedatasphere.linkis.engine.lock.EngineTimedLockManagerTest.lockManager import com.webank.wedatasphere.linkis.scheduler.SchedulerContext import com.webank.wedatasphere.linkis.scheduler.event.{ScheduleEvent, SchedulerEventListener} import com.webank.wedatasphere.linkis.scheduler.executer.ExecutorState.ExecutorState diff --git a/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/log/ScalaLoggingTest.scala b/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/log/ScalaLoggingTest.scala index 31a051c7a9..d050ad715f 100644 --- a/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/log/ScalaLoggingTest.scala +++ b/ujes/engine/src/main/test/scala/com/webank/wedatasphere/linkis/engine/log/ScalaLoggingTest.scala @@ -16,7 +16,6 @@ package scala.com.webank.wedatasphere.linkis.engine.log -import com.webank.wedatasphere.linkis.common.utils.Logging import org.slf4j.{Logger, LoggerFactory} /** diff --git a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/AbstractEngineCreator.scala b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/AbstractEngineCreator.scala index 15af33f400..fe8fca53ff 100644 --- a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/AbstractEngineCreator.scala +++ b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/AbstractEngineCreator.scala @@ -22,7 +22,7 @@ import com.webank.wedatasphere.linkis.common.conf.DWCArgumentsParser import com.webank.wedatasphere.linkis.common.utils.Utils import com.webank.wedatasphere.linkis.enginemanager.conf.EngineManagerConfiguration import com.webank.wedatasphere.linkis.enginemanager.exception.EngineManagerErrorException -import com.webank.wedatasphere.linkis.enginemanager.impl.{UserEngineResource, UserTimeoutEngineResource} +import com.webank.wedatasphere.linkis.enginemanager.impl.UserTimeoutEngineResource import com.webank.wedatasphere.linkis.enginemanager.process.{CommonProcessEngine, ProcessEngine, ProcessEngineBuilder} import com.webank.wedatasphere.linkis.protocol.engine.{EngineCallback, RequestEngine} import com.webank.wedatasphere.linkis.rpc.Sender diff --git a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/AbstractEngineManager.scala b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/AbstractEngineManager.scala index cc6a2b07e4..df2242f6bc 100644 --- a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/AbstractEngineManager.scala +++ b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/AbstractEngineManager.scala @@ -21,7 +21,7 @@ import java.util.concurrent.TimeUnit import com.webank.wedatasphere.linkis.common.log.LogUtils import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} import com.webank.wedatasphere.linkis.enginemanager.conf.EngineManagerConfiguration -import com.webank.wedatasphere.linkis.enginemanager.exception.{EngineManagerErrorException, EngineManagerWarnException} +import com.webank.wedatasphere.linkis.enginemanager.exception.{EMRetryException, EngineManagerErrorException, EngineManagerWarnException} import com.webank.wedatasphere.linkis.protocol.engine.RequestEngine import com.webank.wedatasphere.linkis.resourcemanager.{AvailableResource, NotEnoughResource, Resource} @@ -49,7 +49,7 @@ abstract class AbstractEngineManager extends EngineManager with Logging { val resource = Utils.tryThrow(getEngineManagerContext.getOrCreateEngineResourceFactory .createEngineResource(realRequest)){t => warn(s"In the configuration of ${realRequest.creator}, there is a parameter configuration in the wrong format!(${realRequest.creator}的配置中，存在错误格式的参数配置！)", t) - throw new EngineManagerErrorException(30000, s"In the configuration of ${realRequest.creator}, there is a parameter configuration in the wrong format!(${realRequest.creator}的配置中，存在错误格式的参数配置！)") + throw new EngineManagerErrorException(11011, s"In the configuration of ${realRequest.creator}, there is a parameter configuration in the wrong format!(${realRequest.creator}的配置中，存在错误格式的参数配置！)") } val nodeResourceInfo = this.registerResources() val usedResource = getEngineManagerContext.getOrCreateEngineFactory.getUsedResources.getOrElse(Resource.getZeroResource(resource.getResource)) @@ -59,7 +59,7 @@ abstract class AbstractEngineManager extends EngineManager with Logging { info("ProtectedResource: "+ nodeResourceInfo.protectedResource.toString) info("UsedResource: "+ usedResource.toString) info("RequestResource: "+ resource.getResource.toString) - throw new EngineManagerErrorException(31000, "The remote server resource has been used up, please switch to the remote server and try again!(远程服务器资源已被用光，请切换远程服务器再试！)") + throw new EngineManagerWarnException(31000, "远程服务器资源已被用光，请切换远程服务器再试！") } getEngineManagerContext.getOrCreateResourceRequester.request(resource) match { case NotEnoughResource(reason) => diff --git a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/conf/EnvConfiguration.scala b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/conf/EnvConfiguration.scala index e01d27d370..2c16639b8f 100644 --- a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/conf/EnvConfiguration.scala +++ b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/conf/EnvConfiguration.scala @@ -17,7 +17,7 @@ package com.webank.wedatasphere.linkis.enginemanager.conf import com.webank.wedatasphere.linkis.common.conf.{ByteType, CommonVars} -import EngineManagerConfiguration.ENGINE_SPRING_APPLICATION_NAME +import com.webank.wedatasphere.linkis.enginemanager.conf.EngineManagerConfiguration.ENGINE_SPRING_APPLICATION_NAME import org.apache.commons.lang.time.DateFormatUtils /** diff --git a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/exception/EMRetryException.scala b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/exception/EMRetryException.scala new file mode 100644 index 0000000000..ad30b466da --- /dev/null +++ b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/exception/EMRetryException.scala @@ -0,0 +1,10 @@ +package com.webank.wedatasphere.linkis.enginemanager.exception + +import com.webank.wedatasphere.linkis.common.exception.DWCRetryException + +/** + * created by cooperyang on 2019/12/11 + * Description: + */ +class EMRetryException (errCode:Int, desc:String ) extends DWCRetryException(errCode,desc){ +} diff --git a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/hook/JarLoaderEngineHook.scala b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/hook/JarLoaderEngineHook.scala index c09e944244..e65c20473e 100644 --- a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/hook/JarLoaderEngineHook.scala +++ b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/hook/JarLoaderEngineHook.scala @@ -56,12 +56,13 @@ class JarLoaderEngineHook extends EngineHook with Logging{ } protected def isJarExists(udfInfo: UDFInfo) : Boolean = { - if(FileUtils.getFile(udfInfo.getPath).exists()){ - true - } else { - info(s"The jar file [${udfInfo.getPath}] of UDF [${udfInfo.getUdfName}] doesn't exist, ignore it.") - false - } + true +// if(FileUtils.getFile(udfInfo.getPath).exists()){ +// true +// } else { +// info(s"The jar file [${udfInfo.getPath}] of UDF [${udfInfo.getUdfName}] doesn't exist, ignore it.") +// false +// } } protected def extractUdfInfos(requestEngine: RequestEngine): mutable.ArrayBuffer[UDFInfo] = { diff --git a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/impl/EngineManagerImpl.scala b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/impl/EngineManagerImpl.scala index f751550920..799203a73a 100644 --- a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/impl/EngineManagerImpl.scala +++ b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/impl/EngineManagerImpl.scala @@ -16,7 +16,7 @@ package com.webank.wedatasphere.linkis.enginemanager.impl -import com.webank.wedatasphere.linkis.enginemanager.{AbstractEngineManager, Engine, EngineManagerContext} +import com.webank.wedatasphere.linkis.enginemanager.{AbstractEngineManager, EngineManagerContext} import com.webank.wedatasphere.linkis.resourcemanager.domain.ModuleInfo import com.webank.wedatasphere.linkis.resourcemanager.service.annotation.{EnableResourceManager, RegisterResource} import org.springframework.beans.factory.annotation.Autowired diff --git a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/impl/EngineManagerSpringConfiguration.scala b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/impl/EngineManagerSpringConfiguration.scala index 583fcc488f..599c221187 100644 --- a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/impl/EngineManagerSpringConfiguration.scala +++ b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/impl/EngineManagerSpringConfiguration.scala @@ -16,7 +16,6 @@ package com.webank.wedatasphere.linkis.enginemanager.impl -import com.webank.wedatasphere.linkis.common.utils.ByteTimeUtils import com.webank.wedatasphere.linkis.enginemanager._ import com.webank.wedatasphere.linkis.enginemanager.conf.EngineManagerConfiguration.ENGINE_SPRING_APPLICATION_NAME import com.webank.wedatasphere.linkis.enginemanager.conf.EnvConfiguration._ diff --git a/ujes/entrance/pom.xml b/ujes/entrance/pom.xml index 90294dcacb..596fe5123b 100644 --- a/ujes/entrance/pom.xml +++ b/ujes/entrance/pom.xml @@ -53,6 +53,19 @@ provided + + com.webank.wedatasphere.linkis + linkis-cs-ujes-client + ${linkis.version} + + + + org.apache.httpcomponents + httpclient + ${httpclient.version} + provided + + junit junit diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceExecutorManagerBeanAnnotation.java b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceExecutorManagerBeanAnnotation.java index 2cbe6561f7..902733be5f 100644 --- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceExecutorManagerBeanAnnotation.java +++ b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/annotation/EntranceExecutorManagerBeanAnnotation.java @@ -22,7 +22,10 @@ import org.springframework.core.annotation.AliasFor; import org.springframework.stereotype.Component; -import java.lang.annotation.*; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; /** * Created by enjoyyin on 2019/2/14. diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/EntranceSpringConfiguration.java b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/EntranceSpringConfiguration.java index 62445f5ea1..360e8ffa1f 100644 --- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/EntranceSpringConfiguration.java +++ b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/conf/EntranceSpringConfiguration.java @@ -90,18 +90,24 @@ public PersistenceManager generatePersistenceManager(@PersistenceEngineBeanAnnot @EntranceListenerBusBeanAnnotation @ConditionalOnMissingBean(name = {EntranceListenerBusBeanAnnotation.BEAN_NAME}) - public EntranceEventListenerBus generateEntranceEventListenerBus(){ + public EntranceEventListenerBus generateEntranceEventListenerBus() { EntranceEventListenerBus entranceEventListenerBus = new EntranceEventListenerBus(); entranceEventListenerBus.start(); return entranceEventListenerBus; } + /** + * Update by peaceWong add CSEntranceInterceptor + * + * @return + */ @EntranceInterceptorBeanAnnotation @ConditionalOnMissingBean(name = {EntranceInterceptorBeanAnnotation.BEAN_NAME}) public EntranceInterceptor[] generateEntranceInterceptors() { - return new EntranceInterceptor[] {new PythonCodeCheckInterceptor(), new DBInfoCompleteInterceptor(), new SparkCodeCheckInterceptor(), + return new EntranceInterceptor[]{new CSEntranceInterceptor(), new PythonCodeCheckInterceptor(), new DBInfoCompleteInterceptor(), new SparkCodeCheckInterceptor(), new SQLCodeCheckInterceptor(), new VarSubstitutionInterceptor(), new LogPathCreateInterceptor(), - new StorePathEntranceInterceptor(), new ScalaCodeInterceptor(), new SQLLimitEntranceInterceptor(),new CommentInterceptor()}; + new StorePathEntranceInterceptor(), new ScalaCodeInterceptor(), new SQLLimitEntranceInterceptor(), new CommentInterceptor(), + }; } @ErrorCodeListenerBeanAnnotation diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceIllegalParamException.java b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceIllegalParamException.java index a66d74f952..82da5b66b5 100644 --- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceIllegalParamException.java +++ b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/exception/EntranceIllegalParamException.java @@ -16,9 +16,7 @@ package com.webank.wedatasphere.linkis.entrance.exception; -import com.webank.wedatasphere.linkis.common.exception.DWCRuntimeException; import com.webank.wedatasphere.linkis.common.exception.ErrorException; -import com.webank.wedatasphere.linkis.common.exception.ExceptionLevel; /** * created by enjoyyin on 2018/10/8 diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/job/EntranceExecutionJob.java b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/job/EntranceExecutionJob.java index 2d600b0c5e..1f5585b500 100644 --- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/job/EntranceExecutionJob.java +++ b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/job/EntranceExecutionJob.java @@ -16,7 +16,6 @@ package com.webank.wedatasphere.linkis.entrance.job; -import com.webank.wedatasphere.linkis.common.conf.TimeType; import com.webank.wedatasphere.linkis.common.log.LogUtils; import com.webank.wedatasphere.linkis.common.utils.Utils; import com.webank.wedatasphere.linkis.entrance.execute.*; diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/parser/AbstractEntranceParser.java b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/parser/AbstractEntranceParser.java index 634a48af0b..8910c70fb2 100644 --- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/parser/AbstractEntranceParser.java +++ b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/parser/AbstractEntranceParser.java @@ -107,6 +107,7 @@ public Job parseToJob(Task task) throws EntranceIllegalParamException { //job.setProgressListener(entranceContext.getOrCreatePersistenceManager()); //job.setJobListener(entranceContext.getOrCreatePersistenceManager()); job.setEntranceListenerBus(entranceContext.getOrCreateEventListenerBus()); + job.setEntranceContext(entranceContext); job.setListenerEventBus(null); job.setProgress(0f); } diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceEngine.java b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceEngine.java index 7088234ee9..67daab244a 100644 --- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceEngine.java +++ b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceEngine.java @@ -23,6 +23,7 @@ package com.webank.wedatasphere.linkis.entrance.persistence; import com.google.gson.Gson; +import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration; import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration$; import com.webank.wedatasphere.linkis.entrance.exception.EntranceIllegalParamException; import com.webank.wedatasphere.linkis.entrance.exception.EntranceRPCException; @@ -82,10 +83,6 @@ public void persist(Task task) throws QueryFailedException, EntranceIllegalParam if (object == null){ throw new QueryFailedException(20011, "insert task failed, reason: " + message); } -/* if (object instanceof Long){ - Long taskID = (Long)object; - ((RequestPersistTask) task).setTaskID(taskID); - }*/ String taskStr = object.toString(); Long taskID = Long.parseLong(taskStr.substring(0,taskStr.indexOf("."))); ((RequestPersistTask) task).setTaskID(taskID); diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceManager.java b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceManager.java index 9a89ff86d3..27edd6acb5 100644 --- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceManager.java +++ b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/persistence/QueryPersistenceManager.java @@ -25,6 +25,7 @@ import com.webank.wedatasphere.linkis.common.exception.ErrorException; import com.webank.wedatasphere.linkis.common.io.FsPath; import com.webank.wedatasphere.linkis.entrance.EntranceContext; +import com.webank.wedatasphere.linkis.entrance.cs.CSEntranceHelper; import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob; import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo; import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask; @@ -79,7 +80,6 @@ public ResultSetEngine createResultSetEngine() { public void onResultSetCreated(Job job, OutputExecuteResponse response) { String path; boolean isEntranceJob = job instanceof EntranceJob; -// if(isEntranceJob) ((EntranceJob)job).incrementResultSetPersist(); try { path = createResultSetEngine().persistResultSet(job, response); } catch (Throwable e) { @@ -98,14 +98,18 @@ public void onResultSetCreated(Job job, OutputExecuteResponse response) { } catch (Throwable e1){ logger.error("job {} onLogUpdate error, reason:", job.getId(), e1); } //ignore it - if(isEntranceJob) ((EntranceJob)job).incrementResultSetPersisted(); + if(isEntranceJob) { + ((EntranceJob)job).incrementResultSetPersisted(); + } return; } if(task instanceof RequestPersistTask) { RequestPersistTask requestPersistTask = (RequestPersistTask) task; if(StringUtils.isEmpty(requestPersistTask.getResultLocation())) synchronized (task) { if(StringUtils.isNotEmpty(requestPersistTask.getResultLocation())) { - if(isEntranceJob) ((EntranceJob)job).incrementResultSetPersisted(); + if(isEntranceJob) { + ((EntranceJob)job).incrementResultSetPersisted(); + } return; } try { @@ -117,7 +121,9 @@ public void onResultSetCreated(Job job, OutputExecuteResponse response) { } } } - if(isEntranceJob) ((EntranceJob)job).incrementResultSetPersisted(); + if(isEntranceJob) { + ((EntranceJob)job).incrementResultSetPersisted(); + } } @Override @@ -153,6 +159,15 @@ public void onJobWaitForRetry(Job job) { @Override public void onJobCompleted(Job job) { + //update by peaceWong(2020/05/10) to set jobID to CS + try { + if (job.isSucceed()) { + CSEntranceHelper.registerCSRSData(job); + } + } catch (Throwable e) { + logger.error("Failed to register cs rs data ", e); + } + //end update updateJobStatus(job); } @@ -163,12 +178,15 @@ private void updateJobStatus(Job job){ } try{ task = this.entranceContext.getOrCreateEntranceParser().parseToTask(job); + if (job.isSucceed()){ + //如果是job是成功的，那么需要将task的错误描述等都要设置为null + ((RequestPersistTask)task).setErrCode(null); + ((RequestPersistTask)task).setErrDesc(null); + } }catch(ErrorException e){ entranceContext.getOrCreateLogManager().onLogUpdate(job, e.getMessage()); logger.error("update job status failed, reason:", e); } - //TODO If the execution fails, there may be an error message, you need to call job.getErrorResponse to persist the error message.(如果是执行失败了，可能会有错误信息，需要调用job.getErrorResponse持久化错误信息) - //TODO The error message is compared with the error code of errorListener(错误信息要跟errorListener的错误码信息对比) try { createPersistenceEngine().updateIfNeeded(task); } catch (ErrorException e) { diff --git a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulApi.java b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulApi.java index 2a23bb542d..44238f8f42 100644 --- a/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulApi.java +++ b/ujes/entrance/src/main/java/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulApi.java @@ -20,9 +20,9 @@ import com.webank.wedatasphere.linkis.entrance.EntranceServer; import com.webank.wedatasphere.linkis.entrance.annotation.EntranceServerBeanAnnotation; import com.webank.wedatasphere.linkis.entrance.background.BackGroundService; -import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration; import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob; import com.webank.wedatasphere.linkis.entrance.log.LogReader; +import com.webank.wedatasphere.linkis.entrance.utils.JobHistoryHelper; import com.webank.wedatasphere.linkis.protocol.constants.TaskConstant; import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo; import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask; @@ -37,9 +37,7 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.springframework.web.bind.annotation.RequestParam; import scala.Option; import javax.servlet.http.HttpServletRequest; @@ -113,10 +111,22 @@ private void pushLog(String log, Job job){ @Override @GET @Path("/{id}/status") - public Response status(@PathParam("id") String id) { + public Response status(@PathParam("id") String id, @QueryParam("taskID")String taskID) { Message message = null; String realId = ZuulEntranceUtils.parseExecID(id)[3]; - Option job = entranceServer.getJob(realId); + Option job = Option.apply(null); + try{ + job = entranceServer.getJob(realId); + }catch(Exception e){ + logger.warn("获取任务 {} 状态时出现错误", realId, e); + //如果获取错误了,证明在内存中已经没有了,去jobhistory找寻一下taskID代表的任务的状态，然后返回 + long realTaskID = Long.parseLong(taskID); + String status = JobHistoryHelper.getStatusByTaskID(realTaskID); + message = Message.ok(); + message.setMethod("/api/entrance/" + id + "/status"); + message.data("status",status).data("execID", id); + return Message.messageToResponse(message); + } if (job.isDefined()){ message = Message.ok(); message.setMethod("/api/entrance/" + id + "/status"); @@ -139,20 +149,20 @@ public Response progress(@PathParam("id")String id) { Option job = entranceServer.getJob(realId); if (job.isDefined()){ JobProgressInfo[] jobProgressInfos = ((EntranceJob)job.get()).getProgressInfo(); - Map map = new HashMap<>(); if (jobProgressInfos == null){ message = Message.error("Can not get the corresponding progress information, it may be that the corresponding progress information has not been generated(不能获取相应的进度信息,可能是相应的进度信息还未生成)"); message.setMethod("/api/entrance/" + id + "/progress"); }else{ - for(JobProgressInfo jobProgressInfo : jobProgressInfos){ + List> list = new ArrayList<>(); + for(JobProgressInfo jobProgressInfo : jobProgressInfos) { + Map map = new HashMap<>(); map.put("id", jobProgressInfo.id()); map.put("succeedTasks", jobProgressInfo.succeedTasks()); map.put("failedTasks", jobProgressInfo.failedTasks()); map.put("runningTasks", jobProgressInfo.runningTasks()); map.put("totalTasks", jobProgressInfo.totalTasks()); + list.add(map); } - List> list = new ArrayList<>(); - list.add(map); message = Message.ok(); message.setMethod("/api/entrance/" + id + "/progress"); message.data("progress",job.get().getProgress()).data("execID", id).data("progressInfo", list); @@ -279,9 +289,21 @@ public Response log(@Context HttpServletRequest req, @PathParam("id")String id) @Override @GET @Path("/{id}/kill") - public Response kill(@PathParam("id")String id) { + public Response kill(@PathParam("id")String id, @QueryParam("taskID") long taskID) { String realId = ZuulEntranceUtils.parseExecID(id)[3]; - Option job = entranceServer.getJob(realId); + //通过jobid获取job,可能会由于job找不到而导致有looparray的报错,一旦报错的话，就可以将该任务直接置为Cancenlled + Option job = Option.apply(null); + try{ + job = entranceServer.getJob(realId); + }catch(Exception e){ + logger.warn("can not find a job in entranceServer, will force to kill it", e); + //如果在内存中找不到该任务，那么该任务可能已经完成了，或者就是重启导致的 + JobHistoryHelper.forceKill(taskID); + Message message = Message.ok("强制杀死任务"); + message.setMethod("/api/entrance/" + id + "/kill"); + message.setStatus(0); + return Message.messageToResponse(message); + } Message message = null; if (job.isEmpty()){ message = Message.error("Can't find execID(不能找到execID): " + id + "Corresponding job, can't kill(对应的job，不能进行kill)"); diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceServer.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceServer.scala index faec2d8d58..da2188f763 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceServer.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceServer.scala @@ -18,7 +18,6 @@ package com.webank.wedatasphere.linkis.entrance import com.webank.wedatasphere.linkis.common.exception.{DWCException, DWCRuntimeException, ErrorException} import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} -import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration import com.webank.wedatasphere.linkis.entrance.exception.{EntranceErrorException, SubmitFailedException} import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob import com.webank.wedatasphere.linkis.entrance.log.LogReader diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceWebSocketService.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceWebSocketService.scala index 49f26c85da..89ae6100fd 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceWebSocketService.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/EntranceWebSocketService.scala @@ -309,7 +309,7 @@ class EntranceWebSocketService extends ServerEventService with EntranceEventList if (StringUtils.isBlank(log)) return var message:Message = null val logs:Array[String] = new Array[String](4) - val logArr:Array[String] = log.split("\n") + val logArr:Array[String] = log.split("\n\n").filter(StringUtils.isNotBlank) val info = new StringBuilder val warn = new StringBuilder val error = new StringBuilder @@ -321,14 +321,29 @@ class EntranceWebSocketService extends ServerEventService with EntranceEventList case ERROR_HEADER1() | ERROR_HEADER2() => concatLog(length, singleLog, error, all) case WARN_HEADER1() | WARN_HEADER2() => - val arr = EntranceConfiguration.LOG_EXCLUDE.getValue.split(",").map (word => word.trim) + val arr = EntranceConfiguration.LOG_WARN_EXCLUDE.getValue.split(",").map (word => word.trim) var flag = false for (keyword <- arr){ flag = singleLog.contains(keyword) || flag } - if (!flag) concatLog(length, singleLog, warn, all) + if (!flag) { + val message = singleLog.split("\n")(0) + concatLog(length, message, warn, all) + } case INFO_HEADER1() | INFO_HEADER2() => val hiveLogSpecial:String = EntranceConfiguration.HIVE_SPECIAL_LOG_INCLUDE.getValue + val sparkLogSpecial:String = EntranceConfiguration.SPARK_SPECIAL_LOG_INCLUDE.getValue + val hiveCreateTableLog:String = EntranceConfiguration.HIVE_CREATE_TABLE_LOG.getValue + if (singleLog.contains(hiveLogSpecial) && singleLog.contains(hiveCreateTableLog)){ + val threadName = EntranceConfiguration.HIVE_THREAD_NAME.getValue + val printInfo = EntranceConfiguration.HIVE_PRINT_INFO_LOG.getValue + val start = singleLog.indexOf(threadName) + val end = singleLog.indexOf(printInfo) + printInfo.length + if(start > 0 && end > 0) { + val realLog = singleLog.substring(0, start) + singleLog.substring(end, singleLog.length) + concatLog(length, realLog, info, all) + } + } if (singleLog.contains(hiveLogSpecial) && singleLog.contains("map") && singleLog.contains("reduce")){ val threadName = EntranceConfiguration.HIVE_THREAD_NAME.getValue val stageName = EntranceConfiguration.HIVE_STAGE_NAME.getValue @@ -338,6 +353,15 @@ class EntranceWebSocketService extends ServerEventService with EntranceEventList val realLog = singleLog.substring(0, start) + singleLog.substring(end, singleLog.length) concatLog(length, realLog, info, all) } + }else if (singleLog.contains(sparkLogSpecial)){ + val className = EntranceConfiguration.SPARK_PROGRESS_NAME.getValue + val endFlag = EntranceConfiguration.END_FLAG.getValue + val start = singleLog.indexOf(className) + val end = singleLog.indexOf(endFlag) + endFlag.length + if(start > 0 && end > 0) { + val realLog = singleLog.substring(0, start) + singleLog.substring(end, singleLog.length) + concatLog(length, realLog, info, all) + } }else{ val arr = EntranceConfiguration.LOG_EXCLUDE.getValue.split(",").map (word => word.trim) var flag = false diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/EntranceConfiguration.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/EntranceConfiguration.scala index 9fc0aa2483..feb665a3a1 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/EntranceConfiguration.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/conf/EntranceConfiguration.scala @@ -85,6 +85,7 @@ object EntranceConfiguration { */ val DEFAULT_RUN_TYPE = CommonVars("wds.linkis.default.runType", "sql") + val LOG_WARN_EXCLUDE = CommonVars("wds.linkis.warn.log.exclude", "org.apache,hive.ql,hive.metastore,com.netflix,com.webank.wedatasphere") val CLEAR_LOG = CommonVars("wds.linkis.log.clear", false) @@ -92,7 +93,7 @@ object EntranceConfiguration { * LOG_EXCLUDE is used to remove the log of the framework log, such as hive spark spring, so that it is not pushed to the front end through websocket. * LOG_EXCLUDE 是用来进行把框架日志，比如hive spark spring等日志进行剔除，不让其通过websocket进行推送到前端 */ - val LOG_EXCLUDE = CommonVars("wds.linkis.log.exclude", "org.apache,hive.ql,hive.metastore,com.netflix,cn.webank.bdp,com.webank") + val LOG_EXCLUDE = CommonVars("wds.linkis.log.exclude", "org.apache,hive.ql,hive.metastore,com.netflix,com.webank.wedatasphere,com.webank") /** * wds.linkis.dwc.instance is a parameter used to control the number of engines each user starts. @@ -119,4 +120,26 @@ object EntranceConfiguration { val HIVE_STAGE_NAME = CommonVars("wds.linkis.hive.stage.name", "Stage-") + val SPARK_SPECIAL_LOG_INCLUDE = CommonVars("wds.linkis.spark.special.log.include", "com.webank.wedatasphere.linkis.engine.spark.utils.JobProgressUtil") + + + val SPARK_PROGRESS_NAME = CommonVars("wds.linkis.spark.progress.name", "com.webank.wedatasphere.linkis.engine.spark.utils.JobProgressUtil$") + + val END_FLAG = CommonVars("bdp.dataworkcloud.entrance.end.flag", "info -") + + val HIVE_CREATE_TABLE_LOG = CommonVars("wds.linkis.hive.create.table.log", "numFiles") + + val HIVE_PRINT_INFO_LOG = CommonVars("wds.linkis.hive.printinfo.log", "printInfo -") + + + + val IS_BDP_ENV = CommonVars("wds.linkis.entrance.bdp.env", "true") + + + val SHELL_DANGER_USAGE = CommonVars("wds.linkis.shell.danger.usage", "rm,sh,find,kill,python,for,source,hdfs,hadoop,spark-sql,spark-submit,pyspark,spark-shell,hive,yarn") + val SHELL_WHITE_USAGE = CommonVars("wds.linkis.shell.white.usage", "cd,ls") + + val FLOW_EXECUTION_CREATOR = CommonVars("wds.linkis.entrance.flow.creator", "nodeexecution") + + val SCHEDULER_CREATOR = CommonVars("wds.linkis.entrance.scheduler.creator", "scheduler") } diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/cs/CSEntranceHelper.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/cs/CSEntranceHelper.scala new file mode 100644 index 0000000000..1caa1325a4 --- /dev/null +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/cs/CSEntranceHelper.scala @@ -0,0 +1,168 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.entrance.cs + +import java.util + +import com.webank.wedatasphere.linkis.common.utils.Logging +import com.webank.wedatasphere.linkis.cs.client.service.{CSNodeServiceImpl, CSVariableService, LinkisJobDataServiceImpl} +import com.webank.wedatasphere.linkis.cs.client.utils.{ContextServiceUtils, SerializeHelper} +import com.webank.wedatasphere.linkis.cs.common.entity.`object`.LinkisVariable +import com.webank.wedatasphere.linkis.cs.common.entity.data.LinkisJobData +import com.webank.wedatasphere.linkis.cs.common.entity.enumeration.{ContextScope, ContextType} +import com.webank.wedatasphere.linkis.cs.common.entity.source.{CommonContextKey, LinkisWorkflowContextID} +import com.webank.wedatasphere.linkis.cs.common.utils.CSCommonUtils +import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration +import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob +import com.webank.wedatasphere.linkis.protocol.constants.TaskConstant +import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask +import com.webank.wedatasphere.linkis.protocol.utils.TaskUtils +import com.webank.wedatasphere.linkis.scheduler.queue.Job +import org.apache.commons.lang.StringUtils + +import scala.collection.JavaConversions._ +import scala.collection.mutable + +/** + * @author peacewong + * @date 2020/3/5 15:33 + */ +object CSEntranceHelper extends Logging { + + + def getContextInfo(params: util.Map[String, Any]): (String, String) = { + + val runtimeMap = params.get(TaskConstant.PARAMS_CONFIGURATION) match { + case map: util.Map[String, AnyRef] => map.get(TaskConstant.PARAMS_CONFIGURATION_RUNTIME) + case _ => null + } + + if (null != runtimeMap) { + runtimeMap match { + case map: util.Map[String, AnyRef] => + val name = ContextServiceUtils.getNodeNameStrByMap(map) + return (ContextServiceUtils.getContextIDStrByMap(map), name) + case _ => + } + } + (null, null) + } + + def setContextInfo(params: util.Map[String, Any], copyMap: util.Map[String, String]): Unit = { + val (contextIDValueStr, nodeNameStr) = getContextInfo(params) + if (StringUtils.isNotBlank(contextIDValueStr)) { + copyMap.put(CSCommonUtils.CONTEXT_ID_STR, contextIDValueStr) + copyMap.put(CSCommonUtils.NODE_NAME_STR, nodeNameStr) + } + } + + + /** + * register job id to cs + * + * @param job + */ + def registerCSRSData(job: Job): Unit = { + job match { + case entranceJob: EntranceJob => { + val (contextIDValueStr, nodeNameStr) = getContextInfo(entranceJob.getParams) + info(s"registerCSRSData: nodeName:$nodeNameStr") + if (StringUtils.isBlank(contextIDValueStr) || StringUtils.isBlank(nodeNameStr)) return null + + val contextKey = new CommonContextKey + contextKey.setContextScope(ContextScope.PUBLIC) + contextKey.setContextType(ContextType.DATA) + contextKey.setKey(CSCommonUtils.NODE_PREFIX + nodeNameStr + CSCommonUtils.JOB_ID) + entranceJob.getTask match { + case requestPersistTask: RequestPersistTask => + val data = new LinkisJobData + data.setJobID(requestPersistTask.getTaskID) + LinkisJobDataServiceImpl.getInstance().putLinkisJobData(contextIDValueStr, SerializeHelper.serializeContextKey(contextKey), data) + info(s"(${contextKey.getKey} put ${requestPersistTask.getTaskID} of taskID to cs)") + case _ => + } + info(s"registerCSRSData end: nodeName:$nodeNameStr") + } + case _ => + } + } + + /** + * initNodeCSInfo + * + * @param requestPersistTask + * @return + */ + def initNodeCSInfo(requestPersistTask: RequestPersistTask): Unit = { + + val (contextIDValueStr, nodeNameStr) = getContextInfo(requestPersistTask.getParams.asInstanceOf[util.Map[String, Any]]) + + if (StringUtils.isNotBlank(contextIDValueStr) && StringUtils.isNotBlank(nodeNameStr)) { + info(s"init node($nodeNameStr) cs info") + CSNodeServiceImpl.getInstance().initNodeCSInfo(contextIDValueStr, nodeNameStr) + } + } + + + /** + * reset creator by contextID information + * 1. Not set If contextID does not exists + * 2. If env of contextID are dev set nodeexecution + * 3. If env of contextID are prod set scheduler + * + * @param requestPersistTask + */ + def resetCreator(requestPersistTask: RequestPersistTask): Unit = { + + val (contextIDValueStr, nodeNameStr) = getContextInfo(requestPersistTask.getParams.asInstanceOf[util.Map[String, Any]]) + + if (StringUtils.isNotBlank(contextIDValueStr) && StringUtils.isNotBlank(nodeNameStr)) { + SerializeHelper.deserializeContextID(contextIDValueStr) match { + case contextID: LinkisWorkflowContextID => + if (CSCommonUtils.CONTEXT_ENV_PROD.equalsIgnoreCase(contextID.getEnv)) { + info(s"reset creator from ${requestPersistTask.getRequestApplicationName} to " + EntranceConfiguration.SCHEDULER_CREATOR.getValue) + requestPersistTask.setRequestApplicationName(EntranceConfiguration.SCHEDULER_CREATOR.getValue) + } else { + info(s"reset creator from ${requestPersistTask.getRequestApplicationName} to " + EntranceConfiguration.FLOW_EXECUTION_CREATOR.getValue) + requestPersistTask.setRequestApplicationName(EntranceConfiguration.FLOW_EXECUTION_CREATOR.getValue) + } + case _ => + } + } + } + + + /** + * From cs to get variable + * + * @param requestPersistTask + * @return + */ + def addCSVariable(requestPersistTask: RequestPersistTask): Unit = { + val variableMap = new mutable.HashMap[String, String]() + val (contextIDValueStr, nodeNameStr) = getContextInfo(requestPersistTask.getParams.asInstanceOf[util.Map[String, Any]]) + + if (StringUtils.isNotBlank(contextIDValueStr)) { + info(s"parse variable nodeName:$nodeNameStr") + val linkisVariableList: util.List[LinkisVariable] = CSVariableService.getInstance().getUpstreamVariables(contextIDValueStr, nodeNameStr); + if (null != linkisVariableList) { + linkisVariableList.foreach { linkisVariable => + variableMap.put(linkisVariable.getKey, linkisVariable.getValue) + } + } + TaskUtils.addVariableMap(requestPersistTask.getParams.asInstanceOf[util.Map[String, Any]], variableMap) + info(s"parse variable end nodeName:$nodeNameStr") + } + } +} diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/CacheNotReadyException.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/CacheNotReadyException.scala index 4a9e8b11e4..29ef5f0cd4 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/CacheNotReadyException.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/CacheNotReadyException.scala @@ -22,7 +22,7 @@ */ package com.webank.wedatasphere.linkis.entrance.exception -import com.webank.wedatasphere.linkis.common.exception.{DWCException, ErrorException, ExceptionLevel} +import com.webank.wedatasphere.linkis.common.exception.{ErrorException, ExceptionLevel} case class CacheNotReadyException(errCode:Int, desc:String) extends ErrorException(errCode, desc){ diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/JobHistoryFailedException.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/JobHistoryFailedException.scala new file mode 100644 index 0000000000..132a561d66 --- /dev/null +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/exception/JobHistoryFailedException.scala @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.linkis.entrance.exception + +import com.webank.wedatasphere.linkis.common.exception.ErrorException + +/** + * created by cooperyang on 2020/1/2 + * Description: + */ +case class JobHistoryFailedException(errorMsg:String) extends ErrorException(50081, errorMsg) diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceExecutorManager.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceExecutorManager.scala index 40fe6dbc14..f3ea61c5c7 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceExecutorManager.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceExecutorManager.scala @@ -16,8 +16,12 @@ package com.webank.wedatasphere.linkis.entrance.execute +import java.util.Date + import com.webank.wedatasphere.linkis.common.exception.WarnException import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} +import com.webank.wedatasphere.linkis.entrance.job.EntranceExecutionJob +import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask import com.webank.wedatasphere.linkis.scheduler.executer.ExecutorState.ExecutorState import com.webank.wedatasphere.linkis.scheduler.executer.{Executor, ExecutorManager} import com.webank.wedatasphere.linkis.scheduler.listener.ExecutorListener @@ -77,11 +81,8 @@ abstract class EntranceExecutorManager(groupFactory: GroupFactory) extends Execu protected def findExecutors(job: Job): Array[EntranceEngine] = { val groupName = groupFactory.getGroupNameByEvent(job) - //logger.info(s"${job.getId} groupName is ${groupName}") var engines = getOrCreateEngineManager().listEngines(_.getGroup.getGroupName == groupName) - //logger.info(s"in findExecutors fun engines is $engines") getOrCreateEntranceExecutorRulers().foreach(ruler => engines = ruler.rule(engines, job)) - //logger.info(s"after findExecutors fun engines is $engines") engines } @@ -111,6 +112,11 @@ abstract class EntranceExecutorManager(groupFactory: GroupFactory) extends Execu findUsefulExecutor(job).orElse { val executor = createExecutor(job) if(executor != null) { + job match{ + case entranceExecutionJob: EntranceExecutionJob => val task = entranceExecutionJob.getTask + task.asInstanceOf[RequestPersistTask].setEngineStartTime(new Date()) + case _ => + } if(!job.isCompleted){ val lock = getOrCreateEngineSelector().lockEngine(executor) setLock(lock, job) diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceJob.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceJob.scala index bb49a1eca5..45ae26e017 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceJob.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceJob.scala @@ -20,6 +20,7 @@ import java.util import java.util.concurrent.atomic.AtomicInteger import com.webank.wedatasphere.linkis.common.log.LogUtils +import com.webank.wedatasphere.linkis.entrance.EntranceContext import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration import com.webank.wedatasphere.linkis.entrance.event._ import com.webank.wedatasphere.linkis.entrance.exception.EntranceErrorException @@ -47,6 +48,7 @@ abstract class EntranceJob extends LockJob { private var progressInfo:Array[JobProgressInfo] = Array.empty private val persistedResultSets = new AtomicInteger(0) private var resultSize = -1 + private var entranceContext:EntranceContext = _ def getTask:Task = task def setTask(task:Task):Unit = this.task = task def setCreator(creator: String): Unit = this.creator = creator @@ -60,6 +62,9 @@ abstract class EntranceJob extends LockJob { def getEntranceListenerBus = this.entranceListenerBus def setProgressInfo(progressInfo:Array[JobProgressInfo]):Unit = this.progressInfo = progressInfo def getProgressInfo:Array[JobProgressInfo] = this.progressInfo + def setEntranceContext(entranceContext: EntranceContext):Unit = this.entranceContext = entranceContext + def getEntraceCotnext:EntranceContext = this.entranceContext + def setResultSize(resultSize: Int): Unit = { this.resultSize = resultSize @@ -138,6 +143,11 @@ abstract class EntranceJob extends LockJob { super.transitionCompleted(executeCompleted) } + def transitionCompleted(executeCompleted: CompletedExecuteResponse, reason: String): Unit = { + info("Job directly completed with reason: " + reason) + transitionCompleted(executeCompleted) + } + override protected def isJobShouldRetry(errorExecuteResponse: ErrorExecuteResponse): Boolean = isJobSupportRetry && errorExecuteResponse != null && (if(RPCUtils.isReceiverNotExists(errorExecuteResponse.t)) { getExecutor match { diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceReceiver.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceReceiver.scala index 5803fbcc6b..99e9ea093a 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceReceiver.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/execute/EntranceReceiver.scala @@ -19,10 +19,9 @@ package com.webank.wedatasphere.linkis.entrance.execute import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} import com.webank.wedatasphere.linkis.entrance.EntranceContext import com.webank.wedatasphere.linkis.entrance.annotation.EntranceContextBeanAnnotation -import com.webank.wedatasphere.linkis.entrance.event.{EntranceProgressEvent} +import com.webank.wedatasphere.linkis.entrance.event.EntranceProgressEvent import com.webank.wedatasphere.linkis.entrance.utils.RPCUtils import com.webank.wedatasphere.linkis.protocol.engine._ - import com.webank.wedatasphere.linkis.rpc.exception.DWCRPCRetryException import com.webank.wedatasphere.linkis.rpc.{Receiver, Sender} import com.webank.wedatasphere.linkis.scheduler.executer.AliasOutputExecuteResponse diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/EntranceInterceptor.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/EntranceInterceptor.scala index c15e618e36..0d88e5afdb 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/EntranceInterceptor.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/EntranceInterceptor.scala @@ -16,7 +16,7 @@ package com.webank.wedatasphere.linkis.entrance.interceptor -import com.webank.wedatasphere.linkis.common.exception.{ErrorException, WarnException} +import com.webank.wedatasphere.linkis.common.exception.ErrorException import com.webank.wedatasphere.linkis.protocol.task.Task /** diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/VarSubstitutionException.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/VarSubstitutionException.scala index 927425292e..b371fdacbe 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/VarSubstitutionException.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/exception/VarSubstitutionException.scala @@ -16,7 +16,7 @@ package com.webank.wedatasphere.linkis.entrance.interceptor.exception -import com.webank.wedatasphere.linkis.common.exception.{ErrorException, WarnException} +import com.webank.wedatasphere.linkis.common.exception.ErrorException /** * created by enjoyyin on 2018/10/19 diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CSEntranceInterceptor.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CSEntranceInterceptor.scala new file mode 100644 index 0000000000..26629f847a --- /dev/null +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CSEntranceInterceptor.scala @@ -0,0 +1,42 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.entrance.interceptor.impl + +import java.lang + +import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} +import com.webank.wedatasphere.linkis.entrance.cs.CSEntranceHelper +import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor +import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask +import com.webank.wedatasphere.linkis.protocol.task.Task + +/** + * @author peacewong + * @date 2020/3/24 18:28 + */ +class CSEntranceInterceptor extends EntranceInterceptor with Logging { + + override def apply(task: Task, logAppender: lang.StringBuilder): Task = { + task match { + case requestPersistTask: RequestPersistTask => + logger.info("Start to execute CSEntranceInterceptor") + Utils.tryAndWarn(CSEntranceHelper.addCSVariable(requestPersistTask)) + Utils.tryAndWarn(CSEntranceHelper.resetCreator(requestPersistTask)) + Utils.tryAndWarn(CSEntranceHelper.initNodeCSInfo(requestPersistTask)) + logger.info("Finished to execute CSEntranceInterceptor") + case _ => + } + task + } +} diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CommentInterceptor.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CommentInterceptor.scala index 0d9534b86c..4d34ed4e9f 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CommentInterceptor.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CommentInterceptor.scala @@ -22,9 +22,8 @@ import java.util.regex.Pattern import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask import com.webank.wedatasphere.linkis.protocol.task.Task -import org.apache.commons.lang.StringUtils +import org.slf4j.{Logger, LoggerFactory} -import scala.collection.mutable.ArrayBuffer import scala.util.matching.Regex /** @@ -47,6 +46,7 @@ class CommentInterceptor extends EntranceInterceptor { case "sql" | "hql" => requestPersistTask.setExecutionCode(SQLCommentHelper.dealComment(requestPersistTask.getExecutionCode)) case "python" | "py" => requestPersistTask.setExecutionCode(PythonCommentHelper.dealComment(requestPersistTask.getExecutionCode)) case "scala" | "java" => requestPersistTask.setExecutionCode(ScalaCommentHelper.dealComment(requestPersistTask.getExecutionCode)) + case "sh" | "shell" => case _ => requestPersistTask.setExecutionCode(SQLCommentHelper.dealComment(requestPersistTask.getExecutionCode)) } requestPersistTask @@ -62,26 +62,18 @@ trait CommentHelper{ object SQLCommentHelper extends CommentHelper { override val commentPattern: Regex = """\s*--.+\s*""".r.unanchored private val comment = "(?ms)([\"']*['\"])|--.*?$|/\\*.*?\\*/" + private val logger:Logger = LoggerFactory.getLogger(getClass) override def dealComment(code: String): String = { - val p = Pattern.compile(comment) - val sql = p.matcher(code).replaceAll("$1") - sql -// val clearCode = new ArrayBuffer[String]() -//// code.split(";") foreach { -//// case commentPattern() => -//// case singleCode:String => clearCode += singleCode -//// case _ => -//// } -//// clearCode.mkString(";") -// code.split(";") foreach(singleLine =>{ -// val tempCode = new ArrayBuffer[String]() -// singleLine.split("\n") foreach (line => tempCode += -// line.substring(0, line.length - commentPattern.findFirstIn(line).getOrElse("").length)) -// clearCode += tempCode.mkString("\n") -// }) -// val resultCode = new ArrayBuffer[String]() -// clearCode.filter(StringUtils.isNotBlank).foreach(resultCode += _) -// resultCode.mkString(";") + try{ + val p = Pattern.compile(comment) + val sql = p.matcher(code).replaceAll("$1") + sql + }catch{ + case e:Exception => logger.warn("sql comment failed") + code + case t:Throwable => logger.warn("sql comment failed") + code + } } } @@ -89,16 +81,7 @@ object PythonCommentHelper extends CommentHelper{ override val commentPattern: Regex = """^\s*#.+\s*""".r.unanchored val pythonCommentPattern:String = "(?ms)([\"'](?:|[^'])*['\"])|#.*?$|/\\*.*?\\*/" override def dealComment(code: String): String = { - //val p = Pattern.compile(pythonCommentPattern) - //p.matcher(code).replaceAll("$1") code -// val clearCode = new ArrayBuffer[String]() -// code.split("\n") foreach { -// case commentPattern() => -// case singleCode:String => clearCode += singleCode -// case _ => -// } -// clearCode.mkString("\n") } } @@ -109,13 +92,6 @@ object ScalaCommentHelper extends CommentHelper{ override def dealComment(code: String): String = { val p = Pattern.compile(scalaCommentPattern) p.matcher(code).replaceAll("$1") -// val clearCode = new ArrayBuffer[String]() -// code.split("\n") foreach { -// case commentPattern() => -// case singleCode:String => clearCode += singleCode -// case _ => -// } -// clearCode.mkString("\n") } } diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CustomVariableUtils.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CustomVariableUtils.scala index 99b6e98631..7350f9308e 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CustomVariableUtils.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CustomVariableUtils.scala @@ -21,7 +21,6 @@ import java.text.SimpleDateFormat import java.util import java.util.{Calendar, Date} -//import com.sun.jdi.FloatValue import com.webank.wedatasphere.linkis.common.utils.Logging import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration import com.webank.wedatasphere.linkis.entrance.interceptor.exception.VarSubstitutionException @@ -31,13 +30,11 @@ import com.webank.wedatasphere.linkis.protocol.utils.TaskUtils import com.webank.wedatasphere.linkis.protocol.variable.{RequestQueryAppVariable, ResponseQueryVariable} import com.webank.wedatasphere.linkis.rpc.Sender import org.apache.commons.lang.StringUtils - -import scala.collection.mutable.ArrayBuffer -//import com.webank.wedatasphere.linkis.entrance.log.LogManager import org.apache.commons.lang.time.DateUtils import scala.collection.JavaConversions._ import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer import scala.util.control.Exception._ /** @@ -66,8 +63,8 @@ object CustomVariableUtils extends Logging { * 1. 从代码中得到用户定义的变量，进行替换 * 2. 如果1没有做，那么从args中得到用户定义的变量，进行替换 - * 3. 如果2没有做，从控制台中得到用户定义的变量，进行替换 - * + * 3. 如果2没有做，从CS中得到用户定义的变量，进行替换 + *3. 如果3没有做，从控制台中得到用户定义的变量，进行替换 * @param task : requestPersistTask * @return */ @@ -77,7 +74,7 @@ object CustomVariableUtils extends Logging { var codeType = SQL_TYPE runType match { case "hql" | "sql" | "jdbc" | "hive" => codeType = SQL_TYPE - case "python" => codeType = PY_TYPE + case "python" | "py" => codeType = PY_TYPE case "java" => codeType = JAVA_TYPE case "scala" => codeType = SCALA_TYPE case _ => return (false, code) @@ -86,106 +83,57 @@ object CustomVariableUtils extends Logging { var run_date:CustomDateType = null val nameAndType = mutable.Map[String, VariableType]() val nameAndValue: mutable.Map[String, String] = getCustomVar(code, codeType) - /* The first step is to replace the variable from code*/ - /*第一步来自code的变量替换*/ - nameAndValue.foreach { - case (name, value) => { - name match { - case RUN_DATE => { - if (value.length == 8) run_date = new CustomDateType(value, false) - else - throw VarSubstitutionException(20040, "please use correct date format,example:run_date=20170101") - } - case _ => /*if ((allCatch opt value.toLong).isDefined) { - nameAndType(name) = LongType(value.toLong) - } else*/ if ((allCatch opt value.toDouble).isDefined) { - nameAndType(name) = DoubleValue(value.toDouble) - } else { - nameAndType(name) = StringType(value) + + def putNameAndType(data: mutable.Map[String, String]): Unit = if (null != data) data foreach { + case (key, value) => key match { + case RUN_DATE => if (nameAndType.get(RUN_DATE).isEmpty) { + val run_date_str = value.asInstanceOf[String] + if (StringUtils.isNotEmpty(run_date_str)) { + run_date = new CustomDateType(run_date_str, false) + nameAndType(RUN_DATE) = DateType(run_date) } } - } - } - if (run_date != null){ - nameAndType(RUN_DATE) = DateType(run_date) - } - /* Perform the second step to replace the parameters passed in args*/ - /* 进行第二步，对args传进的参数进行替换*/ - task match { - case requestPersistTask:RequestPersistTask => TaskUtils.getVariableMap(requestPersistTask.getParams.map{case (k, v) => k -> v.asInstanceOf[Any]}) match { - case args:java.util.Map[String, Any] => - //Implicit conversion to scala map for subsequent operations(隐式转成scala的Map，方便后续操作) - val scalaArgs:mutable.Map[String, Any] = args - scalaArgs foreach { - case (key, value) => key match { - case RUN_DATE => if (nameAndType.get(RUN_DATE).isEmpty) { - val run_date_str = value.asInstanceOf[String] - if (StringUtils.isNotEmpty(run_date_str)){ - run_date = new CustomDateType(run_date_str, false) - nameAndType(RUN_DATE) = DateType(run_date) - } - } - case _ => if (nameAndType.get(key).isEmpty){ - val value_str = value.asInstanceOf[String] - if(StringUtils.isNotEmpty(value_str)){ - /*if ((allCatch opt value_str.toLong).isDefined) { - nameAndType(key) = LongType(value_str.toLong) - } else*/ if ((allCatch opt value_str.toDouble).isDefined) { - nameAndType(key) = DoubleValue(value_str.toDouble) - } else { - nameAndType(key) = StringType(value_str) - } - } - } + case _ => if (nameAndType.get(key).isEmpty && StringUtils.isNotEmpty(value)) { + if ((allCatch opt value.toDouble).isDefined) { + nameAndType(key) = DoubleValue(value.toDouble) + } else { + nameAndType(key) = StringType(value) } } - case _ => } + } + + //The first step is to replace the variable from code + //第一步来自code的变量替换 + putNameAndType(nameAndValue) + + task match { + case requestPersistTask:RequestPersistTask => + /* Perform the second step to replace the parameters passed in args*/ + /* 进行第二步，对args传进的参数进行替换*/ + val variableMap = TaskUtils.getVariableMap(requestPersistTask.getParams.asInstanceOf[util.Map[String, Any]]) + .map{case (k, v) => k -> v.asInstanceOf[String]} + putNameAndType(variableMap) case _ => } - /* Go to the third step and take the user's parameters to the cloud-publicservice module.*/ - /*进行第三步，向cloud-publicservice模块去拿用户的参数*/ + /* Go to the four step and take the user's parameters to the cloud-publicservice module.*/ + /*进行第四步，向cloud-publicservice模块去拿用户的参数*/ val sender = Sender.getSender(EntranceConfiguration.CLOUD_CONSOLE_VARIABLE_SPRING_APPLICATION_NAME.getValue) task match { case requestPersistTask:RequestPersistTask => val umUser:String = requestPersistTask.getUmUser val creator:String = requestPersistTask.getRequestApplicationName val runType:String = requestPersistTask.getRunType -// val requestQueryAppConfig:RequestQueryAppConfig = RequestQueryAppConfig(umUser, creator, runType) -// val responseQueryConfig = sender.ask(requestQueryAppConfig).asInstanceOf[ResponseQueryConfig] -// val keyAndValue = responseQueryConfig.getKeyAndValue val requestQueryAppVariable:RequestQueryAppVariable = RequestQueryAppVariable(umUser, creator, runType) val response:ResponseQueryVariable = sender.ask(requestQueryAppVariable).asInstanceOf[ResponseQueryVariable] val keyAndValue = response.getKeyAndValue val keyAndValueScala:mutable.Map[String, String] = keyAndValue - keyAndValueScala foreach { - case (key, value) => key match { - case RUN_DATE => if (nameAndType.get(RUN_DATE).isEmpty) { - val run_date_str = value.asInstanceOf[String] - if (StringUtils.isNotEmpty(run_date_str)){ - run_date = new CustomDateType(run_date_str, false) - nameAndType(RUN_DATE) = DateType(run_date) - } - } - case _ => if (nameAndType.get(key).isEmpty){ - val value_str = value.asInstanceOf[String] - if(StringUtils.isNotEmpty(value_str)){ - /*if ((allCatch opt value_str.toLong).isDefined) { - nameAndType(key) = LongType(value_str.toLong) - } else*/ if ((allCatch opt value_str.toDouble).isDefined) { - nameAndType(key) = DoubleValue(value_str.toDouble) - } else { - nameAndType(key) = StringType(value_str) - } - } - } - } - } + putNameAndType(keyAndValueScala) case _ => } /*The last step, if you have not set run_date, then it is the default */ /*最后一步，如果都没有设置run_date，那么就是默认*/ - if (nameAndType.get(RUN_DATE).isEmpty){ + if (nameAndType.get(RUN_DATE).isEmpty || null == run_date){ run_date = new CustomDateType(getYesterday(false), false) nameAndType(RUN_DATE) = DateType(new CustomDateType(run_date.toString, false)) } @@ -213,8 +161,8 @@ object CustomVariableUtils extends Logging { */ def parserVar(code: String, nameAndType: mutable.Map[String, VariableType]): String = { - val codeReg = "\\$\\{\\s*[A-Za-z][A-Za-z0-9_]*\\s*[\\+\\-\\*/]?\\s*[A-Za-z0-9_\\.]*\\s*\\}".r - val calReg = "(\\s*[A-Za-z][A-Za-z0-9_]*\\s*)([\\+\\-\\*/]?)(\\s*[A-Za-z0-9_\\.]*\\s*)".r + val codeReg = "\\$\\{\\s*[A-Za-z][A-Za-z0-9_\\.]*\\s*[\\+\\-\\*/]?\\s*[A-Za-z0-9_\\.]*\\s*\\}".r + val calReg = "(\\s*[A-Za-z][A-Za-z0-9_\\.]*\\s*)([\\+\\-\\*/]?)(\\s*[A-Za-z0-9_\\.]*\\s*)".r val parseCode = new StringBuilder val codes = codeReg.split(code) val expressionCache = mutable.HashSet[String]() @@ -369,11 +317,11 @@ object CustomVariableUtils extends Logging { val nameSet = res(0).split("@set") if (nameSet != null && nameSet.length == 2) { val name = nameSet(1).trim - if (nameAndValue.getOrElse(name, null) == null) { - nameAndValue(name) = res(1).trim - } else { - throw VarSubstitutionException(20043, s"$name is defined repeatedly") - } + //if (nameAndValue.getOrElse(name, null) == null) { + nameAndValue(name) = res(1).trim + // } else { + // throw VarSubstitutionException(20043, s"$name is defined repeatedly") + // } } } else { if (res.length > 2) { diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/Explain.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/Explain.scala index 00ac48a0b6..e6c54a95f4 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/Explain.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/Explain.scala @@ -21,7 +21,7 @@ import java.util.regex.Pattern import com.webank.wedatasphere.linkis.common.conf.CommonVars import com.webank.wedatasphere.linkis.common.exception.ErrorException import com.webank.wedatasphere.linkis.common.log.LogUtils -import com.webank.wedatasphere.linkis.common.utils.Logging +import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} import com.webank.wedatasphere.linkis.entrance.interceptor.exception.{PythonCodeCheckException, ScalaCodeCheckException} import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask import org.apache.commons.lang.StringUtils @@ -82,6 +82,8 @@ object SQLExplain extends Explain { val SQL_APPEND_LIMIT:String = " limit " + SQL_DEFAULT_LIMIT.getValue val DROP_TABLE_SQL = "\\s*drop\\s+table\\s+\\w+\\s*" val CREATE_DATABASE_SQL = "\\s*create\\s+database\\s+\\w+\\s*" + private val LINE_BREAK = "\n" + private val COMMENT_FLAG = "--" val SET_OWN_USER = "set owner user" private val LIMIT:String = "limit" private val LIMIT_UPPERCASE:String = "LIMIT" @@ -131,7 +133,9 @@ object SQLExplain extends Explain { if (StringUtils.isEmpty(code)) { return false } - code.trim.split("\\s+")(0).equalsIgnoreCase("select") + //如果一段sql是 --xxx回车select * from default.users，那么他也是select语句 + val realCode = cleanComment(code) + realCode.trim.split("\\s+")(0).equalsIgnoreCase("select") } def continueWhenError = false @@ -141,7 +145,24 @@ object SQLExplain extends Explain { return false } val realCode = cmd.trim - if (realCode.toLowerCase().contains(LIMIT)) false else true + //limit往往就是在sql语句中最后的，所以需要进行最后的判断 + val arr = realCode.split("\\s+") + val words = new ArrayBuffer[String]() + arr foreach { + w => w.split("\n") foreach (words += _) + } + val a = words.toArray + val length = a.length + val second_last = a(length - 2) + !"limit".equals(second_last.toLowerCase()) + } + + private def cleanComment(sql:String):String = { + val cleanSql = new StringBuilder + sql.trim.split(LINE_BREAK) foreach { + singleSql => if (!singleSql.trim().startsWith(COMMENT_FLAG)) cleanSql.append(singleSql).append(LINE_BREAK) + } + cleanSql.toString().trim } def isSelectOverLimit(cmd: String): Boolean = { diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/PythonCodeCheckInterceptor.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/PythonCodeCheckInterceptor.scala index 5a30c86104..5c4c95b76d 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/PythonCodeCheckInterceptor.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/PythonCodeCheckInterceptor.scala @@ -31,7 +31,7 @@ class PythonCodeCheckInterceptor extends EntranceInterceptor{ override def apply(task: Task, logAppender: java.lang.StringBuilder): Task = task match{ case requestPersistTask:RequestPersistTask => val error = new StringBuilder - requestPersistTask.getEngineType match { + requestPersistTask.getRunType match { case "python" | "pyspark" => Utils.tryThrow(PythonExplain.authPass(requestPersistTask.getExecutionCode, error)){ case PythonCodeCheckException(errCode,errDesc) => diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SQLLimitEntranceInterceptor.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SQLLimitEntranceInterceptor.scala index 3b6076d192..a0534d13de 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SQLLimitEntranceInterceptor.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SQLLimitEntranceInterceptor.scala @@ -17,7 +17,6 @@ package com.webank.wedatasphere.linkis.entrance.interceptor.impl import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor -import com.webank.wedatasphere.linkis.entrance.interceptor.exception.LimitCheckException import com.webank.wedatasphere.linkis.protocol.query.RequestPersistTask import com.webank.wedatasphere.linkis.protocol.task.Task diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/ScalaCodeInterceptor.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/ScalaCodeInterceptor.scala index dbf71be930..557504f46e 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/ScalaCodeInterceptor.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/ScalaCodeInterceptor.scala @@ -18,7 +18,6 @@ package com.webank.wedatasphere.linkis.entrance.interceptor.impl import java.lang -import com.webank.wedatasphere.linkis.common.exception.ErrorException import com.webank.wedatasphere.linkis.common.utils.Utils import com.webank.wedatasphere.linkis.entrance.interceptor.EntranceInterceptor import com.webank.wedatasphere.linkis.entrance.interceptor.exception.ScalaCodeCheckException diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SparkCodeCheckInterceptor.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SparkCodeCheckInterceptor.scala index 33b13edece..3a6beb5391 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SparkCodeCheckInterceptor.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/SparkCodeCheckInterceptor.scala @@ -33,8 +33,8 @@ class SparkCodeCheckInterceptor extends EntranceInterceptor{ override def apply(task: Task, logAppender: java.lang.StringBuilder): Task = { task match { case requestPersistTask: RequestPersistTask => - requestPersistTask.getExecuteApplicationName.toLowerCase() match { - case "spark" | "scala" => val stringBuilder:StringBuilder = new StringBuilder() + requestPersistTask.getRunType.toLowerCase() match { + case "scala" => val stringBuilder:StringBuilder = new StringBuilder() val isAuth = SparkExplain.authPass(requestPersistTask.getExecutionCode, stringBuilder) if (!isAuth){ throw CodeCheckException(20050, "spark code check failed") diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogWriter.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogWriter.scala index 93367297a8..3c15e1e820 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogWriter.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/CacheLogWriter.scala @@ -33,11 +33,12 @@ class CacheLogWriter(logPath:String, def getCache:Option[Cache] = Some(sharedCache) private def cache(msg:String): Unit = { - val removed = sharedCache.cachedLogs.add(msg) - if (removed != null){ - this synchronized{ + this synchronized { + val removed = sharedCache.cachedLogs.add(msg) + if (removed != null){ val logs = sharedCache.cachedLogs.toList val sb = new StringBuilder + sb.append(removed).append("\n") logs.filter(_ != null).foreach(log => sb.append(log).append("\n")) sharedCache.cachedLogs.fakeClear() super.write(sb.toString()) diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/ErrorCodeManager.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/ErrorCodeManager.scala index c9142f17f1..31f1a125c4 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/ErrorCodeManager.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/log/ErrorCodeManager.scala @@ -16,18 +16,6 @@ package com.webank.wedatasphere.linkis.entrance.log -import java.io.{BufferedReader, FileInputStream, InputStreamReader} -import java.util -import java.util.concurrent.TimeUnit - -import com.webank.wedatasphere.linkis.common.io.FsPath -import com.webank.wedatasphere.linkis.common.utils.Utils -import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration -import com.webank.wedatasphere.linkis.storage.FSFactory -import org.slf4j.{Logger, LoggerFactory} - -import scala.collection.mutable.ArrayBuffer - /** * Created by enjoyyin on 2018/9/4. */ diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/parser/CommonEntranceParser.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/parser/CommonEntranceParser.scala index 52a8a451c5..c1be9a5d8a 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/parser/CommonEntranceParser.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/parser/CommonEntranceParser.scala @@ -56,14 +56,13 @@ class CommonEntranceParser extends AbstractEntranceParser{ } val formatCode = params.get(TaskConstant.FORMATCODE).asInstanceOf[Boolean] var creator = params.get(TaskConstant.REQUESTAPPLICATIONNAME).asInstanceOf[String] - val scriptPath = params.get(TaskConstant.SCRIPTPATH).asInstanceOf[String] val source = params.getOrDefault(TaskConstant.SOURCE,new util.HashMap[String,String]()).asInstanceOf[util.Map[String,String]] val executeApplicationName = params.get(TaskConstant.EXECUTEAPPLICATIONNAME).asInstanceOf[String] if (StringUtils.isEmpty(creator)) creator = EntranceConfiguration.DEFAULT_REQUEST_APPLICATION_NAME.getValue if (StringUtils.isEmpty(executeApplicationName)) throw new EntranceIllegalParamException(20006, "param executeApplicationName can not be empty or null") /* When the execution type is IDE, executionCode and scriptPath cannot be empty at the same time*/ /*当执行类型为IDE的时候，executionCode和scriptPath不能同时为空*/ - if (EntranceConfiguration.DEFAULT_REQUEST_APPLICATION_NAME.getValue.equals(creator) && StringUtils.isEmpty(scriptPath) && + if (EntranceConfiguration.DEFAULT_REQUEST_APPLICATION_NAME.getValue.equals(creator) && StringUtils.isEmpty(source.get(TaskConstant.SCRIPTPATH)) && StringUtils.isEmpty(executionCode)) throw new EntranceIllegalParamException(20007, "param executionCode and scriptPath can not be empty at the same time") var runType:String = null @@ -74,20 +73,7 @@ class CommonEntranceParser extends AbstractEntranceParser{ if (formatCode) executionCode = format(executionCode) task.setExecutionCode(executionCode) } - if (source.isEmpty) source.put(TaskConstant.SCRIPTPATH,scriptPath) task.setSource(source) - if (StringUtils.isNotEmpty(scriptPath)) { - task.setScriptPath(scriptPath) -// val strings = StringUtils.split(scriptPath, ".") -// if (strings.length >= 2) { -// /*获得执行脚本文件的后缀名,如果不能从一个执行脚本的后缀名判断出类型，可能需要报错*/ -// //todo If you can't get it from the file suffix name(如果不能从文件后缀名得到) -// runType = strings(strings.length - 1) -// if (ParserUtils.getCorrespondingType(runType) != null) runType = ParserUtils.getCorrespondingType(runType) -// else logger.warn("未能找到相应的执行类型:" + runType) -// } -// else logger.warn("scriptPath:" + scriptPath + "没有后缀名, 无法判断任务是哪种类型") - } task.setEngineType(runType) //为了兼容代码，让engineType和runType都有同一个属性 task.setRunType(runType) diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/EntranceResultSetEngine.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/EntranceResultSetEngine.scala index 32a0990459..13bd941e8f 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/EntranceResultSetEngine.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/EntranceResultSetEngine.scala @@ -21,8 +21,6 @@ import com.webank.wedatasphere.linkis.entrance.execute.StorePathExecuteRequest import com.webank.wedatasphere.linkis.entrance.job.EntranceExecutionJob import com.webank.wedatasphere.linkis.scheduler.executer.{AliasOutputExecuteResponse, OutputExecuteResponse} import com.webank.wedatasphere.linkis.scheduler.queue.Job -import com.webank.wedatasphere.linkis.storage.FSFactory -import com.webank.wedatasphere.linkis.storage.fs.FileSystem import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetWriter} import com.webank.wedatasphere.linkis.storage.utils.{FileSystemUtils, StorageUtils} import org.apache.commons.io.IOUtils diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/PersistenceEngine.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/PersistenceEngine.scala index 29b760beaf..a436caa32b 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/PersistenceEngine.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/persistence/PersistenceEngine.scala @@ -19,7 +19,6 @@ package com.webank.wedatasphere.linkis.entrance.persistence import java.io.{Closeable, Flushable} import com.webank.wedatasphere.linkis.common.exception.ErrorException -import com.webank.wedatasphere.linkis.entrance.exception.EntranceIllegalParamException import com.webank.wedatasphere.linkis.protocol.task.Task /** diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulRemote.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulRemote.scala index 47e6fd5261..620af1c46a 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulRemote.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/restful/EntranceRestfulRemote.scala @@ -19,6 +19,7 @@ package com.webank.wedatasphere.linkis.entrance.restful import java.util import javax.servlet.http.HttpServletRequest +import javax.ws.rs.QueryParam import javax.ws.rs.core.{Context, Response} import org.springframework.web.bind.annotation.{PathVariable, RequestBody, RequestMapping, RequestMethod} @@ -34,7 +35,7 @@ trait EntranceRestfulRemote { // def get(@PathVariable("id") id: String): Response @RequestMapping(value = Array("/entrance/{id}/status"), method = Array(RequestMethod.GET)) - def status(@PathVariable("id") id: String): Response + def status(@PathVariable("id") id: String, @QueryParam("taskID") taskID:String): Response @RequestMapping(value = Array("/entrance/{id}/progress"), method = Array(RequestMethod.POST)) @@ -48,7 +49,7 @@ trait EntranceRestfulRemote { def log(@Context req: HttpServletRequest, @PathVariable("id") id: String): Response @RequestMapping(value = Array("/entrance/{id}/kill"), method = Array(RequestMethod.POST)) - def kill(@PathVariable("id") id: String): Response + def kill(@PathVariable("id") id: String, @QueryParam("taskID") taskID:Long): Response @RequestMapping(value = Array("/entrance/{id}/pause"), method = Array(RequestMethod.POST)) def pause(@PathVariable("id") id: String): Response diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceGroupFactory.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceGroupFactory.scala index d7fdf6b121..7913bc375f 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceGroupFactory.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceGroupFactory.scala @@ -16,11 +16,10 @@ package com.webank.wedatasphere.linkis.entrance.scheduler -import com.webank.wedatasphere.linkis.common.conf.CommonVars import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration import com.webank.wedatasphere.linkis.entrance.execute.EntranceJob import com.webank.wedatasphere.linkis.entrance.persistence.HaPersistenceTask -import com.webank.wedatasphere.linkis.protocol.config.{RequestQueryAppConfig, RequestQueryGlobalConfig, ResponseQueryConfig} +import com.webank.wedatasphere.linkis.protocol.config.{RequestQueryAppConfig, ResponseQueryConfig} import com.webank.wedatasphere.linkis.rpc.Sender import com.webank.wedatasphere.linkis.scheduler.queue.parallelqueue.ParallelGroup import com.webank.wedatasphere.linkis.scheduler.queue.{Group, GroupFactory, SchedulerEvent} diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceSchedulerContext.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceSchedulerContext.scala index 0858d85bdb..2d02502895 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceSchedulerContext.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceSchedulerContext.scala @@ -21,8 +21,6 @@ import com.webank.wedatasphere.linkis.scheduler.SchedulerContext import com.webank.wedatasphere.linkis.scheduler.event.{ScheduleEvent, SchedulerEventListener} import com.webank.wedatasphere.linkis.scheduler.executer.ExecutorManager import com.webank.wedatasphere.linkis.scheduler.queue.{ConsumerManager, GroupFactory} -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Component /** * Created by enjoyyin on 2019/1/22. diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/utils/JobHistoryHelper.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/utils/JobHistoryHelper.scala new file mode 100644 index 0000000000..f330fcd4bf --- /dev/null +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/utils/JobHistoryHelper.scala @@ -0,0 +1,87 @@ +package com.webank.wedatasphere.linkis.entrance.utils + +import java.util + +import com.google.gson.Gson +import com.webank.wedatasphere.linkis.common.exception.ErrorException +import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} +import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration +import com.webank.wedatasphere.linkis.entrance.exception.JobHistoryFailedException +import com.webank.wedatasphere.linkis.protocol.query.cache.{CacheTaskResult, RequestReadCache} +import com.webank.wedatasphere.linkis.protocol.query.{RequestPersistTask, RequestQueryTask, RequestUpdateTask, ResponsePersist} +import com.webank.wedatasphere.linkis.rpc.Sender +import com.webank.wedatasphere.linkis.scheduler.queue.SchedulerEventState + +/** + * created by cooperyang on 2020/1/2 + * Description: + */ +object JobHistoryHelper extends Logging{ + + private val sender = Sender.getSender(EntranceConfiguration.QUERY_PERSISTENCE_SPRING_APPLICATION_NAME.getValue) + + private val SUCCESS_FLAG = 0 + private val TASK_MAP_KEY = "task" + + def getCache(executionCode: String, engineType: String, user: String, readCacheBefore: Long): CacheTaskResult ={ + val requestReadCache = new RequestReadCache(executionCode, engineType, user, readCacheBefore) + sender.ask(requestReadCache) match { + case c: CacheTaskResult => c + case _ => null + } + } + + def getStatusByTaskID(taskID:Long):String = { + val task = getTaskByTaskID(taskID) + if (task == null) SchedulerEventState.Cancelled.toString else task.getStatus + } + + /** + * 对于一个在内存中找不到这个任务的话，可以直接干掉 + * @param taskID + */ + def forceKill(taskID:Long):Unit = { + val requestUpdateTask = new RequestUpdateTask + requestUpdateTask.setTaskID(taskID) + requestUpdateTask.setStatus("Cancelled") + sender.ask(requestUpdateTask) + } + + private def getTaskByTaskID(taskID:Long):RequestPersistTask = { + val requestQueryTask = new RequestQueryTask() + requestQueryTask.setTaskID(taskID) + requestQueryTask.setSource(null) + val task = Utils.tryCatch{ + val taskResponse = sender.ask(requestQueryTask) + taskResponse match { + case responsePersist:ResponsePersist => val status = responsePersist.getStatus + if (status != SUCCESS_FLAG){ + logger.error(s"query from jobHistory status failed, status is $status") + throw JobHistoryFailedException("query from jobHistory status failed") + }else{ + val data = responsePersist.getData + data.get(TASK_MAP_KEY) match { + case tasks:util.List[util.Map[String, Object]] => tasks.get(0) match { + case map:util.Map[String, Object] => val gson = new Gson() + val json = gson.toJson(map) + val requestPersistTask = gson.fromJson(json, classOf[RequestPersistTask]) + requestPersistTask + case _ => throw JobHistoryFailedException(s"query from jobhistory not a correct RequestPersistTask type taskId is $taskID") + } + case _ => throw JobHistoryFailedException(s"query from jobhistory not a correct List type taskId is $taskID") + } + } + case _ => logger.error("get query response incorrectly") + throw JobHistoryFailedException("get query response incorrectly") + } + }{ + case errorException:ErrorException => throw errorException + case e:Exception => val e1 = JobHistoryFailedException(s"query taskId $taskID error") + e1.initCause(e) + throw e + } + task + } + + +} diff --git a/ujes/entranceclient/pom.xml b/ujes/entranceclient/pom.xml index 0657ff0d72..8111b2e3d8 100644 --- a/ujes/entranceclient/pom.xml +++ b/ujes/entranceclient/pom.xml @@ -40,6 +40,12 @@ + + com.webank.wedatasphere.linkis + linkis-cloudRPC + ${linkis.version} + provided + diff --git a/ujes/jdbc/pom.xml b/ujes/jdbc/pom.xml index ebbdfee216..80d779088e 100644 --- a/ujes/jdbc/pom.xml +++ b/ujes/jdbc/pom.xml @@ -1,5 +1,18 @@ + + @@ -15,7 +28,7 @@ com.webank.wedatasphere.linkis linkis-ujes-client - 0.9.4 + ${linkis.version} From 75c56f159e21dc11de67f3ffb8bfd540a43e42e2 Mon Sep 17 00:00:00 2001 From: yangzhiyue <904666286@qq.com> Date: Wed, 3 Jun 2020 22:46:22 +0800 Subject: [PATCH 8/8] adjust gateway module --- .../cs/listener/test/TestListenerManager.java | 1 - .../config/GatewaySpringConfiguration.scala | 2 +- .../linkis/gateway/http/GatewayContext.scala | 9 ++ .../gateway/security/GatewaySSOUtils.scala | 2 +- .../linkis/gateway/security/UserRestful.scala | 4 +- gateway/gateway-httpclient-support/pom.xml | 2 +- .../linkis/httpclient/dws/DWSHttpClient.scala | 21 +-- .../StaticAuthenticationStrategy.scala | 4 +- .../TokenAuthenticationStrategy.scala | 7 +- .../dws/config/DWSClientConfigBuilder.scala | 2 +- .../dws/discovery/DWSGatewayDiscovery.scala | 4 +- .../response/DWSAuthenticationResult.scala | 32 +++-- .../dws/response/DWSHeartbeatResult.scala | 25 +++- gateway/gateway-ujes-support/pom.xml | 6 + .../parser/EntranceRequestGatewayParser.scala | 3 +- .../ujes/route/HaContextGatewayRouter.scala | 124 ++++++++++++++++++ .../contextservice/ContextIdParserImpl.scala | 52 ++++++++ .../SpringCloudGatewayConfiguration.scala | 1 - 18 files changed, 263 insertions(+), 38 deletions(-) create mode 100644 gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/route/HaContextGatewayRouter.scala create mode 100644 gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/route/contextservice/ContextIdParserImpl.scala diff --git a/contextservice/cs-listener/src/test/java/com/webank/wedatasphere/linkis/cs/listener/test/TestListenerManager.java b/contextservice/cs-listener/src/test/java/com/webank/wedatasphere/linkis/cs/listener/test/TestListenerManager.java index 3835c672d8..c86d8a4c85 100644 --- a/contextservice/cs-listener/src/test/java/com/webank/wedatasphere/linkis/cs/listener/test/TestListenerManager.java +++ b/contextservice/cs-listener/src/test/java/com/webank/wedatasphere/linkis/cs/listener/test/TestListenerManager.java @@ -7,7 +7,6 @@ import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextKeyValue; import com.webank.wedatasphere.linkis.cs.common.entity.source.ContextValue; import com.webank.wedatasphere.linkis.cs.listener.ListenerBus.ContextAsyncListenerBus; -import com.webank.wedatasphere.linkis.cs.listener.callback.imp.ClientSource; import com.webank.wedatasphere.linkis.cs.listener.callback.imp.ContextKeyValueBean; import com.webank.wedatasphere.linkis.cs.listener.callback.imp.DefaultContextIDCallbackEngine; import com.webank.wedatasphere.linkis.cs.listener.callback.imp.DefaultContextKeyCallbackEngine; diff --git a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/config/GatewaySpringConfiguration.scala b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/config/GatewaySpringConfiguration.scala index 67eab5ff64..f8ef9a71e9 100644 --- a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/config/GatewaySpringConfiguration.scala +++ b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/config/GatewaySpringConfiguration.scala @@ -16,7 +16,7 @@ package com.webank.wedatasphere.linkis.gateway.config -import com.webank.wedatasphere.linkis.gateway.security.{LDAPUserRestful, SecurityHook, SecurityFilter, UserRestful} +import com.webank.wedatasphere.linkis.gateway.security.{LDAPUserRestful, SecurityFilter, SecurityHook, UserRestful} import javax.annotation.PostConstruct import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean diff --git a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/http/GatewayContext.scala b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/http/GatewayContext.scala index 87906ee5ee..092ece9e1e 100644 --- a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/http/GatewayContext.scala +++ b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/http/GatewayContext.scala @@ -16,6 +16,10 @@ package com.webank.wedatasphere.linkis.gateway.http +import java.util + +import com.webank.wedatasphere.linkis.server.JMap + /** * created by cooperyang on 2019/1/9. */ @@ -32,6 +36,7 @@ trait GatewayContext { def setGatewayRoute(gatewayRoute: GatewayRoute): Unit def getGatewayRoute: GatewayRoute + def getParams: JMap[String, String] } class BaseGatewayContext extends GatewayContext { private var request: GatewayHttpRequest = _ @@ -39,6 +44,8 @@ class BaseGatewayContext extends GatewayContext { private var webSocketRequest: Boolean = false private var gatewayRoute: GatewayRoute = _ + private val props: JMap[String, String] = new util.HashMap[String, String]() + override def getRequest: GatewayHttpRequest = request override def setRequest(request: GatewayHttpRequest): Unit = this.request = request @@ -54,4 +61,6 @@ class BaseGatewayContext extends GatewayContext { override def setGatewayRoute(gatewayRoute: GatewayRoute): Unit = this.gatewayRoute = gatewayRoute override def getGatewayRoute: GatewayRoute = gatewayRoute + + override def getParams: JMap[String, String] = this.props } \ No newline at end of file diff --git a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/GatewaySSOUtils.scala b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/GatewaySSOUtils.scala index 2d459d5228..08dd795913 100644 --- a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/GatewaySSOUtils.scala +++ b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/GatewaySSOUtils.scala @@ -19,8 +19,8 @@ package com.webank.wedatasphere.linkis.gateway.security import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} import com.webank.wedatasphere.linkis.gateway.http.{GatewayContext, GatewayHttpRequest} import com.webank.wedatasphere.linkis.server.exception.LoginExpireException -import com.webank.wedatasphere.linkis.server.security.{SSOUtils, ServerSSOUtils} import com.webank.wedatasphere.linkis.server.security.SecurityFilter._ +import com.webank.wedatasphere.linkis.server.security.{SSOUtils, ServerSSOUtils} import javax.servlet.http.Cookie import scala.collection.JavaConversions._ diff --git a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/UserRestful.scala b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/UserRestful.scala index 8678f531ef..d01b9dbf07 100644 --- a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/UserRestful.scala +++ b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/UserRestful.scala @@ -16,9 +16,7 @@ package com.webank.wedatasphere.linkis.gateway.security -import java.util.Random - -import com.google.gson.{Gson, JsonObject} +import com.google.gson.Gson import com.webank.wedatasphere.linkis.common.utils.{Logging, RSAUtils, Utils} import com.webank.wedatasphere.linkis.gateway.config.GatewayConfiguration import com.webank.wedatasphere.linkis.gateway.http.GatewayContext diff --git a/gateway/gateway-httpclient-support/pom.xml b/gateway/gateway-httpclient-support/pom.xml index dce31146f9..c0daf49b0e 100644 --- a/gateway/gateway-httpclient-support/pom.xml +++ b/gateway/gateway-httpclient-support/pom.xml @@ -42,7 +42,7 @@ commons-beanutils commons-beanutils - 1.7.0 + 1.9.4 org.reflections diff --git a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/DWSHttpClient.scala b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/DWSHttpClient.scala index 6f9c481e4f..d36005c115 100644 --- a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/DWSHttpClient.scala +++ b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/DWSHttpClient.scala @@ -24,7 +24,6 @@ import java.text.SimpleDateFormat import java.util import com.fasterxml.jackson.databind.ObjectMapper -import com.ning.http.client.Response import com.webank.wedatasphere.linkis.common.io.{Fs, FsPath} import com.webank.wedatasphere.linkis.httpclient.AbstractHttpClient import com.webank.wedatasphere.linkis.httpclient.discovery.Discovery @@ -37,6 +36,7 @@ import com.webank.wedatasphere.linkis.httpclient.response.{HttpResult, ListResul import com.webank.wedatasphere.linkis.storage.FSFactory import org.apache.commons.beanutils.BeanUtils import org.apache.commons.lang.ClassUtils +import org.apache.http.HttpResponse import scala.collection.JavaConversions.mapAsJavaMap @@ -44,7 +44,7 @@ import scala.collection.JavaConversions.mapAsJavaMap * created by cooperyang on 2019/5/20. */ class DWSHttpClient(clientConfig: DWSClientConfig, clientName: String) - extends AbstractHttpClient(clientConfig, clientName) { + extends AbstractHttpClient(clientConfig, clientName) { override protected def createDiscovery(): Discovery = new DWSGatewayDiscovery @@ -57,21 +57,25 @@ class DWSHttpClient(clientConfig: DWSClientConfig, clientName: String) requestAction } - override protected def httpResponseToResult(response: Response, requestAction: HttpAction): Option[Result] = { - val url = requestAction.getURL + override protected def httpResponseToResult(response: HttpResponse, requestAction: HttpAction, responseBody: String): Option[Result] = { + var entity = response.getEntity + val statusCode: Int = response.getStatusLine.getStatusCode + val url: String = requestAction.getURL + val contentType: String = entity.getContentType.getValue DWSHttpMessageFactory.getDWSHttpMessageResult(url).map { case DWSHttpMessageResultInfo(_, clazz) => clazz match { case c if ClassUtils.isAssignable(c, classOf[DWSResult]) => val dwsResult = clazz.getConstructor().newInstance().asInstanceOf[DWSResult] - dwsResult.set(response.getResponseBody, response.getStatusCode, response.getUri.toString, response.getContentType) + dwsResult.set(responseBody, statusCode, url, contentType) BeanUtils.populate(dwsResult, dwsResult.getData) return Some(dwsResult) case _ => } + def transfer(value: Result, map: Map[String, Object]): Unit = { value match { case httpResult: HttpResult => - httpResult.set(response.getResponseBody, response.getStatusCode, response.getUri.toString, response.getContentType) + httpResult.set(responseBody, statusCode, url, contentType) case _ => } val javaMap = mapAsJavaMap(map) @@ -89,17 +93,18 @@ class DWSHttpClient(clientConfig: DWSClientConfig, clientName: String) transfer(value, map) value }.toArray - new ListResult(response.getResponseBody, results) + new ListResult(responseBody, results) } }.orElse(nonDWSResponseToResult(response, requestAction)) } - protected def nonDWSResponseToResult(response: Response, requestAction: HttpAction): Option[Result] = None + protected def nonDWSResponseToResult(response: HttpResponse, requestAction: HttpAction): Option[Result] = None protected def fillResultFields(responseMap: util.Map[String, Object], value: Result): Unit = {} //TODO Consistent with workspace, plus expiration time(与workspace保持一致，加上过期时间) override protected def getFsByUser(user: String, path: FsPath): Fs = FSFactory.getFsByProxyUser(path, user) + } object DWSHttpClient { val jacksonJson = new ObjectMapper().setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ")) diff --git a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/authentication/StaticAuthenticationStrategy.scala b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/authentication/StaticAuthenticationStrategy.scala index ab967b9558..129a2a0f77 100644 --- a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/authentication/StaticAuthenticationStrategy.scala +++ b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/authentication/StaticAuthenticationStrategy.scala @@ -16,7 +16,6 @@ package com.webank.wedatasphere.linkis.httpclient.dws.authentication -import com.ning.http.client.Response import com.webank.wedatasphere.linkis.common.utils.ByteTimeUtils import com.webank.wedatasphere.linkis.httpclient.authentication.{AbstractAuthenticationStrategy, AuthenticationAction, AuthenticationResult} import com.webank.wedatasphere.linkis.httpclient.dws.exception.AuthenticationFailedException @@ -24,6 +23,7 @@ import com.webank.wedatasphere.linkis.httpclient.dws.request.DWSAuthenticationAc import com.webank.wedatasphere.linkis.httpclient.dws.response.DWSAuthenticationResult import com.webank.wedatasphere.linkis.httpclient.request.{Action, UserAction, UserPwdAction} import org.apache.commons.lang.StringUtils +import org.apache.http.HttpResponse /** * created by cooperyang on 2019/5/22. @@ -51,7 +51,7 @@ class StaticAuthenticationStrategy(override protected val sessionMaxAliveTime: L action } - override def getAuthenticationResult(response: Response, requestAction: AuthenticationAction): AuthenticationResult = { + override def getAuthenticationResult(response: HttpResponse, requestAction: AuthenticationAction): AuthenticationResult = { new DWSAuthenticationResult(response, requestAction.serverUrl) } } \ No newline at end of file diff --git a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/authentication/TokenAuthenticationStrategy.scala b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/authentication/TokenAuthenticationStrategy.scala index ac74199fd0..061bcc27f0 100644 --- a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/authentication/TokenAuthenticationStrategy.scala +++ b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/authentication/TokenAuthenticationStrategy.scala @@ -18,11 +18,11 @@ package com.webank.wedatasphere.linkis.httpclient.dws.authentication import java.util -import com.ning.http.client.Response -import com.ning.http.client.cookie.Cookie import com.webank.wedatasphere.linkis.httpclient.authentication._ import com.webank.wedatasphere.linkis.httpclient.dws.exception.AuthenticationFailedException import com.webank.wedatasphere.linkis.httpclient.request.{Action, UserAction} +import org.apache.http.HttpResponse +import org.apache.http.cookie.Cookie /** * Created by enjoyyin on 2019/10/4. @@ -35,6 +35,7 @@ class TokenAuthenticationStrategy(override protected val sessionMaxAliveTime: Lo case _: AuthenticationAction => null case action: UserAction => new HttpAuthentication { import TokenAuthenticationStrategy._ + import scala.collection.JavaConversions._ override def authToCookies: Array[Cookie] = Array.empty @@ -51,7 +52,7 @@ class TokenAuthenticationStrategy(override protected val sessionMaxAliveTime: Lo override protected def getAuthenticationAction(requestAction: Action, serverUrl: String): AuthenticationAction = null - override def getAuthenticationResult(response: Response, requestAction: AuthenticationAction): AuthenticationResult = null + override def getAuthenticationResult(response: HttpResponse, requestAction: AuthenticationAction): AuthenticationResult = null } object TokenAuthenticationStrategy { diff --git a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/config/DWSClientConfigBuilder.scala b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/config/DWSClientConfigBuilder.scala index 51481c7108..2498b8e522 100644 --- a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/config/DWSClientConfigBuilder.scala +++ b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/config/DWSClientConfigBuilder.scala @@ -16,7 +16,7 @@ package com.webank.wedatasphere.linkis.httpclient.dws.config -import com.webank.wedatasphere.linkis.httpclient.config.{ClientConfig, ClientConfigBuilder} +import com.webank.wedatasphere.linkis.httpclient.config.ClientConfigBuilder import com.webank.wedatasphere.linkis.httpclient.dws.exception.UnknownVersionException import org.apache.commons.lang.StringUtils diff --git a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/discovery/DWSGatewayDiscovery.scala b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/discovery/DWSGatewayDiscovery.scala index 2dc2992401..bb77b5beca 100644 --- a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/discovery/DWSGatewayDiscovery.scala +++ b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/discovery/DWSGatewayDiscovery.scala @@ -20,10 +20,10 @@ package com.webank.wedatasphere.linkis.httpclient.dws.discovery -import com.ning.http.client.Response import com.webank.wedatasphere.linkis.httpclient.discovery.{AbstractDiscovery, HeartbeatAction, HeartbeatResult} import com.webank.wedatasphere.linkis.httpclient.dws.request.DWSHeartbeatAction import com.webank.wedatasphere.linkis.httpclient.dws.response.DWSHeartbeatResult +import org.apache.http.HttpResponse import scala.util.Random @@ -43,7 +43,7 @@ class DWSGatewayDiscovery extends AbstractDiscovery { override protected def getHeartbeatAction(serverUrl: String): HeartbeatAction = new DWSHeartbeatAction(serverUrl) - override def getHeartbeatResult(response: Response, requestAction: HeartbeatAction): HeartbeatResult = requestAction match { + override def getHeartbeatResult(response: HttpResponse, requestAction: HeartbeatAction): HeartbeatResult = requestAction match { case h: DWSHeartbeatAction => new DWSHeartbeatResult(response, h.serverUrl) } } diff --git a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/response/DWSAuthenticationResult.scala b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/response/DWSAuthenticationResult.scala index de939f884a..bc2c479b0c 100644 --- a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/response/DWSAuthenticationResult.scala +++ b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/response/DWSAuthenticationResult.scala @@ -18,22 +18,37 @@ package com.webank.wedatasphere.linkis.httpclient.dws.response import java.util -import com.ning.http.client.Response -import com.ning.http.client.cookie.Cookie import com.webank.wedatasphere.linkis.httpclient.authentication.{Authentication, AuthenticationResult, HttpAuthentication} import com.webank.wedatasphere.linkis.httpclient.exception.HttpMessageParseException - -import scala.collection.JavaConversions +import org.apache.http.HttpResponse +import org.apache.http.cookie.Cookie +import org.apache.http.util.EntityUtils /** * created by cooperyang on 2019/5/22. */ -class DWSAuthenticationResult(response: Response, serverUrl: String) extends AuthenticationResult with DWSResult { +class DWSAuthenticationResult(response: HttpResponse, serverUrl: String) extends AuthenticationResult with DWSResult { + + setResponse() + + private def setResponse(): Unit = { + val entity = response.getEntity + val responseBody: String = if (entity != null) { + EntityUtils.toString(entity, "UTF-8") + } else { + null + } + val statusCode: Int = response.getStatusLine.getStatusCode + val url: String = serverUrl + val contentType: String = entity.getContentType.getValue + set(responseBody, statusCode, url, contentType) + } - set(response.getResponseBody, response.getStatusCode, response.getUri.toString, response.getContentType) - override def getAuthentication: Authentication = if(getStatus == 0) new HttpAuthentication { + + override def getAuthentication: Authentication = if (getStatus == 0) new HttpAuthentication { private var lastAccessTime: Long = System.currentTimeMillis - override def authToCookies: Array[Cookie] = JavaConversions.asScalaBuffer(response.getCookies).toArray + + override def authToCookies: Array[Cookie] = Array.empty override def authToHeaders: util.Map[String, String] = new util.HashMap[String, String]() @@ -44,4 +59,5 @@ class DWSAuthenticationResult(response: Response, serverUrl: String) extends Aut override def updateLastAccessTime(): Unit = lastAccessTime = System.currentTimeMillis } else throw new HttpMessageParseException(s"login to gateway $serverUrl failed! Reason: " + getMessage) + } diff --git a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/response/DWSHeartbeatResult.scala b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/response/DWSHeartbeatResult.scala index 3697e33cd6..4030a5a05e 100644 --- a/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/response/DWSHeartbeatResult.scala +++ b/gateway/gateway-httpclient-support/src/main/scala/com/webank/wedatasphere/linkis/httpclient/dws/response/DWSHeartbeatResult.scala @@ -18,16 +18,33 @@ package com.webank.wedatasphere.linkis.httpclient.dws.response import java.util -import com.ning.http.client.Response import com.webank.wedatasphere.linkis.httpclient.discovery.HeartbeatResult +import org.apache.http.HttpResponse +import org.apache.http.util.EntityUtils /** * created by cooperyang on 2019/5/22. */ -class DWSHeartbeatResult(response: Response, serverUrl: String) extends HeartbeatResult with DWSResult { +class DWSHeartbeatResult(response: HttpResponse, serverUrl: String) extends HeartbeatResult with DWSResult { - set(response.getResponseBody, response.getStatusCode, response.getUri.toString, response.getContentType) - if(getStatus != 0) warn(s"heartbeat to gateway $serverUrl failed! message: $getMessage.") + + setResponse() + + private def setResponse(): Unit = { + val entity = response.getEntity + val responseBody: String = if (entity != null) { + EntityUtils.toString(entity, "UTF-8") + } else { + null + } + val statusCode: Int = response.getStatusLine.getStatusCode + val url: String = serverUrl + val contentType: String = entity.getContentType.getValue + set(responseBody, statusCode, url, contentType) + } + + + if (getStatus != 0) warn(s"heartbeat to gateway $serverUrl failed! message: $getMessage.") override val isHealthy: Boolean = getData.get("isHealthy") match { case b: java.lang.Boolean => b case s if s != null => s.toString.toBoolean diff --git a/gateway/gateway-ujes-support/pom.xml b/gateway/gateway-ujes-support/pom.xml index 9a44a17cac..bc041956e6 100644 --- a/gateway/gateway-ujes-support/pom.xml +++ b/gateway/gateway-ujes-support/pom.xml @@ -51,6 +51,12 @@ 2.1 + + com.webank.wedatasphere.linkis + linkis-cs-common + ${linkis.version} + + diff --git a/gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala b/gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala index 348934e1c8..4775203889 100644 --- a/gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala +++ b/gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/parser/EntranceRequestGatewayParser.scala @@ -19,12 +19,11 @@ package com.webank.wedatasphere.linkis.gateway.ujes.parser import com.webank.wedatasphere.linkis.common.ServiceInstance import com.webank.wedatasphere.linkis.gateway.http.GatewayContext import com.webank.wedatasphere.linkis.gateway.parser.AbstractGatewayParser +import com.webank.wedatasphere.linkis.gateway.ujes.parser.EntranceExecutionGatewayParser._ import com.webank.wedatasphere.linkis.protocol.constants.TaskConstant import com.webank.wedatasphere.linkis.protocol.utils.ZuulEntranceUtils import org.springframework.stereotype.Component -import EntranceExecutionGatewayParser._ - /** * created by cooperyang on 2019/5/15. */ diff --git a/gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/route/HaContextGatewayRouter.scala b/gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/route/HaContextGatewayRouter.scala new file mode 100644 index 0000000000..39ddbe9bfa --- /dev/null +++ b/gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/route/HaContextGatewayRouter.scala @@ -0,0 +1,124 @@ +package com.webank.wedatasphere.linkis.gateway.ujes.route + +import java.util + +import com.webank.wedatasphere.linkis.common.ServiceInstance +import com.webank.wedatasphere.linkis.cs.common.entity.source.{ContextID, ContextIDParser} +import com.webank.wedatasphere.linkis.cs.common.protocol.ContextHTTPConstant +import com.webank.wedatasphere.linkis.cs.common.serialize.helper.ContextSerializationHelper +import com.webank.wedatasphere.linkis.gateway.http.GatewayContext +import com.webank.wedatasphere.linkis.gateway.route.AbstractGatewayRouter +import com.webank.wedatasphere.linkis.gateway.springcloud.SpringCloudGatewayConfiguration.{API_URL_PREFIX, normalPath} +import com.webank.wedatasphere.linkis.rpc.interceptor.ServiceInstanceUtils +import org.apache.commons.lang.StringUtils +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + +import scala.collection.JavaConversions._ +import scala.util.Random +import scala.util.matching.Regex + +/** + * created by cooperyang on 2020/2/19 + * Description: + */ +@Component +class HaContextGatewayRouter extends AbstractGatewayRouter{ + + @Autowired + private var contextIDParser: ContextIDParser = _ + private val serializationHelper = ContextSerializationHelper.getInstance(); + + override def route(gatewayContext: GatewayContext): ServiceInstance = { + + if (gatewayContext.getGatewayRoute.getRequestURI.contains("contextservice")){ + val params: util.HashMap[String, String] = gatewayContext.getGatewayRoute.getParams + if (!gatewayContext.getRequest.getQueryParams.isEmpty) { + for ((k, vArr) <- gatewayContext.getRequest.getQueryParams) { + if (vArr.nonEmpty) { + params.putIfAbsent(k, vArr.head) + } + } + } + if (gatewayContext.getRequest.getHeaders.containsKey(ContextHTTPConstant.CONTEXT_ID_STR)) { + params.putIfAbsent(ContextHTTPConstant.CONTEXT_ID_STR, gatewayContext.getRequest.getHeaders.get(ContextHTTPConstant.CONTEXT_ID_STR)(0)) + } + if (null == params || params.isEmpty) { + dealContextCreate(gatewayContext) + } else { + var contextId : String = null + for ((key, value) <- params) { + if (key.equalsIgnoreCase(ContextHTTPConstant.CONTEXT_ID_STR)) { + contextId = value + } + } + if (StringUtils.isNotBlank(contextId)) { + dealContextAccess(contextId.toString, gatewayContext) + } else { + dealContextCreate(gatewayContext) + } + } + }else{ + null + } + } + + def dealContextCreate(gatewayContext:GatewayContext):ServiceInstance = { + val serviceId = findService(HaContextGatewayRouter.CONTEXT_SERVICE_STR, list => { + val services = list.filter(_.contains(HaContextGatewayRouter.CONTEXT_SERVICE_STR)) + services.headOption + }) + val serviceInstances = ServiceInstanceUtils.getRPCServerLoader.getServiceInstances(serviceId.orNull) + if (serviceInstances.size > 0) { + val index = new Random().nextInt(serviceInstances.size) + serviceInstances(index) + } else { + logger.error(s"No valid instance for service : " + serviceId.orNull) + null + } + } + + def dealContextAccess(contextIdStr:String, gatewayContext: GatewayContext):ServiceInstance = { + val contextId : String = { + var tmpId : String = null + if (serializationHelper.accepts(contextIdStr)) { + val contextID : ContextID = serializationHelper.deserialize(contextIdStr).asInstanceOf[ContextID] + if (null != contextID) { + tmpId = contextID.getContextId + } else { + error(s"Deserializate contextID null. contextIDStr : " + contextIdStr) + } + } else { + error(s"ContxtIDStr cannot be deserialized. contextIDStr : " + contextIdStr) + } + if (null == tmpId) { + contextIdStr + } else { + tmpId + } + } + val instances = contextIDParser.parse(contextId) + var serviceId:Option[String] = None + serviceId = findService(HaContextGatewayRouter.CONTEXT_SERVICE_STR, list => { + val services = list.filter(_.contains(HaContextGatewayRouter.CONTEXT_SERVICE_STR)) + services.headOption + }) + val serviceInstances = ServiceInstanceUtils.getRPCServerLoader.getServiceInstances(serviceId.orNull) + if (instances.size() > 0) { + serviceId.map(ServiceInstance(_, instances.get(0))).orNull + } else if (serviceInstances.size > 0) { + serviceInstances(0) + } else { + logger.error(s"No valid instance for service : " + serviceId.orNull) + null + } + } + +} + + +object HaContextGatewayRouter{ + val CONTEXT_ID_STR:String = "contextId" + val CONTEXT_SERVICE_STR:String = "contextservice" + val CONTEXT_REGEX: Regex = (normalPath(API_URL_PREFIX) + "rest_[a-zA-Z][a-zA-Z_0-9]*/(v\\d+)/contextservice/" + ".+").r +} diff --git a/gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/route/contextservice/ContextIdParserImpl.scala b/gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/route/contextservice/ContextIdParserImpl.scala new file mode 100644 index 0000000000..a0fed9ea9c --- /dev/null +++ b/gateway/gateway-ujes-support/src/main/scala/com/webank/wedatasphere/linkis/gateway/ujes/route/contextservice/ContextIdParserImpl.scala @@ -0,0 +1,52 @@ +package com.webank.wedatasphere.linkis.gateway.ujes.route.contextservice + +import java.util +import java.util.{ArrayList, List} + +import com.webank.wedatasphere.linkis.common.utils.Logging +import com.webank.wedatasphere.linkis.cs.common.entity.source.{ContextIDParser, HAContextID} +import com.webank.wedatasphere.linkis.cs.common.utils.CSHighAvailableUtils +import com.webank.wedatasphere.linkis.rpc.instancealias.InstanceAliasManager +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + + +/** + * created by cooperyang on 2020/2/19 + * Description: 如果id为HAID，则解析出对应的instance + */ +@Component +class ContextIdParserImpl extends ContextIDParser with Logging { + + @Autowired + var instanceAliasManager : InstanceAliasManager = _ + + override def parse(contextId: String): util.List[String] = { + + if (CSHighAvailableUtils.checkHAIDBasicFormat(contextId)) { + val instances = new util.ArrayList[String](2) + val haContextID = CSHighAvailableUtils.decodeHAID(contextId) + if (instanceAliasManager.isInstanceAliasValid(haContextID.getInstance)) { + instances.add(instanceAliasManager.getInstanceByAlias(haContextID.getInstance).getInstance) + } else { + error(s"parse HAID instance invalid. haIDKey : " + contextId) + } + if (instanceAliasManager.isInstanceAliasValid(haContextID.getBackupInstance)) { + instances.add(instanceAliasManager.getInstanceByAlias(haContextID.getBackupInstance).getInstance) + } else { + error("parse HAID backupInstance invalid. haIDKey : " + contextId) + } + instances + } else { + new util.ArrayList[String](0) + } + } + + private def isNumberic(s:String):Boolean = { + s.toCharArray foreach { + c => if (c < 48 || c >57) return false + } + true + } + +} diff --git a/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/SpringCloudGatewayConfiguration.scala b/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/SpringCloudGatewayConfiguration.scala index 06b77a67a5..88b4cdf4b5 100644 --- a/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/SpringCloudGatewayConfiguration.scala +++ b/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/SpringCloudGatewayConfiguration.scala @@ -18,7 +18,6 @@ package com.webank.wedatasphere.linkis.gateway.springcloud import com.netflix.loadbalancer.Server import com.webank.wedatasphere.linkis.common.ServiceInstance -import com.webank.wedatasphere.linkis.common.conf.CommonVars import com.webank.wedatasphere.linkis.common.utils.Logging import com.webank.wedatasphere.linkis.gateway.config.GatewaySpringConfiguration import com.webank.wedatasphere.linkis.gateway.parser.{DefaultGatewayParser, GatewayParser}