Provides clear methods in contextservice

db/linkis_ddl.sql

@@ -376,6 +376,9 @@ CREATE TABLE `linkis_ps_cs_context_map` (
   `value` mediumtext,
   `context_id` int(11) DEFAULT NULL,
   `keywords` varchar(255) DEFAULT NULL,
+  `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp',
+  `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
+  `access_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'last access time',
   PRIMARY KEY (`id`),
   UNIQUE KEY `key` (`key`,`context_id`,`context_type`),
   KEY `keywords` (`keywords`(191))
@@ -389,6 +392,9 @@ CREATE TABLE `linkis_ps_cs_context_map_listener` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
   `listener_source` varchar(255) DEFAULT NULL,
   `key_id` int(11) DEFAULT NULL,
+  `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp',
+  `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
+  `access_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'last access time',
   PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
@@ -404,7 +410,10 @@ CREATE TABLE `linkis_ps_cs_context_history` (
   `history_json` text,
   `keyword` varchar(255) DEFAULT NULL,
   PRIMARY KEY (`id`),
-  KEY `keyword` (`keyword`(191))
+  `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp',
+  `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
+  `access_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'last access time',
+  KEY `keyword` (`keyword`(191)),
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
 -- ----------------------------
@@ -420,6 +429,9 @@ CREATE TABLE `linkis_ps_cs_context_id` (
   `expire_time` datetime DEFAULT NULL,
   `instance` varchar(128) DEFAULT NULL,
   `backup_instance` varchar(255) DEFAULT NULL,
+  `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp',
+  `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
+  `access_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'last access time',
   PRIMARY KEY (`id`),
   KEY `instance` (`instance`(128)),
   KEY `backup_instance` (`backup_instance`(191)),
@@ -434,6 +446,9 @@ CREATE TABLE `linkis_ps_cs_context_listener` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
   `listener_source` varchar(255) DEFAULT NULL,
   `context_id` int(11) DEFAULT NULL,
+  `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp',
+  `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
+  `access_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'last access time',
   PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 

db/module/linkis-cs.sql

@@ -1,98 +1,93 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- * 
- *   http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-SET FOREIGN_KEY_CHECKS=0;
 
 -- ----------------------------
 -- Table structure for linkis_ps_cs_context_map
 -- ----------------------------
 DROP TABLE IF EXISTS `linkis_ps_cs_context_map`;
 CREATE TABLE `linkis_ps_cs_context_map` (
-  `id` int(11) NOT NULL AUTO_INCREMENT,
-  `key` varchar(128) DEFAULT NULL,
-  `context_scope` varchar(32) DEFAULT NULL,
-  `context_type` varchar(32) DEFAULT NULL,
-  `props` text,
-  `value` mediumtext,
-  `context_id` int(11) DEFAULT NULL,
-  `keywords` varchar(255) DEFAULT NULL,
-  PRIMARY KEY (`id`),
-  UNIQUE KEY `key` (`key`,`context_id`,`context_type`),
-  KEY `keywords` (`keywords`(191))
+	`id` int(11) NOT NULL AUTO_INCREMENT,
+	`key` varchar(128) DEFAULT NULL,
+	`context_scope` varchar(32) DEFAULT NULL,
+	`context_type` varchar(32) DEFAULT NULL,
+	`props` text,
+	`value` mediumtext,
+	`context_id` int(11) DEFAULT NULL,
+	`keywords` varchar(255) DEFAULT NULL,
+	`update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp',
+	`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
+    `access_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'last access time',
+	PRIMARY KEY (`id`),
+	UNIQUE KEY `key` (`key`,`context_id`,`context_type`),
+	KEY `keywords` (`keywords`(191))
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
 -- ----------------------------
 -- Table structure for linkis_ps_cs_context_map_listener
 -- ----------------------------
 DROP TABLE IF EXISTS `linkis_ps_cs_context_map_listener`;
 CREATE TABLE `linkis_ps_cs_context_map_listener` (
-  `id` int(11) NOT NULL AUTO_INCREMENT,
-  `listener_source` varchar(255) DEFAULT NULL,
-  `key_id` int(11) DEFAULT NULL,
-  PRIMARY KEY (`id`)
+	`id` int(11) NOT NULL AUTO_INCREMENT,
+	`listener_source` varchar(255) DEFAULT NULL,
+	`key_id` int(11) DEFAULT NULL,
+	`update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp',
+	`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
+    `access_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'last access time',
+	PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
 -- ----------------------------
 -- Table structure for linkis_ps_cs_context_history
 -- ----------------------------
 DROP TABLE IF EXISTS `linkis_ps_cs_context_history`;
 CREATE TABLE `linkis_ps_cs_context_history` (
-  `id` int(11) NOT NULL AUTO_INCREMENT,
-  `context_id` int(11) DEFAULT NULL,
-  `source` text,
-  `context_type` varchar(32) DEFAULT NULL,
-  `history_json` text,
-  `keyword` varchar(255) DEFAULT NULL,
-  PRIMARY KEY (`id`),
-  KEY `keyword` (`keyword`(191))
+	`id` int(11) NOT NULL AUTO_INCREMENT,
+	`context_id` int(11) DEFAULT NULL,
+	`source` text,
+	`context_type` varchar(32) DEFAULT NULL,
+	`history_json` text,
+	`keyword` varchar(255) DEFAULT NULL,
+    `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp',
+    `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
+    `access_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'last access time',
+	PRIMARY KEY (`id`),
+	KEY `keyword` (`keyword`(191)),
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
 -- ----------------------------
 -- Table structure for linkis_ps_cs_context_id
 -- ----------------------------
 DROP TABLE IF EXISTS `linkis_ps_cs_context_id`;
 CREATE TABLE `linkis_ps_cs_context_id` (
-  `id` int(11) NOT NULL AUTO_INCREMENT,
-  `user` varchar(32) DEFAULT NULL,
-  `application` varchar(32) DEFAULT NULL,
-  `source` varchar(255) DEFAULT NULL,
-  `expire_type` varchar(32) DEFAULT NULL,
-  `expire_time` datetime DEFAULT NULL,
-  `instance` varchar(32) DEFAULT NULL,
-  `backup_instance` varchar(255) DEFAULT NULL,
-  PRIMARY KEY (`id`),
-  KEY `instance` (`instance`),
-  KEY `backup_instance` (`backup_instance`(191)),
-  KEY `instance_2` (`instance`,`backup_instance`(191))
+	`id` int(11) NOT NULL AUTO_INCREMENT,
+	`user` varchar(32) DEFAULT NULL,
+	`application` varchar(32) DEFAULT NULL,
+	`source` varchar(255) DEFAULT NULL,
+	`expire_type` varchar(32) DEFAULT NULL,
+	`expire_time` datetime DEFAULT NULL,
+	`instance` varchar(128) DEFAULT NULL,
+	`backup_instance` varchar(255) DEFAULT NULL,
+	`update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp',
+	`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
+    `access_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'last access time',
+	PRIMARY KEY (`id`),
+	KEY `instance` (`instance`(128)),
+	KEY `backup_instance` (`backup_instance`(191)),
+	KEY `instance_2` (`instance`(128),`backup_instance`(128))
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
 -- ----------------------------
 -- Table structure for linkis_ps_cs_context_listener
 -- ----------------------------
 DROP TABLE IF EXISTS `linkis_ps_cs_context_listener`;
 CREATE TABLE `linkis_ps_cs_context_listener` (
-  `id` int(11) NOT NULL AUTO_INCREMENT,
-  `listener_source` varchar(255) DEFAULT NULL,
-  `context_id` int(11) DEFAULT NULL,
-  PRIMARY KEY (`id`)
+	`id` int(11) NOT NULL AUTO_INCREMENT,
+	`listener_source` varchar(255) DEFAULT NULL,
+	`context_id` int(11) DEFAULT NULL,
+	`update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp',
+	`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time',
+    `access_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'last access time',
+	PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
 
 
 
-
-

db/upgrade/1.1.3_schema/mysql/linkis_ddl.sql

@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+-- cs table
+ALTER TABLE `linkis_ps_cs_context_map` ADD COLUMN `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp' AFTER `keywords`;
+ALTER TABLE `linkis_ps_cs_context_map` ADD COLUMN `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time' AFTER `update_time`;
+ALTER TABLE `linkis_ps_cs_context_map` ADD COLUMN `access_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'last access time' AFTER `create_time`;
+
+ALTER TABLE `linkis_ps_cs_context_map_listener` ADD COLUMN `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp' AFTER `key_id`;
+ALTER TABLE `linkis_ps_cs_context_map_listener` ADD COLUMN `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time' AFTER `update_time`;
+ALTER TABLE `linkis_ps_cs_context_map_listener` ADD COLUMN `access_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'last access time' AFTER `create_time`;
+
+ALTER TABLE `linkis_ps_cs_context_history` ADD COLUMN `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp' AFTER `keyword`;
+ALTER TABLE `linkis_ps_cs_context_history` ADD COLUMN `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time' AFTER `update_time`;
+ALTER TABLE `linkis_ps_cs_context_history` ADD COLUMN `access_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'last access time' AFTER `create_time`;
+
+ALTER TABLE `linkis_ps_cs_context_id` ADD COLUMN `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp' AFTER `backup_instance`;
+ALTER TABLE `linkis_ps_cs_context_id` ADD COLUMN `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time' AFTER `update_time`;
+ALTER TABLE `linkis_ps_cs_context_id` ADD COLUMN `access_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'last access time' AFTER `create_time`;
+
+ALTER TABLE `linkis_ps_cs_context_listener` ADD COLUMN `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'update unix timestamp' AFTER `context_id`;
+ALTER TABLE `linkis_ps_cs_context_listener` ADD COLUMN `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'create time' AFTER `update_time`;
+ALTER TABLE `linkis_ps_cs_context_listener` ADD COLUMN `access_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'last access time' AFTER `create_time`;
+
+
+-- manager ec record
+DROP TABLE IF EXISTS `linkis_cg_ec_resource_info_record`;
+CREATE TABLE `linkis_cg_ec_resource_info_record` (
+	`id` INT(20) NOT NULL AUTO_INCREMENT,
+	`label_value` VARCHAR(255) NOT NULL,
+	`create_user` VARCHAR(128) NOT NULL,
+	`service_instance` varchar(128) COLLATE utf8_bin DEFAULT NULL,
+	`ecm_instance` varchar(128) COLLATE utf8_bin DEFAULT NULL,
+	`ticket_id` VARCHAR(100) NOT NULL,
+	`log_dir_suffix` varchar(128) COLLATE utf8_bin DEFAULT NULL,
+	`request_times` INT(8),
+	`request_resource` VARCHAR(255),
+	`used_times` INT(8),
+	`used_resource` VARCHAR(255),
+	`release_times` INT(8),
+	`released_resource` VARCHAR(255),
+	`release_time` datetime DEFAULT NULL,
+	`used_time` datetime DEFAULT NULL,
+	`create_time` datetime DEFAULT CURRENT_TIMESTAMP,
+	PRIMARY KEY (`id`),
+	KEY (`ticket_id`),
+	UNIQUE KEY `label_value_ticket_id` (`ticket_id`,`label_value`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+

linkis-public-enhancements/linkis-context-service/linkis-cs-cache/src/main/java/org/apache/linkis/cs/contextcache/cache/DefaultContextAddListener.java

@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.cs.contextcache.cache;
+
+import org.apache.linkis.common.listener.Event;
+import org.apache.linkis.cs.listener.CSIDListener;
+import org.apache.linkis.cs.listener.ListenerBus.ContextAsyncListenerBus;
+import org.apache.linkis.cs.listener.event.ContextIDEvent;
+import org.apache.linkis.cs.listener.manager.imp.DefaultContextListenerManager;
+import org.apache.linkis.cs.persistence.entity.PersistenceContextID;
+import org.apache.linkis.cs.persistence.persistence.ContextIDPersistence;
+import org.apache.linkis.cs.persistence.persistence.ContextMapPersistence;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Date;
+
+@Component
+public class DefaultContextAddListener implements CSIDListener {
+
+    private static final Logger logger = LoggerFactory.getLogger(DefaultContextAddListener.class);
+
+    private ContextAsyncListenerBus listenerBus =
+            DefaultContextListenerManager.getInstance().getContextAsyncListenerBus();
+
+    @Autowired private ContextIDPersistence contextIDPersistence;
+
+    @Autowired private ContextMapPersistence contextMapPersistence;
+
+    @PostConstruct
+    private void init() {
+        listenerBus.addListener(this);
+    }
+
+    @Override
+    public void onEventError(Event event, Throwable t) {}
+
+    @Override
+    public void onCSIDAccess(ContextIDEvent contextIDEvent) {}
+
+    @Override
+    public void onCSIDADD(ContextIDEvent contextIDEvent) {
+        try {
+            PersistenceContextID pContextID = new PersistenceContextID();
+            pContextID.setContextId(contextIDEvent.getContextID().getContextId());
+            pContextID.setAccessTime(new Date());
+            contextIDPersistence.updateContextID(pContextID);
+            if (logger.isDebugEnabled()) {
+                logger.debug(
+                        "Updated accesstime for contextID : {}",
+                        contextIDEvent.getContextID().getContextId());
+            }
+        } catch (Exception e) {
+            logger.error("Update context accessTime failed, {}", e.getMessage(), e);
+        }
+    }
+
+    @Override
+    public void onCSIDRemoved(ContextIDEvent contextIDEvent) {}
+
+    @Override
+    public void onEvent(Event event) {}
+}

linkis-public-enhancements/linkis-context-service/linkis-cs-cache/src/main/java/org/apache/linkis/cs/contextcache/cache/DefaultContextCache.java

@@ -22,6 +22,7 @@
 import org.apache.linkis.cs.common.exception.CSErrorException;
 import org.apache.linkis.cs.contextcache.cache.csid.ContextIDValue;
 import org.apache.linkis.cs.contextcache.cache.csid.ContextIDValueGenerator;
+import org.apache.linkis.cs.contextcache.conf.ContextCacheConf;
 import org.apache.linkis.cs.contextcache.metric.ContextCacheMetric;
 import org.apache.linkis.cs.contextcache.metric.ContextIDMetric;
 import org.apache.linkis.cs.contextcache.metric.DefaultContextCacheMetric;
@@ -44,6 +45,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.time.Duration;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -71,7 +73,9 @@ private void init() {
         listenerBus.addListener(this);
         this.cache =
                 CacheBuilder.newBuilder()
-                        .maximumSize(3000)
+                        .maximumSize(ContextCacheConf.MAX_CACHE_SIZE)
+                        .expireAfterWrite(
+                                Duration.ofMillis(ContextCacheConf.MAX_CACHE_READ_EXPIRE_MILLS))
                         .removalListener(contextIDRemoveListener)
                         .recordStats()
                         .build();

linkis-public-enhancements/linkis-context-service/linkis-cs-cache/src/main/java/org/apache/linkis/cs/contextcache/cache/csid/impl/ContextIDValueImpl.java

@@ -109,7 +109,9 @@ public void onCSKeyUpdate(ContextKeyEvent contextKeyEvent) {
                 "Now, The Memory of ContextID({}) are {}",
                 contextID,
                 getContextIDMetric().getMemory());
-        logger.debug("Finished to deal csKeyEvent of csID({})", this.contextID);
+        if (logger.isDebugEnabled()) {
+            logger.debug("Finished to deal csKeyEvent of csID({})", this.contextID);
+        }
     }
 
     @Override