diff --git a/linkis-computation-governance/linkis-manager/label-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala b/linkis-computation-governance/linkis-manager/label-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala index 765c630393..85014ab6aa 100644 --- a/linkis-computation-governance/linkis-manager/label-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala +++ b/linkis-computation-governance/linkis-manager/label-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala @@ -17,8 +17,6 @@ package org.apache.linkis.manager.label.service.impl -import java.util - import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.manager.common.entity.node.ScoreServiceInstance @@ -27,8 +25,7 @@ import org.apache.linkis.manager.common.utils.ManagerUtils import org.apache.linkis.manager.label.LabelManagerUtils import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext import org.apache.linkis.manager.label.conf.LabelManagerConf -import org.apache.linkis.manager.label.entity.Label.ValueRelation -import org.apache.linkis.manager.label.entity.{Feature, Label, UserModifiable} +import org.apache.linkis.manager.label.entity.{Feature, Label} import org.apache.linkis.manager.label.score.NodeLabelScorer import org.apache.linkis.manager.label.service.NodeLabelService import org.apache.linkis.manager.label.utils.LabelUtils @@ -38,6 +35,7 @@ import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional import org.springframework.util.CollectionUtils +import java.util import scala.collection.JavaConversions._ import scala.collection.mutable import scala.collection.mutable.ArrayBuffer @@ -179,13 +177,6 @@ class DefaultNodeLabelService extends NodeLabelService with Logging { labelManagerPersistence.removeNodeLabels(instance, removeLabels.map(_.getId)) } - @Transactional(rollbackFor = Array(classOf[Exception])) - override def removeLabelsFromNodeWithoutPermanent(instance: ServiceInstance, permanentLabel: Array[String] = Array()): Unit = { - val labels = labelManagerPersistence.getLabelByServiceInstance(instance) - val lowerCasePermanentLabels = permanentLabel.map(_.toLowerCase()) - val withoutPermanentLabel = labels.filterNot(label => lowerCasePermanentLabels.contains(label.getLabelKey.toLowerCase)).map(_.getId) - labelManagerPersistence.removeNodeLabels(instance, withoutPermanentLabel) - } /** * Get node instances by labels diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/LabelManagerMapper.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/LabelManagerMapper.java index edccfec522..f21cf4a80a 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/LabelManagerMapper.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/LabelManagerMapper.java @@ -20,9 +20,11 @@ import org.apache.linkis.common.ServiceInstance; import org.apache.linkis.manager.common.entity.label.LabelKeyValue; import org.apache.linkis.manager.common.entity.persistence.PersistenceLabel; +import org.apache.linkis.manager.common.entity.persistence.PersistenceLabelRel; import org.apache.linkis.manager.common.entity.persistence.PersistenceNode; import org.apache.linkis.manager.common.entity.persistence.PersistenceResource; import org.apache.ibatis.annotations.*; +import org.apache.linkis.manager.label.entity.Label; import java.util.List; import java.util.Map; @@ -30,275 +32,65 @@ @Mapper public interface LabelManagerMapper { - @Options(useGeneratedKeys = true, keyProperty = "persistenceLabel.id", keyColumn = "id") - @Insert("INSERT INTO linkis_cg_manager_label (label_key,label_value,label_feature,label_value_size,update_time,create_time) VALUES" + - "(#{persistenceLabel.labelKey},#{persistenceLabel.stringValue},#{persistenceLabel.feature},#{persistenceLabel.labelValueSize},now(), now())") + void registerLabel(@Param("persistenceLabel") PersistenceLabel persistenceLabel); - @Insert({""}) void registerLabelKeyValues(@Param("labelValueKeyAndContent") Map labelValueKeyAndContent, @Param("labelId") int labelId); - @Select("select * from linkis_cg_manager_label where id=#{id}") - @Results({ - @Result(property = "labelKey", column = "label_key"), - @Result(property = "feature", column = "label_feature"), - @Result(property = "labelValueSize", column = "label_value_size"), - @Result(property = "updateTime", column = "update_time"), - @Result(property = "createTime", column = "create_time"), - @Result(property = "stringValue", column = "label_value") - }) PersistenceLabel getLabel(@Param("id") int id); - @Delete("delete from linkis_cg_manager_label where id =#{id}") + void deleteLabel(@Param("id") int id); - @Delete("delete from linkis_cg_manager_label where label_key =#{labelKey} and label_value=#{labelStringValue} ") void deleteByLabel(@Param("labelKey") String labelKey, @Param("labelStringValue") String labelStringValue); - @Delete("delete from linkis_cg_manager_label_value_relation where label_id =#{id}") void deleteLabelKeyVaules(@Param("id") int id); - @Update("update linkis_cg_manager_label set label_key = #{persistenceLabel.labelKey},label_value = #{persistenceLabel.stringValue}," + - "label_feature=#{persistenceLabel.feature},label_value_size=#{persistenceLabel.labelValueSize},update_time=#{persistenceLabel.updateTime} where id=#{id}") void updateLabel(@Param("id") int id, @Param("persistenceLabel") PersistenceLabel persistenceLabel); - - @Insert({ - "" - }) void addLabelServiceInstance(@Param("instance") String instance, @Param("labelIds") List labelIds); - @Select("") - @Results({ - @Result(property = "labelKey", column = "label_key"), - @Result(property = "feature", column = "label_feature"), - @Result(property = "labelValueSize", column = "label_value_size"), - @Result(property = "updateTime", column = "update_time"), - @Result(property = "createTime", column = "create_time"), - @Result(property = "stringValue", column = "label_value") - }) - List getLabelsByLabelKeyValues(@Param("labelKeyValues") Map labelKeyValues); - - - @Select("") - @Results({ - @Result(property = "labelKey", column = "label_key"), - @Result(property = "feature", column = "label_feature"), - @Result(property = "labelValueSize", column = "label_value_size"), - @Result(property = "updateTime", column = "update_time"), - @Result(property = "createTime", column = "create_time"), - @Result(property = "stringValue", column = "label_value") - }) - PersistenceLabel getLabelByLabelKeyValues(@Param("labelKey") String labelKey, @Param("labelKeyValues") Map labelKeyValues); - - @Select("select * from linkis_cg_manager_label where id in (select label_id from linkis_cg_manager_label_service_instance where service_instance=#{instance})") - @Results({ - @Result(property = "labelKey", column = "label_key"), - @Result(property = "feature", column = "label_feature"), - @Result(property = "labelValueSize", column = "label_value_size"), - @Result(property = "updateTime", column = "update_time"), - @Result(property = "createTime", column = "create_time"), - @Result(property = "stringValue", column = "label_value") - }) List getLabelByServiceInstance(@Param("instance") String instance); - @Select("") - List getLabelByLabelKeyValuesOverload(@Param("labelKeyValues") Map labelKeyValues); - - - @Select("select * from linkis_cg_manager_label where id in (select label_id from linkis_cg_manager_label_resource A join linkis_cg_manager_linkis_resources B on A.resource_id = B.id where A.resource_id = #{persistenceResource.id} )") - @Results({ - @Result(property = "labelKey", column = "label_key"), - @Result(property = "feature", column = "label_feature"), - @Result(property = "labelValueSize", column = "label_value_size"), - @Result(property = "updateTime", column = "update_time"), - @Result(property = "createTime", column = "create_time"), - @Result(property = "stringValue", column = "label_value") - }) List getLabelByResource(@Param("persistenceResource") PersistenceResource persistenceResource); -// @Insert("insert into linkis_cg_manager_label_resource (label_id, resource_id, update_time, create_time)" + -// "values(#{labelId}, #{resourceId}, now(), now())") -// void addLabelResource(@Param("labelId")int labelId,@Param("resourceId")int resourceId); - - @Insert({ - "" - }) void addLabelsAndResource(@Param("resourceId") int resourceId, @Param("labelIds") List labelIds); -// @Insert({ -// "" -// }) -// void addLabelAndResources(@Param("labelId")int labelId,@Param("resourceIds")List resourceIds); - -// @Select("select id from linkis_cg_manager_label where creator = #{creator}") -// List getLabelIds (@Param("creator") String creator); -// -// @Select("select resource_id from linkis_cg_manager_label_resource where label_id = #{labelId}") -// List getResourceIdsByLabelId (@Param("labelId") int labelId); -// -// @Select("") -// List getResourcesByLabels (@Param("labelIds") List labelIds); - - @Select("select * from linkis_cg_manager_linkis_resources A join linkis_cg_manager_label_resource B on A.id = B.resource_id " + - "and B.label_id in (select id from linkis_cg_manager_label where label_key=#{labelKey} and label_value=#{stringValue})") - @Results({ - @Result(property = "maxResource", column = "max_resource"), - @Result(property = "minResource", column = "min_resource"), - @Result(property = "usedResource", column = "used_resource"), - @Result(property = "leftResource", column = "left_resource"), - @Result(property = "expectedResource", column = "expected_resource"), - @Result(property = "lockedResource", column = "locked_resource"), - @Result(property = "updateTime", column = "update_time"), - @Result(property = "createTime", column = "create_time") - }) List getResourcesByLabel(@Param("labelKey") String labelKey, @Param("stringValue") String stringValue); - @Select("select label_id from linkis_cg_manager_label_service_instance where service_instance = #{instance}") + List getLabelIdsByInstance(@Param("instance") String instance); - @Select("select * from linkis_cg_manager_label A join linkis_cg_manager_label_service_instance B on A.id = B.label_id where B.service_instance = #{instance}") List getLabelsByInstance(@Param("instance") String instance); - @Select("select * from linkis_cg_manager_service_instance A join linkis_cg_manager_label_service_instance B on A.instance = B.service_instance where B.label_id=#{labelId}") - @Results({ - @Result(property = "updateTime", column = "update_time"), - @Result(property = "createTime", column = "create_time") - }) List getInstanceByLabelId(@Param("labelId") int labelId); + List getResourcesByLabels(@Param("labels") List labels); + - @Select("") List getInstanceIdsByLabelIds(@Param("labelIds") List labelIds); - @Select("") - @Results({ - @Result(property = "labelKey", column = "label_key"), - @Result(property = "feature", column = "label_feature"), - @Result(property = "labelValueSize", column = "label_value_size"), - @Result(property = "updateTime", column = "update_time"), - @Result(property = "createTime", column = "create_time"), - @Result(property = "stringValue", column = "label_value") - }) List getLabelsByLabelIds(@Param("labelIds") List labelIds); - @Insert({ - "" - }) + void addLabelsByUser(@Param("userName") String userName, @Param("labelIds") List labelIds); - @Select("select username from linkis_cg_manager_label_user where label_id = #{labelId}") + List getUserNameByLabelId(@Param("labelId") int labelId); - @Select("") List getUserNamesByLabelIds(@Param("labelIds") List labelIds); - @Select("select * from linkis_cg_manager_label A join linkis_cg_manager_label_user B on A.id = B.label_id and B.username = #{userName}") - @Results({ - @Result(property = "labelKey", column = "label_key"), - @Result(property = "feature", column = "label_feature"), - @Result(property = "labelValueSize", column = "label_value_size"), - @Result(property = "updateTime", column = "update_time"), - @Result(property = "createTime", column = "create_time"), - @Result(property = "stringValue", column = "label_value") - }) List getLabelsByUser(@Param("userName") String userName); - @Select("select * from linkis_cg_manager_label where label_key = #{labelKey}") - @Results({ - @Result(property = "labelKey", column = "label_key"), - @Result(property = "feature", column = "label_feature"), - @Result(property = "labelValueSize", column = "label_value_size"), - @Result(property = "updateTime", column = "update_time"), - @Result(property = "createTime", column = "create_time"), - @Result(property = "stringValue", column = "label_value") - }) List getLabelsByLabelKey(@Param("labelKey") String labelKey); - @Delete("delete from linkis_cg_manager_label_resource A join linkis_cg_manager_linkis_resources B on A.resource_id = B.id and B.ticketId=#{ticketId}") void deleteLabelResourceByByTicketId(@Param("ticketId") String ticketId); - - @Delete({"" - }) void deleteLabelIdsAndInstance(@Param("instance") String instance, @Param("labelIds") List labelIds); - @Delete({"" - }) void deleteLabelIdsByUser(@Param("userName") String userName, @Param("labelIds") List labelIds); - @Delete("delete from linkis_cg_manager_label_user where label_id=#{labelId}") void deleteUserById(int labelId); /** @@ -309,6 +101,8 @@ public interface LabelManagerMapper { */ List> dimListNodeRelationsByKeyValueMap(@Param("keyValueMap") Map> labelKeyAndValuesMap, @Param("valueRelation") String name); + List> getNodeRelationsByLabels(@Param("labels") List labels); + /** * 通过instance信息，同时返回instance信息和label信息 * @@ -328,6 +122,7 @@ public interface LabelManagerMapper { List getNodeByLabelKeyValue(@Param("labelKey") String labelKey, @Param("stringValue") String stringValue); + /** * @param values,可能是多个label的散列 all 和 and 类型的 查询 * @return @@ -342,14 +137,6 @@ public interface LabelManagerMapper { */ List listResourceByLaBelId(Integer labelId); - /** - * 通过label的keyvalues再找到resource - * 和{@link LabelManagerMapper#dimListLabelByKeyValueMap(Map, String)} 区别只是多了关联和资源表,并且返回是resource - * - * @param keyValueMap - * @return - */ - List dimListResourceBykeyValueMap(@Param("keyValueMap") Map> keyValueMap, @Param("valueRelation") String name); /** * 通过labelId删除资源，还有资源和label的关联，并不会删除label表记录 @@ -418,4 +205,6 @@ public interface LabelManagerMapper { * @return */ List dimlistResourceLabelByKeyValueMap(@Param("keyValueMap") Map> labelKeyAndValuesMap, @Param("valueRelation") String name); + + List listLabelBySQLPattern(@Param("pattern") String pattern, @Param("labelKey") String labelKey); } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/NodeMetricManagerMapper.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/NodeMetricManagerMapper.java index f65b1b4e1b..90c6c9131a 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/NodeMetricManagerMapper.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/NodeMetricManagerMapper.java @@ -31,7 +31,7 @@ public interface NodeMetricManagerMapper { void addNodeMetrics(@Param("nodeMetrics") PersistenceNodeMetrics nodeMetrics); @Select("select count(id) from linkis_cg_manager_service_instance_metrics met inner join linkis_cg_manager_service_instance ins" + - " on met.instance = #{instance} and ins.instance = #{instance} and met.instance = ins.instance") + " on met.instance = #{instance} and ins.instance = #{instance} and met.instance = ins.instance") int checkInstanceExist(@Param("instance") String instance); @@ -69,7 +69,6 @@ public interface NodeMetricManagerMapper { " heartbeat_msg = #{nodeMetrics.heartBeatMsg}," + " healthy_status=#{nodeMetrics.healthy}, " + " update_time=#{nodeMetrics.updateTime}," + - " create_time=#{nodeMetrics.createTime}," + " where instance = #{instance}" + ""}) void updateNodeMetrics(@Param("nodeMetrics") PersistenceNodeMetrics nodeMetrics, @Param("instance") String instance); diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/ResourceManagerMapper.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/ResourceManagerMapper.java index 84d8b85ac8..0be125a3d7 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/ResourceManagerMapper.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/ResourceManagerMapper.java @@ -34,39 +34,19 @@ public interface ResourceManagerMapper { @Update({ "update linkis_cg_manager_linkis_resources set max_resource = #{persistenceResource.maxResource},min_resource = #{persistenceResource.minResource}, " + "used_resource=#{persistenceResource.usedResource},left_resource=#{persistenceResource.leftResource},expected_resource=#{persistenceResource.expectedResource}," + - "locked_resource=#{persistenceResource.lockedResource},resourceType=#{persistenceResource.resourceType}," + - "update_time=#{persistenceResource.updateTime},create_time=#{persistenceResource.createTime} where ticketId = #{ticketId}"}) + "locked_resource=#{persistenceResource.lockedResource}," + + "update_time=#{persistenceResource.updateTime} where ticketId = #{ticketId}"}) void nodeResourceUpdate(@Param("ticketId") String ticketId,@Param("persistenceResource") PersistenceResource persistenceResource); @Update({"update linkis_cg_manager_linkis_resources set max_resource = #{persistenceResource.maxResource},min_resource = #{persistenceResource.minResource}," + "used_resource=#{persistenceResource.usedResource},left_resource=#{persistenceResource.leftResource},expected_resource=#{persistenceResource.expectedResource}," + - "locked_resource=#{persistenceResource.lockedResource},resourceType=#{persistenceResource.resourceType},ticketId=#{persistenceResource.ticketId}," + - "update_time=#{persistenceResource.updateTime},create_time=#{persistenceResource.createTime} where id = #{resourceId}"}) + "locked_resource=#{persistenceResource.lockedResource}," + + "update_time=#{persistenceResource.updateTime} where id = #{resourceId}"}) void nodeResourceUpdateByResourceId(@Param("resourceId") int resourceId,@Param("persistenceResource") PersistenceResource persistenceResource); @Select("select id from linkis_cg_manager_linkis_resources where ticketId is null and id in ( select resource_id from linkis_cg_manager_label_resource A join linkis_cg_manager_label_service_instance B on A.label_id=B.label_id and B.service_instance=#{instance})") int getNodeResourceUpdateResourceId(@Param("instance") String instance); -// @Select("select * from linkis_cg_manager_linkis_resources where id in \n" + -// "(select resource_id from linkis_cg_manager_label_resource A join linkis_cg_manager_label_service_instance B on A.label_id=B.label_id and B.service_instance_id=#{serviceInstanceId})") -// @Results({ -// @Result(property = "maxResource", column = "max_resource"), -// @Result(property = "minResource", column = "min_resource"), -// @Result(property = "usedResource", column = "used_resource"), -// @Result(property = "leftResource", column = "left_resource"), -// @Result(property = "expectedResource", column = "expected_resource"), -// @Result(property = "lockedResource", column = "locked_resource"), -// @Result(property = "updateTime", column = "update_time"), -// @Result(property = "createTime", column = "create_time"), -// -// }) -// List getResourceByServiceInstanceId(@Param("serviceInstanceId") int serviceInstanceId); -// -// @Select("select * from linkis_cg_manager_linkis_resources where id = #{id}") -// PersistenceResource getResourceById(@Param("id") int id); -// -// @Delete("delete from linkis_cg_manager_linkis_resources where id = #{id}") -// void deleteResourceById(@Param("id") int id); @Delete("delete from linkis_cg_manager_label_resource where label_id in (select label_id from linkis_cg_manager_label_service_instance where service_instance=#{instance})") void deleteResourceAndLabelId(@Param("instance") String instance); @@ -97,4 +77,10 @@ public interface ResourceManagerMapper { @Select("select * from linkis_cg_manager_label where id in (select label_id from linkis_cg_manager_label_resource A join linkis_cg_manager_linkis_resources B on A.resource_id=B.id and B.ticketId=#{ticketId})") List getLabelsByTicketId(@Param("ticketId") String ticketId); + + void deleteResourceById(@Param("ids") List ids); + + void deleteResourceRelByResourceId(@Param("ids") List ids); + + PersistenceResource getResourceById(@Param("id") Integer id); } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/impl/LabelManagerMapper.xml b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/impl/LabelManagerMapper.xml index d88650c494..2514c2a845 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/impl/LabelManagerMapper.xml +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/dao/impl/LabelManagerMapper.xml @@ -15,11 +15,217 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + INSERT IGNORE INTO linkis_cg_manager_label (label_key,label_value,label_feature,label_value_size,update_time,create_time) + VALUES (#{persistenceLabel.labelKey},#{persistenceLabel.stringValue},#{persistenceLabel.feature},#{persistenceLabel.labelValueSize},now(), now()) + + + + REPLACE INTO linkis_cg_manager_label_value_relation (label_value_key,label_value_content,label_id,update_time,create_time) VALUES + + (#{valueKey}, #{valueContent},#{labelId},now(),now()) + + + + + + + delete from linkis_cg_manager_label where id =#{id} + + + + delete from linkis_cg_manager_label where label_key =#{labelKey} and label_value=#{labelStringValue} + + + + delete from linkis_cg_manager_label_value_relation where label_id =#{id} + + + + update linkis_cg_manager_label set label_key = #{persistenceLabel.labelKey},label_value = #{persistenceLabel.stringValue}, + label_feature=#{persistenceLabel.feature},label_value_size=#{persistenceLabel.labelValueSize},update_time=#{persistenceLabel.updateTime} where id=#{id} + + + + insert into linkis_cg_manager_label_service_instance(label_id, service_instance, update_time,create_time) values + + (#{labelId}, #{instance},now(),now()) + + + + + + + + + + insert into linkis_cg_manager_label_resource(label_id, resource_id,update_time, create_time) values + + (#{labelId},#{resourceId},now(), now()) + + + + + + + + + + + + + + + + + + + insert into linkis_cg_manager_label_user(username, label_id,update_time, create_time) values + + (#{userName},#{labelId},now(), now()) + + + + + + + + + + + + + delete from linkis_cg_manager_label_resource A join linkis_cg_manager_linkis_resources B on A.resource_id = B.id and B.ticketId=#{ticketId} + + + + delete from linkis_cg_manager_label_service_instance where service_instance = #{instance} and label_id in + + #{labelId} + + + + + delete from linkis_cg_manager_label_user where username = #{userName} and label_id in + + #{labelId} + + + + + delete from linkis_cg_manager_label_user where label_id=#{labelId} + + SELECT l.id AS 'id',l.label_key AS 'labelKey',l.label_value AS 'stringValue', - l.label_value_size AS 'labelValueSize',si.instance,si.name AS 'applicationName' FROM - linkis_cg_manager_label l - JOIN linkis_cg_manager_label_service_instance lsi - ON l.id = lsi.label_id - JOIN linkis_cg_manager_service_instance si - ON si.instance=lsi.service_instance - WHERE (si.instance,si.`name`) IN - - (#{node.instance},#{node.applicationName}) - + l.label_value_size AS 'labelValueSize', lsi.service_instance AS 'instance' + FROM ( + SELECT service_instance,label_id + FROM linkis_cg_manager_label_service_instance + WHERE service_instance IN + + (#{node.instance}) + + ) lsi + INNER JOIN linkis_cg_manager_label l ON l.id = lsi.label_id; + SELECT + si.instance, + si.name as 'application_name' + FROM + ( + SELECT + lsi.service_instance + FROM + ( + SELECT + id + FROM + linkis_cg_manager_label + WHERE + label_key = #{labelKey} + and label_value = #{stringValue} + ) l + INNER JOIN linkis_cg_manager_label_service_instance lsi ON l.id = lsi.label_id + ) t + INNER JOIN linkis_cg_manager_service_instance si ON si.instance = t.service_instance + + + + - - - - - - - + + - - delete from linkis_cg_manager_linkis_resources where id in (select resource_id from linkis_cg_manager_label_resource where label_id = #{labelId}); + delete from linkis_cg_manager_linkis_resources where id in (select resource_id from + linkis_cg_manager_label_resource where label_id = #{labelId}); @@ -149,7 +388,8 @@ - delete from linkis_cg_manager_linkis_resources where id in (select resource_id from linkis_cg_manager_label_resource + delete from linkis_cg_manager_linkis_resources where id in (select resource_id from + linkis_cg_manager_label_resource where label_id = #{id} ); @@ -157,7 +397,8 @@ - delete from linkis_cg_manager_linkis_resources where id in (select resource_id from linkis_cg_manager_label_resource + delete from linkis_cg_manager_linkis_resources where id in (select resource_id from + linkis_cg_manager_label_resource where label_id in #{labelId} @@ -171,7 +412,8 @@ - delete from linkis_cg_manager_linkis_resources where id in (select resource_id from linkis_cg_manager_label_resource + delete from linkis_cg_manager_linkis_resources where id in (select resource_id from + linkis_cg_manager_label_resource where label_id in ( select tmp.id from( SELECT l.id,l.label_value_size @@ -265,7 +507,8 @@ - + + + \ No newline at end of file diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/LabelManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/LabelManagerPersistence.java index 9b0e1da18d..4cdfb813f0 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/LabelManagerPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/LabelManagerPersistence.java @@ -19,6 +19,7 @@ import org.apache.linkis.common.ServiceInstance; import org.apache.linkis.manager.common.entity.persistence.PersistenceLabel; +import org.apache.linkis.manager.common.entity.persistence.PersistenceLabelRel; import org.apache.linkis.manager.common.entity.persistence.PersistenceResource; import org.apache.linkis.manager.label.entity.Label; @@ -28,6 +29,8 @@ public interface LabelManagerPersistence { + + List getLabelByPattern(String labelValuePattern, String labelKey, Integer page, Integer size); //插入标签 void addLabel(PersistenceLabel persistenceLabel); diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/NodeManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/NodeManagerPersistence.java index b4160e34d6..1ec531558e 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/NodeManagerPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/NodeManagerPersistence.java @@ -35,7 +35,7 @@ public interface NodeManagerPersistence { */ void addNodeInstance(Node node) throws PersistenceErrorException; - void updateEngineNode(ServiceInstance serviceInstance, Node node) throws PersistenceErrorException,LinkisRetryException;; + void updateEngineNode(ServiceInstance serviceInstance, Node node) throws PersistenceErrorException, LinkisRetryException; /** * 移除node @@ -114,4 +114,5 @@ public interface NodeManagerPersistence { * @throws PersistenceErrorException */ List getEngineNodeByEM(ServiceInstance serviceInstance) throws PersistenceErrorException; + } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/ResourceLabelPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/ResourceLabelPersistence.java index 5d849e4c0f..23b63ffeaa 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/ResourceLabelPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/ResourceLabelPersistence.java @@ -19,6 +19,7 @@ import org.apache.linkis.manager.common.entity.label.LabelKeyValue; import org.apache.linkis.manager.common.entity.persistence.PersistenceLabel; import org.apache.linkis.manager.common.entity.persistence.PersistenceResource; +import org.apache.linkis.manager.exception.PersistenceErrorException; import org.apache.linkis.manager.label.entity.Label; import java.util.List; @@ -69,7 +70,7 @@ public interface ResourceLabelPersistence { * * @param label */ - void removeResourceByLabel(PersistenceLabel label); + void removeResourceByLabel(PersistenceLabel label) throws PersistenceErrorException; /** diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/ResourceManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/ResourceManagerPersistence.java index a70c1667ca..8f73b0c7c5 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/ResourceManagerPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/ResourceManagerPersistence.java @@ -43,6 +43,14 @@ public interface ResourceManagerPersistence { */ List getResourceByLabel(Label label) throws PersistenceErrorException; + /** + * + * @param labels + * @return + * @throws PersistenceErrorException + */ + List getResourceByLabels(List labels) throws PersistenceErrorException; + /** * 根据用户获取资源 * @param user @@ -107,4 +115,10 @@ public interface ResourceManagerPersistence { void lockResource(List labelIds,PersistenceResource persistenceResource); + void deleteResourceById(List id); + + void deleteResourceRelByResourceId(List id); + + PersistenceResource getNodeResourceById (Integer id); + } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLabelManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLabelManagerPersistence.java index b7806beede..69b598eabd 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLabelManagerPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLabelManagerPersistence.java @@ -16,9 +16,12 @@ */ package org.apache.linkis.manager.persistence.impl; + +import org.apache.commons.beanutils.BeanUtils; +import org.apache.commons.collections.CollectionUtils; import org.apache.linkis.common.ServiceInstance; import org.apache.linkis.manager.common.entity.persistence.PersistenceLabel; -import org.apache.linkis.manager.common.entity.persistence.PersistenceLock; +import org.apache.linkis.manager.common.entity.persistence.PersistenceLabelRel; import org.apache.linkis.manager.common.entity.persistence.PersistenceNode; import org.apache.linkis.manager.common.entity.persistence.PersistenceResource; import org.apache.linkis.manager.dao.LabelManagerMapper; @@ -26,11 +29,8 @@ import org.apache.linkis.manager.entity.Tunple; import org.apache.linkis.manager.exception.PersistenceWarnException; import org.apache.linkis.manager.label.entity.Label; -import org.apache.linkis.manager.label.utils.LabelUtils; import org.apache.linkis.manager.persistence.LabelManagerPersistence; import org.apache.linkis.manager.util.PersistenceUtils; -import org.apache.commons.beanutils.BeanUtils; -import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -62,6 +62,11 @@ public void setNodeManagerMapper(NodeManagerMapper nodeManagerMapper) { this.nodeManagerMapper = nodeManagerMapper; } + @Override + public List getLabelByPattern(String labelValuePattern, String labelKey, Integer page, Integer size) { + return labelManagerMapper.listLabelBySQLPattern(labelValuePattern, labelKey); + } + @Override public void addLabel(PersistenceLabel persistenceLabel) { labelManagerMapper.registerLabel(persistenceLabel); @@ -84,8 +89,8 @@ public void removeLabel(int id) { public void removeLabel(PersistenceLabel persistenceLabel) { String labelKey = persistenceLabel.getLabelKey(); String labelStringValue = persistenceLabel.getStringValue(); - int labelId = persistenceLabel.getId() ; - if (labelId <= 0 ) { + int labelId = persistenceLabel.getId(); + if (labelId <= 0) { PersistenceLabel labelByKeyValue = labelManagerMapper.getLabelByKeyValue(labelKey, labelStringValue); if (null == labelByKeyValue) { logger.warn("Can not find label labelKey {}, label value {}", labelKey, labelStringValue); @@ -109,8 +114,8 @@ public void updateLabel(int id, PersistenceLabel persistenceLabel) { persistenceLabel.setUpdateTime(new Date()); labelManagerMapper.updateLabel(id, persistenceLabel); labelManagerMapper.deleteLabelKeyVaules(id); - if(!persistenceLabel.getValue().isEmpty()){ - labelManagerMapper.registerLabelKeyValues(persistenceLabel.getValue(),id); + if (!persistenceLabel.getValue().isEmpty()) { + labelManagerMapper.registerLabelKeyValues(persistenceLabel.getValue(), id); } } @@ -134,7 +139,7 @@ public List getLabelByResource(PersistenceResource persistence @Override public void addLabelToNode(ServiceInstance serviceInstance, List labelIds) { - if(!CollectionUtils.isEmpty(labelIds)){ + if (!CollectionUtils.isEmpty(labelIds)) { labelManagerMapper.addLabelServiceInstance(serviceInstance.getInstance(), labelIds); } } @@ -248,10 +253,11 @@ public List getLabelsByUser(String userName) { public Map> getNodeRelationsByLabels(List persistenceLabels) { //空集合过滤& 转换 if (PersistenceUtils.persistenceLabelListIsEmpty(persistenceLabels)) return Collections.emptyMap(); - Map> keyValueMap = PersistenceUtils.filterEmptyPersistenceLabelList(persistenceLabels).stream().collect(Collectors.toMap(PersistenceLabel::getLabelKey, PersistenceLabel::getValue)); try { + /*Map> keyValueMap = PersistenceUtils.filterEmptyPersistenceLabelList(persistenceLabels).stream().collect(Collectors.toMap(PersistenceLabel::getLabelKey, PersistenceLabel::getValue)); String dimType = Label.ValueRelation.ALL.name(); - List> nodeRelationsByLabels = labelManagerMapper.dimListNodeRelationsByKeyValueMap(keyValueMap, dimType); + List> nodeRelationsByLabels = labelManagerMapper.dimListNodeRelationsByKeyValueMap(keyValueMap, dimType);*/ + List> nodeRelationsByLabels = labelManagerMapper.getNodeRelationsByLabels(persistenceLabels); List> arrays = new ArrayList>(); for (Map nodeRelationsByLabel : nodeRelationsByLabels) { ServiceInstance serviceInstance = new ServiceInstance(); @@ -276,12 +282,13 @@ public Map> getLabelRelationsByServiceIn List> nodeRelationsByLabels = labelManagerMapper.listLabelRelationByServiceInstance(serviceInstances); List> arrays = new ArrayList>(); for (Map nodeRelationsByLabel : nodeRelationsByLabels) { - ServiceInstance serviceInstance = new ServiceInstance(); + Optional instanceOption = serviceInstances.stream() + .filter(serviceInstance1 -> serviceInstance1.getInstance().equalsIgnoreCase(String.valueOf(nodeRelationsByLabel.get("instance")))) + .findFirst(); PersistenceLabel persistenceLabel = new PersistenceLabel(); - BeanUtils.populate(serviceInstance, nodeRelationsByLabel); BeanUtils.populate(persistenceLabel, nodeRelationsByLabel); PersistenceUtils.setValue(persistenceLabel); - arrays.add(new Tunple(serviceInstance, persistenceLabel)); + instanceOption.ifPresent(serviceInstance -> arrays.add(new Tunple(serviceInstance, persistenceLabel))); } return arrays.stream() .collect(Collectors.groupingBy(Tunple::getKey)).entrySet().stream() diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLockManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLockManagerPersistence.java index 348c227a40..3b72853db2 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLockManagerPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultLockManagerPersistence.java @@ -59,8 +59,8 @@ private boolean tryLock(PersistenceLock persistenceLock, Long timeOut){ lockManagerMapper.lock(persistenceLock.getLockObject(),timeOut); return true; } catch (DataAccessException e){ - logger.warn("Failed to obtain lock:" + persistenceLock.getLockObject()); - return false; + logger.warn("Failed to obtain lock:" + persistenceLock.getLockObject()); + return false; } } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeManagerPersistence.java index f91c6f2c64..38bd8e7e38 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeManagerPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeManagerPersistence.java @@ -74,7 +74,7 @@ public void addNodeInstance(Node node) throws PersistenceErrorException { try { nodeManagerMapper.addNodeInstance(persistenceNode); } catch (DuplicateKeyException e) { - NodeInstanceDuplicateException nodeInstanceDuplicateException = new NodeInstanceDuplicateException(41001, "the node instance is exist (Node实例已存在)"); + NodeInstanceDuplicateException nodeInstanceDuplicateException = new NodeInstanceDuplicateException(41001, "Node实例已存在"); nodeInstanceDuplicateException.initCause(e); throw nodeInstanceDuplicateException; } @@ -98,7 +98,7 @@ public void updateEngineNode(ServiceInstance serviceInstance, Node node) throws } catch (DuplicateKeyException e) { throw new LinkisRetryException(41003, "engine instance name is exist, request of created engine will be retry"); } catch (Exception e) { - NodeInstanceNotFoundException nodeInstanceNotFoundException = new NodeInstanceNotFoundException(41002, "the node instance is not exist (Node实例不存在)"); + NodeInstanceNotFoundException nodeInstanceNotFoundException = new NodeInstanceNotFoundException(41002, "Node实例不存在"); nodeInstanceNotFoundException.initCause(e); throw nodeInstanceNotFoundException; } @@ -110,7 +110,7 @@ public void removeNodeInstance(Node node) throws PersistenceErrorException { try { nodeManagerMapper.removeNodeInstance(instance); } catch (Exception e) { - NodeInstanceNotFoundException nodeInstanceNotFoundException = new NodeInstanceNotFoundException(41002, "the node instance is not exist (Node实例不存在)"); + NodeInstanceNotFoundException nodeInstanceNotFoundException = new NodeInstanceNotFoundException(41002, "Node实例不存在"); nodeInstanceNotFoundException.initCause(e); throw nodeInstanceNotFoundException; } @@ -151,6 +151,7 @@ public List getAllNodes() throws PersistenceErrorException { persistenceNodeEntity.setMark(persistenceNode.getMark()); persistenceNodeEntity.setOwner(persistenceNode.getOwner()); persistenceNodeEntity.setStartTime(persistenceNode.getCreateTime()); + persistenceNodeEntity.setUpdateTime(persistenceNode.getUpdateTime()); persistenceNodeEntitys.add(persistenceNodeEntity); } } @@ -195,6 +196,9 @@ public void addEngineNode(EngineNode engineNode) throws PersistenceErrorExceptio addNodeInstance(engineNode); //插入关联关系，todo 异常后续统一处理 String engineNodeInstance = engineNode.getServiceInstance().getInstance(); + if (null == engineNode.getEMNode()) { + throw new PersistenceErrorException(410002, " The emNode is null " + engineNode.getServiceInstance()); + } String emNodeInstance = engineNode.getEMNode().getServiceInstance().getInstance(); nodeManagerMapper.addEngineNode(engineNodeInstance,emNodeInstance); } @@ -202,9 +206,11 @@ public void addEngineNode(EngineNode engineNode) throws PersistenceErrorExceptio @Override public void deleteEngineNode(EngineNode engineNode) throws PersistenceErrorException { String engineNodeInstance = engineNode.getServiceInstance().getInstance(); - String emNodeInstance = engineNode.getEMNode().getServiceInstance().getInstance(); - //清理 engine和em 的关系表 - nodeManagerMapper.deleteEngineNode(engineNodeInstance,emNodeInstance); + if (null != engineNode.getEMNode()) { + String emNodeInstance = engineNode.getEMNode().getServiceInstance().getInstance(); + //清理 engine和em 的关系表 + nodeManagerMapper.deleteEngineNode(engineNodeInstance,emNodeInstance); + } //清理 metric信息 metricManagerMapper.deleteNodeMetricsByInstance(engineNodeInstance); //metricManagerMapper.deleteNodeMetrics(emNodeId); @@ -215,26 +221,29 @@ public void deleteEngineNode(EngineNode engineNode) throws PersistenceErrorExcep @Override public EngineNode getEngineNode(ServiceInstance serviceInstance) throws PersistenceErrorException { //给定引擎的 serviceinstance 查到 emNode - PersistenceNode emNode = nodeManagerMapper.getEMNodeInstanceByEngineNode(serviceInstance.getInstance()); - if (emNode == null) return null; - String emInstance = emNode.getInstance(); - String emName = emNode.getName(); - ServiceInstance emServiceInstance = new ServiceInstance(); - emServiceInstance.setApplicationName(emName); - emServiceInstance.setInstance(emInstance); - AMEMNode amemNode = new AMEMNode(); - amemNode.setMark(emNode.getMark()); - amemNode.setOwner(emNode.getOwner()); - amemNode.setServiceInstance(emServiceInstance); - amemNode.setStartTime(emNode.getCreateTime()); - AMEngineNode amEngineNode = new AMEngineNode(); amEngineNode.setServiceInstance(serviceInstance); PersistenceNode engineNode = nodeManagerMapper.getNodeInstance(serviceInstance.getInstance()); + if(null == engineNode) { + return null; + } amEngineNode.setOwner(engineNode.getOwner()); amEngineNode.setMark(engineNode.getMark()); amEngineNode.setStartTime(engineNode.getCreateTime()); - amEngineNode.setEMNode(amemNode); + PersistenceNode emNode = nodeManagerMapper.getEMNodeInstanceByEngineNode(serviceInstance.getInstance()); + if (emNode != null) { + String emInstance = emNode.getInstance(); + String emName = emNode.getName(); + ServiceInstance emServiceInstance = new ServiceInstance(); + emServiceInstance.setApplicationName(emName); + emServiceInstance.setInstance(emInstance); + AMEMNode amemNode = new AMEMNode(); + amemNode.setMark(emNode.getMark()); + amemNode.setOwner(emNode.getOwner()); + amemNode.setServiceInstance(emServiceInstance); + amemNode.setStartTime(emNode.getCreateTime()); + amEngineNode.setEMNode(amemNode); + } return amEngineNode; } @@ -242,7 +251,9 @@ public EngineNode getEngineNode(ServiceInstance serviceInstance) throws Persiste public List getEngineNodeByEM(ServiceInstance serviceInstance) throws PersistenceErrorException { //给定EM的 serviceinstance PersistenceNode emNode = nodeManagerMapper.getNodeInstance(serviceInstance.getInstance()); - + if (null == emNode) { + return new ArrayList<>(); + } List engineNodeList = nodeManagerMapper.getNodeInstances(serviceInstance.getInstance()); List amEngineNodeList =new ArrayList<>(); for (PersistenceNode engineNode : engineNodeList){ @@ -266,4 +277,5 @@ public List getEngineNodeByEM(ServiceInstance serviceInstance) throw } return amEngineNodeList; } + } diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeMetricManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeMetricManagerPersistence.java index 1c147b65ab..47daa17843 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeMetricManagerPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultNodeMetricManagerPersistence.java @@ -20,6 +20,7 @@ import org.apache.linkis.common.ServiceInstance; import org.apache.linkis.manager.common.entity.metrics.NodeMetrics; import org.apache.linkis.manager.common.entity.node.Node; +import org.apache.linkis.manager.common.entity.persistence.PersistenceNode; import org.apache.linkis.manager.common.entity.persistence.PersistenceNodeMetrics; import org.apache.linkis.manager.common.entity.persistence.PersistenceNodeMetricsEntity; import org.apache.linkis.manager.dao.NodeManagerMapper; @@ -27,6 +28,8 @@ import org.apache.linkis.manager.exception.PersistenceErrorException; import org.apache.linkis.manager.persistence.NodeMetricManagerPersistence; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Collections; import java.util.Date; @@ -35,6 +38,8 @@ public class DefaultNodeMetricManagerPersistence implements NodeMetricManagerPersistence { + private static Logger logger = LoggerFactory.getLogger(DefaultNodeMetricManagerPersistence.class); + private NodeManagerMapper nodeManagerMapper; private NodeMetricManagerMapper nodeMetricManagerMapper; @@ -75,6 +80,11 @@ public void addOrupdateNodeMetrics(NodeMetrics nodeMetrics) throws PersistenceEr PersistenceNodeMetrics persistenceNodeMetrics = new PersistenceNodeMetrics(); String instance = nodeMetrics.getServiceInstance().getInstance(); //todo 异常信息后面统一处理 + PersistenceNode node = nodeManagerMapper.getNodeInstance(instance); + if(node == null){ + logger.warn("The request of update node metrics was ignored, because the node " + instance + " is not exist."); + return; + } int isInstanceIdExist = nodeMetricManagerMapper.checkInstanceExist(instance); //是否存在 if (isInstanceIdExist == 0) { @@ -93,7 +103,6 @@ public void addOrupdateNodeMetrics(NodeMetrics nodeMetrics) throws PersistenceEr persistenceNodeMetrics.setHeartBeatMsg(nodeMetrics.getHeartBeatMsg()); persistenceNodeMetrics.setOverLoad(nodeMetrics.getOverLoad()); persistenceNodeMetrics.setStatus(nodeMetrics.getStatus()); - persistenceNodeMetrics.setCreateTime(new Date()); persistenceNodeMetrics.setUpdateTime(new Date()); nodeMetricManagerMapper.updateNodeMetrics(persistenceNodeMetrics, instance); } else { @@ -139,7 +148,7 @@ public NodeMetrics getNodeMetrics(Node node) throws PersistenceErrorException { @Override public void deleteNodeMetrics(Node node) throws PersistenceErrorException { String instance = node.getServiceInstance().getInstance(); - nodeMetricManagerMapper.deleteNodeMetrics(instance); + nodeMetricManagerMapper.deleteNodeMetricsByInstance(instance); } @Override diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultResourceLabelPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultResourceLabelPersistence.java index f1e4bb5101..8b46b17ec7 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultResourceLabelPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultResourceLabelPersistence.java @@ -23,6 +23,7 @@ import org.apache.linkis.manager.dao.LabelManagerMapper; import org.apache.linkis.manager.dao.ResourceManagerMapper; import org.apache.linkis.manager.entity.Tunple; +import org.apache.linkis.manager.exception.PersistenceErrorException; import org.apache.linkis.manager.label.entity.Label; import org.apache.linkis.manager.persistence.ResourceLabelPersistence; import org.apache.linkis.manager.util.PersistenceUtils; @@ -82,11 +83,9 @@ public List getResourceLabels(Map> } @Override - // @Transactional(rollbackFor = Throwable.class) + // @Transactional(rollbackFor = Throwable.class) public void setResourceToLabel(PersistenceLabel label, PersistenceResource persistenceResource) { if (label == null || persistenceResource == null) return; - - // TODO: 2020/8/31 id 的包装类 if (label.getId() <= 0) { if (StringUtils.isEmpty(label.getStringValue())) return; PersistenceLabel resourceLabel = labelManagerMapper.getLabelByKeyValue(label.getLabelKey(), label.getStringValue()); @@ -97,12 +96,6 @@ public void setResourceToLabel(PersistenceLabel label, PersistenceResource persi } } - //插入resource和relation记录 - persistenceResource.setCreator(System.getProperty("user.name")); - persistenceResource.setUpdator(System.getProperty("user.name")); - - //找到label对应的persistenceResourceId，不存在就插入，存在就更新 - // TODO: 2020/9/8 多resource的判断，persistenceResource 有id的话，直接update这个 List resourceByLabels = labelManagerMapper.listResourceByLaBelId(label.getId()); if (CollectionUtils.isNotEmpty(resourceByLabels)) { if(resourceByLabels.size() > 1){ @@ -111,8 +104,14 @@ public void setResourceToLabel(PersistenceLabel label, PersistenceResource persi PersistenceResource resourceToUpdate = Iterables.getFirst(resourceByLabels, null); resourceManagerMapper.nodeResourceUpdateByResourceId(resourceToUpdate.getId(), persistenceResource); } else { - resourceManagerMapper.registerResource(persistenceResource); - labelManagerMapper.addLabelsAndResource(persistenceResource.getId(), Collections.singletonList(label.getId())); + synchronized (this){ + List resources = labelManagerMapper.listResourceByLaBelId(label.getId()); + if(CollectionUtils.isEmpty(resources)) { + persistenceResource.setCreator(persistenceResource.getUpdator()); + resourceManagerMapper.registerResource(persistenceResource); + labelManagerMapper.addLabelsAndResource(persistenceResource.getId(), Collections.singletonList(label.getId())); + } + } } } @@ -139,12 +138,15 @@ public List getResourceByLabel(PersistenceLabel label) { } @Override - // @Transactional(rollbackFor = Throwable.class) - public void removeResourceByLabel(PersistenceLabel label) { + // @Transactional(rollbackFor = Throwable.class) + public void removeResourceByLabel(PersistenceLabel label) throws PersistenceErrorException { //label id 不为空，则直接通过label_id 查询，否则通过 value_key and value_content 查询 int labelId = label.getId() ; if (labelId <= 0 ) { PersistenceLabel labelByKeyValue = labelManagerMapper.getLabelByKeyValue(label.getLabelKey(), label.getStringValue()); + if(labelByKeyValue == null){ + throw new PersistenceErrorException(210001, "label not found, this label may be removed already."); + } labelId = labelByKeyValue.getId(); } if (labelId > 0) { diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultResourceManagerPersistence.java b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultResourceManagerPersistence.java index fb51070756..d1a1564732 100644 --- a/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultResourceManagerPersistence.java +++ b/linkis-computation-governance/linkis-manager/linkis-manager-persistence/src/main/java/org/apache/linkis/manager/persistence/impl/DefaultResourceManagerPersistence.java @@ -26,7 +26,9 @@ import org.apache.linkis.manager.exception.PersistenceErrorException; import org.apache.linkis.manager.label.entity.Label; import org.apache.linkis.manager.persistence.ResourceManagerPersistence; +import org.apache.commons.collections.CollectionUtils; +import java.util.ArrayList; import java.util.List; @@ -91,6 +93,15 @@ public List getResourceByLabel(Label label) throws Persiste return persistenceResourceList; } + @Override + public List getResourceByLabels(List labels) throws PersistenceErrorException { + if(CollectionUtils.isNotEmpty(labels)) { + return labelManagerMapper.getResourcesByLabels(labels); + }else{ + return new ArrayList(); + } + } + @Override public List getResourceByUser(String user) throws PersistenceErrorException { List persistenceResourceList = resourceManagerMapper.getResourceByUserName(user); @@ -154,4 +165,20 @@ public void lockResource(List labelIds, PersistenceResource persistence int resourceId = persistenceResource.getId(); labelManagerMapper.addLabelsAndResource(resourceId,labelIds); } + + @Override + public void deleteResourceById(List id) { + resourceManagerMapper.deleteResourceById(id); + } + + @Override + public void deleteResourceRelByResourceId(List id) { + resourceManagerMapper.deleteResourceRelByResourceId(id); + } + + @Override + public PersistenceResource getNodeResourceById(Integer id) { + PersistenceResource resource = resourceManagerMapper.getResourceById(id); + return resource; + } } diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/assembly/distribution.xml b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/assembly/distribution.xml deleted file mode 100644 index ce1bca2fb9..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/assembly/distribution.xml +++ /dev/null @@ -1,279 +0,0 @@ - - - - - linkis-resourcemanager - - zip - - true - linkis-resourcemanager - - - - - - lib - true - true - false - false - true - - - - aopalliance:aopalliance:jar - com.fasterxml.jackson.core:jackson-annotations:jar - com.fasterxml.jackson.core:jackson-core:jar - com.fasterxml.jackson.core:jackson-databind:jar - com.fasterxml.jackson.module:jackson-module-paranamer:jar - com.fasterxml.jackson.module:jackson-module-scala_2.11:jar - com.google.code.findbugs:jsr305:jar - com.google.code.gson:gson:jar - com.google.guava:guava:jar - com.google.protobuf:protobuf-java:jar - com.netflix.archaius:archaius-core:jar - com.netflix.hystrix:hystrix-core:jar - com.netflix.netflix-commons:netflix-commons-util:jar - com.netflix.netflix-commons:netflix-statistics:jar - com.netflix.ribbon:ribbon:jar - com.netflix.ribbon:ribbon-core:jar - com.netflix.ribbon:ribbon-httpclient:jar - com.netflix.ribbon:ribbon-loadbalancer:jar - com.netflix.ribbon:ribbon-transport:jar - com.netflix.servo:servo-core:jar - com.ning:async-http-client:jar - com.sun.jersey.contribs:jersey-apache-client4:jar - com.sun.jersey:jersey-client:jar - com.sun.jersey:jersey-core:jar - com.sun.jersey:jersey-json:jar - com.sun.jersey:jersey-server:jar - com.sun.xml.bind:jaxb-impl:jar - com.thoughtworks.paranamer:paranamer:jar - org.apache.linkis:linkis-common:jar - commons-beanutils:commons-beanutils:jar - commons-beanutils:commons-beanutils-core:jar - commons-cli:commons-cli:jar - commons-collections:commons-collections:jar - commons-configuration:commons-configuration:jar - commons-daemon:commons-daemon:jar - commons-digester:commons-digester:jar - commons-httpclient:commons-httpclient:jar - commons-io:commons-io:jar - commons-lang:commons-lang:jar - commons-logging:commons-logging:jar - commons-net:commons-net:jar - - io.netty:netty-all:jar - io.netty:netty-buffer:jar - io.netty:netty-codec:jar - io.netty:netty-codec-http:jar - io.netty:netty-common:jar - io.netty:netty-handler:jar - io.netty:netty-transport:jar - io.netty:netty-transport-native-epoll:jar - io.reactivex:rxjava:jar - io.reactivex:rxnetty:jar - io.reactivex:rxnetty-contexts:jar - io.reactivex:rxnetty-servo:jar - javax.activation:activation:jar - javax.annotation:javax.annotation-api:jar - javax.inject:javax.inject:jar - javax.servlet:javax.servlet-api:jar - javax.servlet.jsp:jsp-api:jar - javax.xml.bind:jaxb-api:jar - javax.xml.stream:stax-api:jar - log4j:log4j:jar - net.databinder.dispatch:dispatch-core_2.11:jar - net.databinder.dispatch:dispatch-json4s-jackson_2.11:jar - org.apache.commons:commons-compress:jar - org.apache.commons:commons-math3:jar - org.apache.curator:curator-client:jar - org.apache.curator:curator-framework:jar - org.apache.curator:curator-recipes:jar - org.apache.directory.api:api-asn1-api:jar - org.apache.directory.api:api-util:jar - org.apache.directory.server:apacheds-i18n:jar - org.apache.directory.server:apacheds-kerberos-codec:jar - org.apache.hadoop:hadoop-annotations:jar - org.apache.hadoop:hadoop-auth:jar - org.apache.hadoop:hadoop-common:jar - org.apache.hadoop:hadoop-hdfs:jar - org.apache.htrace:htrace-core:jar - org.apache.httpcomponents:httpclient:jar - org.apache.httpcomponents:httpcore:jar - org.apache.logging.log4j:log4j-api:jar - org.apache.zookeeper:zookeeper:jar - org.aspectj:aspectjweaver:jar - org.bouncycastle:bcpkix-jdk15on:jar - org.bouncycastle:bcprov-jdk15on:jar - org.codehaus.jackson:jackson-jaxrs:jar - org.codehaus.jackson:jackson-xc:jar - org.codehaus.jettison:jettison:jar - org.eclipse.jetty:jetty-continuation:jar - org.eclipse.jetty:jetty-http:jar - org.eclipse.jetty:jetty-io:jar - org.eclipse.jetty:jetty-jndi:jar - org.eclipse.jetty:jetty-plus:jar - org.eclipse.jetty:jetty-security:jar - org.eclipse.jetty:jetty-server:jar - org.eclipse.jetty:jetty-servlet:jar - org.eclipse.jetty:jetty-util:jar - org.eclipse.jetty:jetty-webapp:jar - org.eclipse.jetty:jetty-xml:jar - org.fusesource.leveldbjni:leveldbjni-all:jar - org.hdrhistogram:HdrHistogram:jar - - - - org.jsoup:jsoup:jar - org.mortbay.jetty:jetty:jar - org.mortbay.jetty:jetty-util:jar - org.scala-lang.modules:scala-parser-combinators_2.11:jar - org.scala-lang.modules:scala-xml_2.11:jar - org.scala-lang:scala-compiler:jar - org.scala-lang:scala-library:jar - org.scala-lang:scala-reflect:jar - org.scala-lang:scalap:jar - org.slf4j:jul-to-slf4j:jar - org.slf4j:slf4j-api:jar - org.springframework.boot:spring-boot:jar - org.springframework.boot:spring-boot-autoconfigure:jar - org.springframework.boot:spring-boot-starter:jar - org.springframework.boot:spring-boot-starter-aop:jar - org.springframework.cloud:spring-cloud-commons:jar - org.springframework.cloud:spring-cloud-context:jar - org.springframework.cloud:spring-cloud-netflix-archaius:jar - org.springframework.cloud:spring-cloud-netflix-ribbon:jar - org.springframework.cloud:spring-cloud-starter:jar - org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar - org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar - org.springframework.security:spring-security-crypto:jar - org.springframework.security:spring-security-rsa:jar - org.springframework:spring-aop:jar - org.springframework:spring-beans:jar - org.springframework:spring-context:jar - org.springframework:spring-expression:jar - org.springframework:spring-web:jar - org.tukaani:xz:jar - org.yaml:snakeyaml:jar - xerces:xercesImpl:jar - xml-apis:xml-apis:jar - xmlenc:xmlenc:jar - - - - - - - ${basedir}/src/main/resources - - * - - 0777 - conf - unix - - - ${basedir}/bin - - * - - 0777 - bin - unix - - - . - - */** - - logs - - - - - diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/java/org/apache/linkis/resourcemanager/domain/RMLabelContainer.java b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/java/org/apache/linkis/resourcemanager/domain/RMLabelContainer.java index 1311fa7866..1c98b17719 100644 --- a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/java/org/apache/linkis/resourcemanager/domain/RMLabelContainer.java +++ b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/java/org/apache/linkis/resourcemanager/domain/RMLabelContainer.java @@ -62,7 +62,7 @@ public RMLabelContainer(List> labels) { }catch (Exception e){ logger.warn("failed to get combinedUserCreatorEngineTypeLabel", e); } - this.labels = LabelUtils.distinctLabel(this.labels, this.labels); + this.labels = LabelUtils.distinctLabel(this.labels, labels); } public List> getLabels() { @@ -102,9 +102,9 @@ public EMInstanceLabel getEMInstanceLabel() throws RMErrorException { public EngineTypeLabel getEngineTypeLabel() throws RMErrorException { if(engineTypeLabel == null){ for (Label label : labels) { - if(label instanceof EngineTypeLabel){ - return (EngineTypeLabel) label; - } + if(label instanceof EngineTypeLabel){ + return (EngineTypeLabel) label; + } } } else { return engineTypeLabel; diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/java/org/apache/linkis/resourcemanager/exception/RMErrorCode.java b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/java/org/apache/linkis/resourcemanager/exception/RMErrorCode.java new file mode 100644 index 0000000000..cc8c9c29dd --- /dev/null +++ b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/java/org/apache/linkis/resourcemanager/exception/RMErrorCode.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.resourcemanager.exception; + + + +public enum RMErrorCode { + + /** + * + */ + LABEL_RESOURCE_NOT_FOUND(110021, "label resource not found, please check!"), + + LOCK_LABEL_FAILED(110022, "lock label failed!"), + + LABEL_DUPLICATED(110019, "label has exist"), + + DRIVER_MEMORY_INSUFFICIENT(12004, "Drive memory resources are insufficient, to reduce the drive memory(内存资源不足，建议调小驱动内存)"), + + DRIVER_CPU_INSUFFICIENT(12005, "Drive core resources are insufficient, to reduce the number of driver cores(CPU资源不足，建议调小驱动核数)"), + + INSTANCES_INSUFFICIENT(12006, "Insufficient number of instances, idle engines can be killed(实例数不足，可以kill空闲的引擎)"), + + QUEUE_MEMORY_INSUFFICIENT(12002, "Insufficient queue memory resources, reduce the executor memory"), + + QUEUE_CPU_INSUFFICIENT (12001, "Queue CPU resources are insufficient, reduce the number of executors.(队列CPU资源不足，建议调小执行器个数)"), + + QUEUE_INSTANCES_INSUFFICIENT (12003, "Insufficient number of queue instances, idle engines can be killed(队列实例数不足，可以kill空闲的引擎)"), + + CLUSTER_QUEUE_MEMORY_INSUFFICIENT(12010, "Insufficient cluster queue memory"), + + CLUSTER_QUEUE_CPU_INSUFFICIENT (12011, "Insufficient cluster queue cpu"), + + CLUSTER_QUEUE_INSTANCES_INSUFFICIENT (12012, "Insufficient cluster queue instance"), + + ECM_RESOURCE_INSUFFICIENT(11000, "ECM resources are insufficient(ECM 资源不足)"), + + ECM_MEMORY_INSUFFICIENT(11001, "ECM memory resources are insufficient(ECM 内存资源不足)"), + + ECM_CPU_INSUFFICIENT(11002, "ECM CPU resources are insufficient(ECM CPU资源不足)"), + + ECM_INSTANCES_INSUFFICIENT(11003, "ECM Insufficient number of instances(ECM实例数不足)"), + ; + + + RMErrorCode(int errorCode, String message) { + this.code = errorCode; + this.message = message; + } + + private int code; + + private String message; + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } +} diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/java/org/apache/linkis/resourcemanager/external/service/impl/ExternalResourceServiceImpl.java b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/java/org/apache/linkis/resourcemanager/external/service/impl/ExternalResourceServiceImpl.java index a5a740940c..42202e132b 100644 --- a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/java/org/apache/linkis/resourcemanager/external/service/impl/ExternalResourceServiceImpl.java +++ b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/java/org/apache/linkis/resourcemanager/external/service/impl/ExternalResourceServiceImpl.java @@ -17,6 +17,10 @@ package org.apache.linkis.resourcemanager.external.service.impl; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; + import org.apache.linkis.manager.common.entity.resource.NodeResource; import org.apache.linkis.manager.common.entity.resource.ResourceType; import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext; @@ -44,6 +48,8 @@ import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.function.Function; @Component @@ -58,6 +64,17 @@ public class ExternalResourceServiceImpl implements ExternalResourceService, Ini ExternalResourceRequester[] resourceRequesters; + private LoadingCache> providerCache = CacheBuilder.newBuilder().maximumSize(20) + .expireAfterAccess(1, TimeUnit.HOURS) + .refreshAfterWrite(RMUtils.EXTERNAL_RESOURCE_REFRESH_TIME().getValue().toLong(), TimeUnit.MINUTES) + .build( new CacheLoader>() { + + @Override + public List load(String resourceType) { + return providerDao.selectByResourceType(resourceType); + } + }); + @Override public void afterPropertiesSet() throws Exception { resourceRequesters = new ExternalResourceRequester[]{ @@ -105,7 +122,7 @@ private Object retry(int retryNum, Function function) throws RMErrorException { return function.apply(null); } catch (Exception e){ errorMsg = "Failed to request external resource" + ExceptionUtils.getRootCauseMessage(e); - logger.warn("failed to request external resource provider", e); + logger.warn("failed to request external resource provider, retryNum {}", times, e); times ++; } } @@ -122,11 +139,15 @@ private ExternalResourceProvider chooseProvider(ResourceType resourceType, RMLab } else { realClusterLabel = (ClusterLabel) label; } - List providers = providerDao.selectByResourceType(resourceType.toString()); - for (ExternalResourceProvider provider : providers) { - if (provider.getName().equals(realClusterLabel.getClusterName())) { - return provider; + try { + List providers = providerCache.get(resourceType.toString()); + for (ExternalResourceProvider provider : providers) { + if (provider.getName().equals(realClusterLabel.getClusterName())) { + return provider; + } } + } catch (ExecutionException e) { + throw new RMErrorException(110013, "No suitable ExternalResourceProvider found for cluster: " + realClusterLabel.getClusterName(), e); } throw new RMErrorException(110013, "No suitable ExternalResourceProvider found for cluster: " + realClusterLabel.getClusterName()); } diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/RMReceiver.scala b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/RMReceiver.scala deleted file mode 100644 index 1c32ce2e31..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/RMReceiver.scala +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.resourcemanager - -import org.apache.linkis.common.ServiceInstance -import org.apache.linkis.common.utils.Logging -import org.apache.linkis.manager.common.entity.node.EMNode -import org.apache.linkis.resourcemanager.service.ResourceManager -import org.apache.linkis.rpc.{Receiver, Sender} -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Component - -import scala.concurrent.duration.Duration - -class RMReceiver extends Receiver with Logging { - - @Autowired - var rm: ResourceManager = _ - - override def receive(message: Any, sender: Sender): Unit = message match { - case eMNode: EMNode => rm.register(eMNode.getServiceInstance, eMNode.getNodeResource) - case serviceInstance: ServiceInstance => rm.unregister(serviceInstance) - case ResourceInited(labels, engineResource) => rm.resourceUsed(labels, engineResource) - case ResourceReleased(labels) => rm.resourceReleased(labels) - } - - override def receiveAndReply(message: Any, sender: Sender): Any = message match { - case RequestResource(labels, resource) => rm.requestResource(labels, resource) - case RequestResourceAndWait(labels, resource, waitTime) => rm.requestResource(labels, resource, waitTime) - case RequestResourceInfo(serviceInstances) => rm.getResourceInfo(serviceInstances) - } - - override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = null -} diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/message/RMMessageService.scala b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/message/RMMessageService.scala index a823176375..472753970d 100644 --- a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/message/RMMessageService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/message/RMMessageService.scala @@ -17,9 +17,9 @@ package org.apache.linkis.resourcemanager.message -import org.apache.linkis.common.utils.Logging +import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.manager.common.entity.enumeration.NodeStatus -import org.apache.linkis.manager.common.protocol.em.{EMInfoClearRequest, RegisterEMRequest, StopEMRequest} +import org.apache.linkis.manager.common.protocol.em.{EMInfoClearRequest, EMResourceRegisterRequest, RegisterEMRequest, StopEMRequest} import org.apache.linkis.manager.common.protocol.engine.EngineInfoClearRequest import org.apache.linkis.manager.common.protocol.node.NodeHeartbeatMsg import org.apache.linkis.manager.common.protocol.resource.ResourceUsedProtocol @@ -27,6 +27,7 @@ import org.apache.linkis.manager.label.service.NodeLabelService import org.apache.linkis.manager.service.common.label.ManagerLabelService import org.apache.linkis.message.annotation.Receiver import org.apache.linkis.message.builder.ServiceMethodContext +import org.apache.linkis.resourcemanager.domain.RMLabelContainer import org.apache.linkis.resourcemanager.service.ResourceManager import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component @@ -45,7 +46,7 @@ class RMMessageService extends Logging { @Autowired private var nodeLabelService: NodeLabelService = _ - @Receiver + def dealWithEMInfoClearRequest(eMInfoClearRequest: EMInfoClearRequest, smc: ServiceMethodContext): Unit = { resourceManager.unregister(eMInfoClearRequest.getEm.getServiceInstance) } @@ -55,7 +56,7 @@ class RMMessageService extends Logging { resourceManager.resourceReleased(engineInfoClearRequest.getEngineNode.getLabels) } - @Receiver + def dealWithNodeHeartbeatMsg(nodeHeartbeatMsg: NodeHeartbeatMsg, smc: ServiceMethodContext): Unit = { debug(s"Start to deal with nodeHeartbeatMsg resource info $nodeHeartbeatMsg") /*val labels = nodeLabelService.getNodeLabels(nodeHeartbeatMsg.getServiceInstance) @@ -82,16 +83,23 @@ class RMMessageService extends Logging { def dealWithResourceUsedProtocol(resourceUsedProtocol: ResourceUsedProtocol): Unit = { info(s"Start to deal with resourceUsedProtocol $resourceUsedProtocol") val labels = nodeLabelService.getNodeLabels(resourceUsedProtocol.serviceInstance) - resourceManager.resourceUsed(labels, resourceUsedProtocol.engineResource) + Utils.tryCatch(resourceManager.resourceUsed(labels, resourceUsedProtocol.engineResource)) { + case exception: Exception => { + val nodeLabels = new RMLabelContainer(labels) + warn(s"usedResource failed, request from:${nodeLabels.getCombinedUserCreatorEngineTypeLabel.getStringValue}, request engine: ${nodeLabels.getEngineInstanceLabel}, " + + s"reason:${exception.getMessage}") + throw exception + } + } } - @Receiver - def dealWithRegisterEMRequest(registerEMRequest: RegisterEMRequest, smc: ServiceMethodContext): Unit = { + + def dealWithRegisterEMRequest(registerEMRequest: EMResourceRegisterRequest): Unit = { resourceManager.register(registerEMRequest.getServiceInstance, registerEMRequest.getNodeResource) } - def dealWithStopEMRequest(stopEMRequest: StopEMRequest, smc: ServiceMethodContext): Unit = { + def dealWithStopEMRequest(stopEMRequest: StopEMRequest): Unit = { resourceManager.unregister(stopEMRequest.getEm) } diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/restful/RMMonitorRest.scala b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/restful/RMMonitorRest.scala index 3df7cd29ef..9da36345a0 100644 --- a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/restful/RMMonitorRest.scala +++ b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/restful/RMMonitorRest.scala @@ -14,22 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package org.apache.linkis.resourcemanager.restful -import java.text.SimpleDateFormat -import java.util -import java.util.concurrent.TimeUnit -import java.util.{Comparator, TimeZone} +package org.apache.linkis.resourcemanager.restful import com.fasterxml.jackson.databind.ObjectMapper import com.google.common.collect.Lists -import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.manager.common.entity.node.EngineNode import org.apache.linkis.manager.common.entity.resource._ -import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest import org.apache.linkis.manager.common.serializer.NodeResourceSerializer import org.apache.linkis.manager.common.utils.ResourceUtils import org.apache.linkis.manager.label.builder.CombinedLabelBuilder @@ -37,22 +30,24 @@ import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContex import org.apache.linkis.manager.label.entity.cluster.ClusterLabel import org.apache.linkis.manager.label.entity.engine.{EngineInstanceLabel, EngineTypeLabel, UserCreatorLabel} import org.apache.linkis.manager.label.service.NodeLabelService -import org.apache.linkis.manager.persistence.{NodeManagerPersistence, NodeMetricManagerPersistence, ResourceManagerPersistence} +import org.apache.linkis.manager.persistence.{LabelManagerPersistence, NodeManagerPersistence, NodeMetricManagerPersistence, ResourceManagerPersistence} import org.apache.linkis.manager.service.common.metrics.MetricsConverter -import org.apache.linkis.message.publisher.MessagePublisher import org.apache.linkis.resourcemanager.domain.RMLabelContainer import org.apache.linkis.resourcemanager.external.service.ExternalResourceService import org.apache.linkis.resourcemanager.external.yarn.{YarnAppInfo, YarnResourceIdentifier} -import org.apache.linkis.resourcemanager.service.LabelResourceService -import org.apache.linkis.resourcemanager.utils.{RMConfiguration, RMUtils, UserConfiguration} -import org.apache.linkis.server.Message +import org.apache.linkis.resourcemanager.service.{LabelResourceService, ResourceManager} +import org.apache.linkis.resourcemanager.utils.{RMConfiguration, UserConfiguration} import org.apache.linkis.server.security.SecurityFilter -import javax.servlet.http.HttpServletRequest +import org.apache.linkis.server.{BDPJettyServerHelper, Message} import org.json4s.DefaultFormats import org.json4s.jackson.Serialization.write import org.springframework.beans.factory.annotation.Autowired import org.springframework.web.bind.annotation.{RequestBody, RequestMapping, RequestMethod, RestController} +import java.text.SimpleDateFormat +import java.util +import java.util.{Comparator, TimeZone} +import javax.servlet.http.HttpServletRequest import scala.collection.JavaConversions._ import scala.collection.mutable import scala.collection.mutable.ArrayBuffer @@ -68,6 +63,10 @@ class RMMonitorRest extends Logging { dateFormat.setTimeZone(TimeZone.getTimeZone("GMT")) val labelFactory = LabelBuilderFactoryContext.getLabelBuilderFactory val combinedLabelBuilder = new CombinedLabelBuilder + val gson = BDPJettyServerHelper.gson + + @Autowired + var labelManagerPersistence: LabelManagerPersistence = _ @Autowired private var nodeLabelService: NodeLabelService = _ @@ -91,42 +90,45 @@ class RMMonitorRest extends Logging { var labelResourceService: LabelResourceService = _ @Autowired - private var messagePublisher: MessagePublisher = _ + var resourceManager: ResourceManager = _ - def appendMessageData(message: Message, key: String, value: AnyRef) = message.data(key, mapper.readTree(write(value))) + var COMBINED_USERCREATOR_ENGINETYPE: String = _ - @RequestMapping(path =Array("applicationlist"),method = Array(RequestMethod.POST)) - def getApplicationList(request: HttpServletRequest,@RequestBody param: util.Map[String, AnyRef]): Message = { + def appendMessageData(message: Message, key: String, value: AnyRef): Message = message.data(key, mapper.readTree(write(value))) + + @RequestMapping(path = Array("applicationlist"), method = Array(RequestMethod.POST)) + def getApplicationList(request: HttpServletRequest, @RequestBody param: util.Map[String, AnyRef]): Message = { val message = Message.ok("") val userName = SecurityFilter.getLoginUsername(request) val userCreator = param.get("userCreator").asInstanceOf[String] - val engineType = if(param.get("engineType") == null) null else param.get("engineType").asInstanceOf[String] + val engineType = if (param.get("engineType") == null) null else param.get("engineType").asInstanceOf[String] val nodes = getEngineNodes(userName, true) val creatorToApplicationList = new mutable.HashMap[String, mutable.HashMap[String, Any]] nodes.foreach{ node => val userCreatorLabel = node.getLabels.find(_.isInstanceOf[UserCreatorLabel]).get.asInstanceOf[UserCreatorLabel] val engineTypeLabel = node.getLabels.find(_.isInstanceOf[EngineTypeLabel]).get.asInstanceOf[EngineTypeLabel] - if(getUserCreator(userCreatorLabel).equals(userCreator)){ - if(engineType == null || getEngineType(engineTypeLabel).equals(engineType)){ - if(!creatorToApplicationList.contains(userCreatorLabel.getCreator)){ + if (getUserCreator(userCreatorLabel).equals(userCreator)) { + if (engineType == null || getEngineType(engineTypeLabel).equals(engineType)) { + if (!creatorToApplicationList.contains(userCreatorLabel.getCreator)) { val applicationList = new mutable.HashMap[String, Any] applicationList.put("engineInstances", new mutable.ArrayBuffer[Any]) applicationList.put("usedResource", Resource.initResource(ResourceType.LoadInstance)) applicationList.put("maxResource", Resource.initResource(ResourceType.LoadInstance)) applicationList.put("minResource", Resource.initResource(ResourceType.LoadInstance)) + applicationList.put("lockedResource", Resource.initResource(ResourceType.LoadInstance)) creatorToApplicationList.put(userCreatorLabel.getCreator, applicationList) } - val applicationList = creatorToApplicationList.get(userCreatorLabel.getCreator).get - applicationList.put("usedResource", applicationList.get("usedResource").get.asInstanceOf[Resource] + node.getNodeResource.getUsedResource) - applicationList.put("maxResource", applicationList.get("maxResource").get.asInstanceOf[Resource] + node.getNodeResource.getMaxResource) - applicationList.put("minResource", applicationList.get("minResource").get.asInstanceOf[Resource] + node.getNodeResource.getMinResource) + val applicationList = creatorToApplicationList(userCreatorLabel.getCreator) + applicationList.put("usedResource", applicationList("usedResource").asInstanceOf[Resource] + node.getNodeResource.getUsedResource) + applicationList.put("maxResource", applicationList("maxResource").asInstanceOf[Resource] + node.getNodeResource.getMaxResource) + applicationList.put("minResource", applicationList("minResource").asInstanceOf[Resource] + node.getNodeResource.getMinResource) val engineInstance = new mutable.HashMap[String, Any] engineInstance.put("creator", userCreatorLabel.getCreator) engineInstance.put("engineType", engineTypeLabel.getEngineType) engineInstance.put("instance", node.getServiceInstance.getInstance) engineInstance.put("label", engineTypeLabel.getStringValue) engineInstance.put("resource", node.getNodeResource) - if(node.getNodeStatus == null){ + if (node.getNodeStatus == null) { engineInstance.put("status", "Busy") } else { engineInstance.put("status", node.getNodeStatus.toString) @@ -147,8 +149,8 @@ class RMMonitorRest extends Logging { message } - @RequestMapping(path = Array("userresources"),method =Array( RequestMethod.POST)) - def getUserResource(request: HttpServletRequest,@RequestBody(required=false) param: util.Map[String, AnyRef]): Message = { + @RequestMapping(path = Array("userresources"), method = Array(RequestMethod.POST)) + def getUserResource(request: HttpServletRequest, @RequestBody(required = false) param: util.Map[String, AnyRef]): Message = { val message = Message.ok("") val userName = SecurityFilter.getLoginUsername(request) var nodes = getEngineNodes(userName, true) @@ -162,28 +164,28 @@ class RMMonitorRest extends Logging { node.getLabels.find(_.isInstanceOf[EngineTypeLabel]).get != null }) } - val userCreatorEngineTypeResourceMap =new mutable.HashMap[String, mutable.HashMap[String, NodeResource]] + val userCreatorEngineTypeResourceMap = new mutable.HashMap[String, mutable.HashMap[String, NodeResource]] nodes.foreach { node => val userCreatorLabel = node.getLabels.find(_.isInstanceOf[UserCreatorLabel]).get.asInstanceOf[UserCreatorLabel] val engineTypeLabel = node.getLabels.find(_.isInstanceOf[EngineTypeLabel]).get.asInstanceOf[EngineTypeLabel] val userCreator = getUserCreator(userCreatorLabel) - if(!userCreatorEngineTypeResourceMap.contains(userCreator)){ + if (!userCreatorEngineTypeResourceMap.contains(userCreator)) { userCreatorEngineTypeResourceMap.put(userCreator, new mutable.HashMap[String, NodeResource]) } - val engineTypeResourceMap = userCreatorEngineTypeResourceMap.get(userCreator).get + val engineTypeResourceMap = userCreatorEngineTypeResourceMap(userCreator) val engineType = getEngineType(engineTypeLabel) - if(!engineTypeResourceMap.contains(engineType)){ + if (!engineTypeResourceMap.contains(engineType)) { val nodeResource = CommonNodeResource.initNodeResource(ResourceType.LoadInstance) engineTypeResourceMap.put(engineType, nodeResource) } - val resource = engineTypeResourceMap.get(engineType).get + val resource = engineTypeResourceMap(engineType) resource.setUsedResource(node.getNodeResource.getUsedResource + resource.getUsedResource) //combined label val combinedLabel = combinedLabelBuilder.build("", Lists.newArrayList(userCreatorLabel, engineTypeLabel)); var labelResource = labelResourceService.getLabelResource(combinedLabel) - if(labelResource == null){ + if (labelResource == null) { resource.setLeftResource(node.getNodeResource.getMaxResource - resource.getUsedResource) - }else { + } else { resource.setLeftResource(labelResource.getLeftResource) } } @@ -211,8 +213,8 @@ class RMMonitorRest extends Logging { "(" + engineTypeLabel.getEngineType + "," + engineTypeLabel.getVersion + ")" } - @RequestMapping(path = Array("engines"),method = Array(RequestMethod.POST)) - def getEngines(request: HttpServletRequest, @RequestBody(required=false) param: util.Map[String, AnyRef]): Message = { + @RequestMapping(path = Array("engines"), method = Array(RequestMethod.POST)) + def getEngines(request: HttpServletRequest, @RequestBody(required = false) param: util.Map[String, AnyRef]): Message = { val message = Message.ok("") val userName = SecurityFilter.getLoginUsername(request) val nodes = getEngineNodes(userName, true) @@ -230,11 +232,11 @@ class RMMonitorRest extends Logging { } record.put("creator", userCreatorLabel.getCreator) record.put("engineType", engineTypeLabel.getEngineType) - if(node.getNodeResource != null){ + if (node.getNodeResource != null) { if (node.getNodeResource.getLockedResource != null) record.put("preUsedResource", node.getNodeResource.getLockedResource) if (node.getNodeResource.getUsedResource != null) record.put("usedResource", node.getNodeResource.getUsedResource) } - if(node.getNodeStatus == null){ + if (node.getNodeStatus == null) { record.put("engineStatus", "Busy") } else { record.put("engineStatus", node.getNodeStatus.toString) @@ -244,28 +246,8 @@ class RMMonitorRest extends Logging { appendMessageData(message, "engines", engines) } - /** - * 仅用于向下兼容老接口 - * @param request - * @param param - * @return - */ - @RequestMapping(path = Array("enginekill"),method = Array(RequestMethod.POST)) - def killEngine(request: HttpServletRequest, @RequestBody param: util.ArrayList[util.Map[String, AnyRef]]): Message = { - val userName = SecurityFilter.getLoginUsername(request) - for (engineParam <- param) { - val moduleName = engineParam.get("applicationName").asInstanceOf[String] - val engineInstance = engineParam.get("engineInstance").asInstanceOf[String] - val stopEngineRequest = new EngineStopRequest(ServiceInstance(moduleName, engineInstance), userName) - val job = messagePublisher.publish(stopEngineRequest) - Utils.tryAndWarn(job.get(RMUtils.MANAGER_KILL_ENGINE_EAIT.getValue.toLong, TimeUnit.MILLISECONDS)) - info(s"Finished to kill engine ") - } - Message.ok("成功提交kill引擎请求。") - } - - @RequestMapping(path = Array("queueresources"),method = Array(RequestMethod.POST)) + @RequestMapping(path = Array("queueresources"), method = Array(RequestMethod.POST)) def getQueueResource(request: HttpServletRequest, @RequestBody param: util.Map[String, AnyRef]): Message = { val message = Message.ok("") val yarnIdentifier = new YarnResourceIdentifier(param.get("queuename").asInstanceOf[String]) @@ -286,102 +268,113 @@ class RMMonitorRest extends Logging { usedCPUPercentage = usedResource.queueCores.asInstanceOf[Double] / maxResource.queueCores.asInstanceOf[Double] queueInfo.put("usedPercentage", Map("memory" -> usedMemoryPercentage, "cores" -> usedCPUPercentage)) appendMessageData(message, "queueInfo", queueInfo) - case _ => Message.error("获取队列资源失败") + case _ => Message.error("Failed to get queue resource") } val userResourceRecords = new ArrayBuffer[mutable.HashMap[String, Any]]() val yarnAppsInfo = externalResourceService.getAppInfo(ResourceType.Yarn, labelContainer, yarnIdentifier) - yarnAppsInfo.groupBy(_.asInstanceOf[YarnAppInfo].user).foreach { userAppInfo => - val nodes = getEngineNodes(userAppInfo._1, true) - var busyResource = Resource.initResource(ResourceType.Yarn).asInstanceOf[YarnResource] - var idleResource = Resource.initResource(ResourceType.Yarn).asInstanceOf[YarnResource] - val appIdToEngineNode = new mutable.HashMap[String, EngineNode]() - nodes.foreach { node => - if (node.getNodeResource != null && node.getNodeResource.getUsedResource != null) node.getNodeResource.getUsedResource match { - case driverYarn: DriverAndYarnResource if driverYarn.yarnResource.queueName.equals(yarnIdentifier.getQueueName) => - appIdToEngineNode.put(driverYarn.yarnResource.applicationId, node) - case yarn: YarnResource if yarn.queueName.equals(yarnIdentifier.getQueueName) => - appIdToEngineNode.put(yarn.applicationId, node) - case _ => + Utils.tryCatch{ + yarnAppsInfo.groupBy(_.asInstanceOf[YarnAppInfo].user).foreach { userAppInfo => + var nodes = getEngineNodes(userAppInfo._1, true) + var busyResource = Resource.initResource(ResourceType.Yarn).asInstanceOf[YarnResource] + var idleResource = Resource.initResource(ResourceType.Yarn).asInstanceOf[YarnResource] + val appIdToEngineNode = new mutable.HashMap[String, EngineNode]() + if(nodes == null) { + nodes = new Array[EngineNode](0) } - } - userAppInfo._2.foreach { appInfo => - appIdToEngineNode.get(appInfo.asInstanceOf[YarnAppInfo].id) match { - case Some(node) => - if(NodeStatus.Busy == node.getNodeStatus){ + nodes.foreach { node => + if (node.getNodeResource != null && node.getNodeResource.getUsedResource != null) node.getNodeResource.getUsedResource match { + case driverYarn: DriverAndYarnResource if driverYarn.yarnResource.queueName.equals(yarnIdentifier.getQueueName) => + appIdToEngineNode.put(driverYarn.yarnResource.applicationId, node) + case yarn: YarnResource if yarn.queueName.equals(yarnIdentifier.getQueueName) => + appIdToEngineNode.put(yarn.applicationId, node) + case _ => + } + } + userAppInfo._2.foreach { appInfo => + appIdToEngineNode.get(appInfo.asInstanceOf[YarnAppInfo].id) match { + case Some(node) => + if (NodeStatus.Busy == node.getNodeStatus) { + busyResource = busyResource.add(appInfo.asInstanceOf[YarnAppInfo].usedResource) + } else { + idleResource = idleResource.add(appInfo.asInstanceOf[YarnAppInfo].usedResource) + } + case None => busyResource = busyResource.add(appInfo.asInstanceOf[YarnAppInfo].usedResource) - } else { - idleResource = idleResource.add(appInfo.asInstanceOf[YarnAppInfo].usedResource) - } - case None => - busyResource = busyResource.add(appInfo.asInstanceOf[YarnAppInfo].usedResource) + } } - } - val totalResource = busyResource.add(idleResource) - if (totalResource > Resource.getZeroResource(totalResource)) { - val userResource = new mutable.HashMap[String, Any]() - userResource.put("username", userAppInfo._1) - val queueResource = providedYarnResource.getMaxResource.asInstanceOf[YarnResource] - if (usedMemoryPercentage > usedCPUPercentage) { - userResource.put("busyPercentage", busyResource.queueMemory.asInstanceOf[Double] / queueResource.queueMemory.asInstanceOf[Double]) - userResource.put("idlePercentage", idleResource.queueMemory.asInstanceOf[Double] / queueResource.queueMemory.asInstanceOf[Double]) - userResource.put("totalPercentage", totalResource.queueMemory.asInstanceOf[Double] / queueResource.queueMemory.asInstanceOf[Double]) - } else { - userResource.put("busyPercentage", busyResource.queueCores.asInstanceOf[Double] / queueResource.queueCores.asInstanceOf[Double]) - userResource.put("idlePercentage", idleResource.queueCores.asInstanceOf[Double] / queueResource.queueCores.asInstanceOf[Double]) - userResource.put("totalPercentage", totalResource.queueCores.asInstanceOf[Double] / queueResource.queueCores.asInstanceOf[Double]) + val totalResource = busyResource.add(idleResource) + if (totalResource > Resource.getZeroResource(totalResource)) { + val userResource = new mutable.HashMap[String, Any]() + userResource.put("username", userAppInfo._1) + val queueResource = providedYarnResource.getMaxResource.asInstanceOf[YarnResource] + if (usedMemoryPercentage > usedCPUPercentage) { + userResource.put("busyPercentage", busyResource.queueMemory.asInstanceOf[Double] / queueResource.queueMemory.asInstanceOf[Double]) + userResource.put("idlePercentage", idleResource.queueMemory.asInstanceOf[Double] / queueResource.queueMemory.asInstanceOf[Double]) + userResource.put("totalPercentage", totalResource.queueMemory.asInstanceOf[Double] / queueResource.queueMemory.asInstanceOf[Double]) + } else { + userResource.put("busyPercentage", busyResource.queueCores.asInstanceOf[Double] / queueResource.queueCores.asInstanceOf[Double]) + userResource.put("idlePercentage", idleResource.queueCores.asInstanceOf[Double] / queueResource.queueCores.asInstanceOf[Double]) + userResource.put("totalPercentage", totalResource.queueCores.asInstanceOf[Double] / queueResource.queueCores.asInstanceOf[Double]) + } + userResourceRecords.add(userResource) } - userResourceRecords.add(userResource) } + }{ + case exception: Exception => { + error(s"queresource search failed!", exception) + } + case _ => } - //排序 + userResourceRecords.sort(new Comparator[mutable.Map[String, Any]]() { override def compare(o1: mutable.Map[String, Any], o2: mutable.Map[String, Any]): Int = if (o1.get("totalPercentage").getOrElse(0.0).asInstanceOf[Double] > o2.get("totalPercentage").getOrElse(0.0).asInstanceOf[Double]) -1 else 1 }) appendMessageData(message, "userResources", userResourceRecords) } - private def getEngineNodes(user: String, withResource: Boolean = false): Array[EngineNode] ={ + private def getEngineNodes(user: String, withResource: Boolean = false): Array[EngineNode] = { val nodes = nodeManagerPersistence.getNodes(user).map(_.getServiceInstance).map(nodeManagerPersistence.getEngineNode).filter(_ != null) - val metrics = nodeMetricManagerPersistence.getNodeMetrics(nodes).map(m => (m.getServiceInstance.toString,m)).toMap + val metrics = nodeMetricManagerPersistence.getNodeMetrics(nodes).map(m => (m.getServiceInstance.toString, m)).toMap val configurationMap = new mutable.HashMap[String, Resource] nodes.map{ node => node.setLabels(nodeLabelService.getNodeLabels(node.getServiceInstance)) - if(!node.getLabels.exists(_.isInstanceOf[UserCreatorLabel])){ - return null - } - metrics.get(node.getServiceInstance.toString).foreach(metricsConverter.fillMetricsToNode(node,_)) - if(withResource){ - val userCreatorLabel = node.getLabels.find(_.isInstanceOf[UserCreatorLabel]).get.asInstanceOf[UserCreatorLabel] - val engineTypeLabel = node.getLabels.find(_.isInstanceOf[EngineTypeLabel]).get.asInstanceOf[EngineTypeLabel] - val engineInstanceLabel = node.getLabels.find(_.isInstanceOf[EngineInstanceLabel]).get.asInstanceOf[EngineInstanceLabel] - engineInstanceLabel.setServiceName(node.getServiceInstance.getApplicationName) - engineInstanceLabel.setInstance(node.getServiceInstance.getInstance) - val nodeResource = labelResourceService.getLabelResource(engineInstanceLabel) - val configurationKey = getUserCreator(userCreatorLabel) + getEngineType(engineTypeLabel) - val configuredResource = configurationMap.get(configurationKey) match { - case Some(resource) => resource - case None => - if(nodeResource != null){ - val resource = UserConfiguration.getUserConfiguredResource(nodeResource.getResourceType, userCreatorLabel, engineTypeLabel) - configurationMap.put(configurationKey, resource) - resource - }else null - } - if(nodeResource != null){ - nodeResource.setMaxResource(configuredResource) - if(null == nodeResource.getUsedResource) nodeResource.setUsedResource(nodeResource.getLockedResource) - if(null == nodeResource.getMinResource) nodeResource.setMinResource(Resource.initResource(nodeResource.getResourceType)) - node.setNodeResource(ResourceUtils.convertTo(nodeResource, ResourceType.LoadInstance)) + if(!node.getLabels.exists(_.isInstanceOf[UserCreatorLabel])) { + null + } else { + metrics.get(node.getServiceInstance.toString).foreach(metricsConverter.fillMetricsToNode(node, _)) + if (withResource) { + val userCreatorLabel = node.getLabels.find(_.isInstanceOf[UserCreatorLabel]).get.asInstanceOf[UserCreatorLabel] + val engineTypeLabel = node.getLabels.find(_.isInstanceOf[EngineTypeLabel]).get.asInstanceOf[EngineTypeLabel] + val engineInstanceLabel = node.getLabels.find(_.isInstanceOf[EngineInstanceLabel]).get.asInstanceOf[EngineInstanceLabel] + engineInstanceLabel.setServiceName(node.getServiceInstance.getApplicationName) + engineInstanceLabel.setInstance(node.getServiceInstance.getInstance) + val nodeResource = labelResourceService.getLabelResource(engineInstanceLabel) + val configurationKey = getUserCreator(userCreatorLabel) + getEngineType(engineTypeLabel) + val configuredResource = configurationMap.get(configurationKey) match { + case Some(resource) => resource + case None => + if (nodeResource != null) { + val resource = UserConfiguration.getUserConfiguredResource(nodeResource.getResourceType, userCreatorLabel, engineTypeLabel) + configurationMap.put(configurationKey, resource) + resource + } else null + } + if(nodeResource != null) { + nodeResource.setMaxResource(configuredResource) + if(null == nodeResource.getUsedResource) nodeResource.setUsedResource(nodeResource.getLockedResource) + if(null == nodeResource.getMinResource) nodeResource.setMinResource(Resource.initResource(nodeResource.getResourceType)) + node.setNodeResource(ResourceUtils.convertTo(nodeResource, ResourceType.LoadInstance)) + } } + node } - node }.filter(_ != null).toArray } - @RequestMapping(path = Array("queues"),method =Array(RequestMethod.POST)) - def getQueues(request: HttpServletRequest,@RequestBody(required=false) param: util.Map[String, AnyRef]): Message = { + @RequestMapping(path = Array("queues"), method = Array(RequestMethod.POST)) + def getQueues(request: HttpServletRequest, @RequestBody(required = false) param: util.Map[String, AnyRef]): Message = { val message = Message.ok() val userName = SecurityFilter.getLoginUsername(request) val clusters = new mutable.ArrayBuffer[Any]() diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/LabelResourceService.scala b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/LabelResourceService.scala index 8b5183a9e2..29ba359ce0 100644 --- a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/LabelResourceService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/LabelResourceService.scala @@ -17,6 +17,8 @@ package org.apache.linkis.resourcemanager.service +import java.util + import org.apache.linkis.manager.common.entity.persistence.PersistenceResource import org.apache.linkis.manager.common.entity.resource.NodeResource import org.apache.linkis.manager.label.entity.Label @@ -26,20 +28,18 @@ abstract class LabelResourceService { def getLabelResource(label: Label[_]): NodeResource - def setLabelResource(label: Label[_], nodeResource: NodeResource) : Unit + def setLabelResource(label: Label[_], nodeResource: NodeResource, source: String) : Unit /** - * 方法同 setLabelResource 只适用于启动引擎申请资源后设置engineConn资源 * this function is the same to setLabelResource * @param label * @param nodeResource */ - def setEngineConnLabelResource(label: Label[_], nodeResource: NodeResource) : Unit + def setEngineConnLabelResource(label: Label[_], nodeResource: NodeResource, source: String) : Unit def getResourcesByUser(user: String) : Array[NodeResource] - // 需要通过AM传入的Label（用户的Label和EMLabel）从LabelResource拿到所有关系的Label：里面包含所有的单Label和CombineLabel。 - def enrichLabels(labelContainer: RMLabelContainer) : RMLabelContainer + def enrichLabels(labels: util.List[Label[_]]) : RMLabelContainer def removeResourceByLabel(label: Label[_]): Unit diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/RequestResourceService.scala b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/RequestResourceService.scala index 0273b8a184..f7ad10a83f 100644 --- a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/RequestResourceService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/RequestResourceService.scala @@ -17,13 +17,12 @@ package org.apache.linkis.resourcemanager.service -import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.utils.Logging import org.apache.linkis.manager.common.entity.resource._ +import org.apache.linkis.manager.label.entity.em.EMInstanceLabel import org.apache.linkis.resourcemanager.domain.RMLabelContainer -import org.apache.linkis.resourcemanager.exception.RMWarnException -import org.apache.linkis.resourcemanager.utils.RMUtils.aggregateResource -import org.apache.linkis.resourcemanager.utils.{AlertUtils, RMConfiguration, RMUtils, UserConfiguration} +import org.apache.linkis.resourcemanager.exception.{RMErrorCode, RMWarnException} +import org.apache.linkis.resourcemanager.utils.{RMUtils, UserConfiguration} abstract class RequestResourceService(labelResourceService: LabelResourceService) extends Logging{ @@ -32,105 +31,138 @@ abstract class RequestResourceService(labelResourceService: LabelResourceService val enableRequest = RMUtils.RM_REQUEST_ENABLE.getValue def canRequest(labelContainer: RMLabelContainer, resource: NodeResource): Boolean = { + + labelContainer.getCurrentLabel match { + case emInstanceLabel: EMInstanceLabel => + return checkEMResource(labelContainer.getUserCreatorLabel.getUser, emInstanceLabel, resource) + case _ => + } + var labelResource = labelResourceService.getLabelResource(labelContainer.getCurrentLabel) + val requestResource = resource.getMinResource // for configuration resource - if(labelContainer.getCombinedUserCreatorEngineTypeLabel.equals(labelContainer.getCurrentLabel)){ - if(labelResource == null) { + if (labelContainer.getCombinedUserCreatorEngineTypeLabel.equals(labelContainer.getCurrentLabel)) { + if (labelResource == null) { labelResource = new CommonNodeResource labelResource.setResourceType(resource.getResourceType) labelResource.setUsedResource(Resource.initResource(resource.getResourceType)) labelResource.setLockedResource(Resource.initResource(resource.getResourceType)) - info(s"ResourceInit:${labelContainer.getCurrentLabel.getStringValue}") + logger.info(s"ResourceInit: ${labelContainer.getCurrentLabel.getStringValue} ") } val configuredResource = UserConfiguration.getUserConfiguredResource(resource.getResourceType, labelContainer.getUserCreatorLabel, labelContainer.getEngineTypeLabel) - debug(s"Get configured resource ${configuredResource} for [${labelContainer.getUserCreatorLabel}] and [${labelContainer.getEngineTypeLabel}]") + logger.debug(s"Get configured resource ${configuredResource} for [${labelContainer.getUserCreatorLabel}] and [${labelContainer.getEngineTypeLabel}] ") labelResource.setMaxResource(configuredResource) labelResource.setMinResource(Resource.initResource(labelResource.getResourceType)) labelResource.setLeftResource(labelResource.getMaxResource - labelResource.getUsedResource - labelResource.getLockedResource) - labelResourceService.setLabelResource(labelContainer.getCurrentLabel, labelResource) - info(s"${labelContainer.getCurrentLabel} to request [${resource.getMinResource}] \t labelResource: Max: ${labelResource.getMaxResource} \t " + + labelResourceService.setLabelResource(labelContainer.getCurrentLabel, labelResource, labelContainer.getCombinedUserCreatorEngineTypeLabel.getStringValue) + logger.debug(s"${labelContainer.getCurrentLabel} to request [${requestResource}] \t labelResource: Max: ${labelResource.getMaxResource} \t " + s"use: ${labelResource.getUsedResource} \t locked: ${labelResource.getLockedResource}") } - debug(s"Label [${labelContainer.getCurrentLabel}] has resource + [${labelResource }]") - if(labelResource != null){ + logger.debug(s"Label [${labelContainer.getCurrentLabel}] has resource + [${labelResource }]") + if (labelResource != null) { val labelAvailableResource = labelResource.getLeftResource - if(labelAvailableResource < resource.getMinResource && enableRequest){ - info(s"Failed check: ${labelContainer.getUserCreatorLabel.getUser} want to use label [${labelContainer.getCurrentLabel}] resource[${resource.getMinResource}] > label available resource[${labelAvailableResource}]") + if(labelAvailableResource < requestResource && enableRequest) { + logger.info(s"Failed check: ${labelContainer.getUserCreatorLabel.getUser} want to use label [${labelContainer.getCurrentLabel}] resource[${requestResource}] > label available resource[${labelAvailableResource}]") // TODO sendAlert(moduleInstance, user, creator, requestResource, moduleAvailableResource.resource, moduleLeftResource) - val notEnoughMessage = generateNotEnoughMessage(aggregateResource(labelResource.getUsedResource, labelResource.getLockedResource), labelAvailableResource) + val notEnoughMessage = generateNotEnoughMessage(requestResource, labelAvailableResource) throw new RMWarnException(notEnoughMessage._1, notEnoughMessage._2) } - info(s"Passed check: ${labelContainer.getUserCreatorLabel.getUser} want to use label [${labelContainer.getCurrentLabel}] resource[${resource.getMinResource}] <= label available resource[${labelAvailableResource}]") + logger.debug(s"Passed check: ${labelContainer.getUserCreatorLabel.getUser} want to use label [${labelContainer.getCurrentLabel}] resource[${requestResource}] <= label available resource[${labelAvailableResource}]") + true + } else { + logger.warn(s"No resource available found for label ${labelContainer.getCurrentLabel}") + throw new RMWarnException(11201, s"Resource label ${labelContainer.getCurrentLabel} has no resource, please check resource in db.") } - warn(s"No resource available found for label ${labelContainer.getCurrentLabel}") - return true } - def sendAlert(moduleInstance: ServiceInstance, user: String, creator:String, requestResource: Resource, availableResource: Resource, moduleLeftResource: Resource) = { - if(RMConfiguration.ALERT_ENABLED.getValue){ - info("start sending alert") - val title = s"user ${user} failed to request resource on EM(${moduleInstance.getApplicationName},${moduleInstance.getInstance})" - val queueContact = requestResource match { - case d: DriverAndYarnResource => AlertUtils.getContactByQueue(d.yarnResource.queueName) - case y: YarnResource => AlertUtils.getContactByQueue(y.queueName) - case _ => RMConfiguration.ALERT_DEFAULT_CONTACT.getValue + private def checkEMResource(user: String, emInstanceLabel: EMInstanceLabel, resource: NodeResource): Boolean = { + val labelResource = labelResourceService.getLabelResource(emInstanceLabel) + val requestResource = resource.getMinResource + logger.debug(s"emInstanceLabel resource info ${labelResource }") + if (labelResource != null) { + val labelAvailableResource = labelResource.getLeftResource + if(labelAvailableResource < requestResource && enableRequest) { + logger.info(s"user want to use resource[${requestResource}] > em ${emInstanceLabel.getInstance()} available resource[${labelAvailableResource}]") + // TODO sendAlert(moduleInstance, user, creator, requestResource, moduleAvailableResource.resource, moduleLeftResource) + val notEnoughMessage = generateECMNotEnoughMessage(requestResource, labelAvailableResource) + throw new RMWarnException(notEnoughMessage._1, notEnoughMessage._2) } - val detail = - s"请联系用户[${user}]或相关人员[${queueContact}]\n" + - s"user request resource: ${requestResource}\n " + - s"user available resource: ${availableResource}\n " + - s"EM left resource: ${moduleLeftResource}\n " - AlertUtils.sendAlertAsync(title, detail); - info("finished sending alert") + logger.debug(s"Passed check: resource[${requestResource}] want to use em ${emInstanceLabel.getInstance()} available resource[${labelAvailableResource}]") + true + } else { + logger.warn(s"No resource available found for em ${emInstanceLabel.getInstance()} ") + throw new RMWarnException(11201, s"No resource available found for em ${emInstanceLabel.getInstance()} ") + } + } + + def generateECMNotEnoughMessage(requestResource: Resource, availableResource: Resource) : (Int, String) = { + val loadRequestResource = requestResource match { + case li: LoadInstanceResource => li + case driverAndYarnResource: DriverAndYarnResource => driverAndYarnResource.loadInstanceResource + case _ => null + } + loadRequestResource match { + case li: LoadInstanceResource => + val loadInstanceAvailable = availableResource.asInstanceOf[LoadInstanceResource] + if(li.cores > loadInstanceAvailable.cores) { + (RMErrorCode.ECM_CPU_INSUFFICIENT.getCode, RMErrorCode.ECM_CPU_INSUFFICIENT.getMessage) + } else if (li.memory > loadInstanceAvailable.memory) { + (RMErrorCode.ECM_MEMORY_INSUFFICIENT.getCode, RMErrorCode.ECM_MEMORY_INSUFFICIENT.getMessage) + } else { + (RMErrorCode.ECM_INSTANCES_INSUFFICIENT.getCode, RMErrorCode.ECM_INSTANCES_INSUFFICIENT.getMessage) + } + case _ => + (RMErrorCode.ECM_RESOURCE_INSUFFICIENT.getCode, RMErrorCode.ECM_RESOURCE_INSUFFICIENT.getMessage) } } + def generateNotEnoughMessage(requestResource: Resource, availableResource: Resource) : (Int, String) = { requestResource match { case m: MemoryResource => - (11011, s"Drive memory resources are insufficient, to reduce the drive memory(内存资源不足，建议调小驱动内存)") + (RMErrorCode.DRIVER_MEMORY_INSUFFICIENT.getCode, RMErrorCode.DRIVER_MEMORY_INSUFFICIENT.getMessage) case c: CPUResource => - (11012, s"CPU resources are insufficient, to reduce the number of driver cores(CPU资源不足，建议调小驱动核数)") + (RMErrorCode.DRIVER_CPU_INSUFFICIENT.getCode, RMErrorCode.DRIVER_CPU_INSUFFICIENT.getMessage) case i: InstanceResource => - (11013, s"Insufficient number of instances, idle engines can be killed(实例数不足，可以kill空闲的引擎)") + (RMErrorCode.INSTANCES_INSUFFICIENT.getCode, RMErrorCode.INSTANCES_INSUFFICIENT.getMessage) case l: LoadResource => val loadAvailable = availableResource.asInstanceOf[LoadResource] - if(l.cores > loadAvailable.cores){ - (11012, s"CPU resources are insufficient, to reduce the number of driver cores(CPU资源不足，建议调小驱动核数)") + if(l.cores > loadAvailable.cores) { + (RMErrorCode.DRIVER_CPU_INSUFFICIENT.getCode, RMErrorCode.DRIVER_CPU_INSUFFICIENT.getMessage) } else { - (11011, s"Drive memory resources are insufficient, to reduce the drive memory(内存资源不足，建议调小驱动内存)") + (RMErrorCode.DRIVER_MEMORY_INSUFFICIENT.getCode, RMErrorCode.DRIVER_MEMORY_INSUFFICIENT.getMessage) } case li: LoadInstanceResource => val loadInstanceAvailable = availableResource.asInstanceOf[LoadInstanceResource] - if(li.cores > loadInstanceAvailable.cores){ - (11012, s"CPU resources are insufficient, to reduce the number of driver cores(CPU资源不足，建议调小驱动核数)") + if(li.cores > loadInstanceAvailable.cores) { + (RMErrorCode.DRIVER_CPU_INSUFFICIENT.getCode, RMErrorCode.DRIVER_CPU_INSUFFICIENT.getMessage) } else if (li.memory > loadInstanceAvailable.memory) { - (11011, s"Drive memory resources are insufficient, to reduce the drive memory(内存资源不足，建议调小驱动内存)") + (RMErrorCode.DRIVER_MEMORY_INSUFFICIENT.getCode, RMErrorCode.DRIVER_MEMORY_INSUFFICIENT.getMessage) } else { - (11013, s"Insufficient number of instances, idle engines can be killed(实例数不足，可以kill空闲的引擎)") + (RMErrorCode.INSTANCES_INSUFFICIENT.getCode, RMErrorCode.INSTANCES_INSUFFICIENT.getMessage) } case yarn: YarnResource => val yarnAvailable = availableResource.asInstanceOf[YarnResource] - if(yarn.queueCores > yarnAvailable.queueCores){ - (11014, s"Queue CPU resources are insufficient, reduce the number of executors.(队列CPU资源不足，建议调小执行器个数)") - } else if (yarn.queueMemory > yarnAvailable.queueMemory){ - (11015, s"Insufficient queue memory resources, reduce the executor memory") + if(yarn.queueCores > yarnAvailable.queueCores) { + (RMErrorCode.QUEUE_CPU_INSUFFICIENT.getCode, RMErrorCode.QUEUE_CPU_INSUFFICIENT.getMessage) + } else if (yarn.queueMemory > yarnAvailable.queueMemory) { + (RMErrorCode.QUEUE_MEMORY_INSUFFICIENT.getCode, RMErrorCode.QUEUE_MEMORY_INSUFFICIENT.getMessage) } else { - (11016, s"Insufficient number of queue instances, idle engines can be killed(队列实例数不足，可以kill空闲的引擎)") + (RMErrorCode.QUEUE_INSTANCES_INSUFFICIENT.getCode, RMErrorCode.QUEUE_INSTANCES_INSUFFICIENT.getMessage) } case dy: DriverAndYarnResource => val dyAvailable = availableResource.asInstanceOf[DriverAndYarnResource] if(dy.loadInstanceResource.memory > dyAvailable.loadInstanceResource.memory || dy.loadInstanceResource.cores > dyAvailable.loadInstanceResource.cores || - dy.loadInstanceResource.instances > dyAvailable.loadInstanceResource.instances){ + dy.loadInstanceResource.instances > dyAvailable.loadInstanceResource.instances) { val detail = generateNotEnoughMessage(dy.loadInstanceResource, dyAvailable.loadInstanceResource) - (detail._1, s"When requesting server resources，${detail._2}") + (detail._1, {detail._2}) } else { val detail = generateNotEnoughMessage(dy.yarnResource, dyAvailable.yarnResource) - (detail._1, s"When requesting server resources，${detail._2}") + (detail._1, {detail._2}) } - case s: SpecialResource => throw new RMWarnException(11003,"not supported resource type " + s.getClass) - case r: Resource => throw new RMWarnException(11003,"not supported resource type " + r.getClass) + case s: SpecialResource => throw new RMWarnException(11003, " not supported resource type " + s.getClass) + case r: Resource => throw new RMWarnException(11003, "not supported resource type " + r.getClass) } } } diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/ResourceLockService.scala b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/ResourceLockService.scala index 398950f4bd..159d61b34e 100644 --- a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/ResourceLockService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/ResourceLockService.scala @@ -40,12 +40,12 @@ class ResourceLockService extends Logging { def tryLock(labelContainer: RMLabelContainer): Boolean = tryLock(labelContainer, Long.MaxValue) def tryLock(labelContainer: RMLabelContainer, timeout: Long): Boolean = { - if(StringUtils.isBlank(labelContainer.getCurrentLabel.getStringValue) + if (StringUtils.isBlank(labelContainer.getCurrentLabel.getStringValue) || !labelContainer.getCurrentLabel.isInstanceOf[ResourceLabel] - || labelContainer.getLockedLabels.contains(labelContainer.getCurrentLabel)){ + || labelContainer.getLockedLabels.contains(labelContainer.getCurrentLabel)) { return true } - val lockedBy = if(labelContainer.getUserCreatorLabel == null) DEFAULT_LOCKED_BY else labelContainer.getUserCreatorLabel.getUser + val lockedBy = if (labelContainer.getUserCreatorLabel == null) DEFAULT_LOCKED_BY else labelContainer.getUserCreatorLabel.getUser val persistenceLock = new PersistenceLock persistenceLock.setLockObject(labelContainer.getCurrentLabel.getStringValue) persistenceLock.setCreateTime(new Date) @@ -53,9 +53,13 @@ class ResourceLockService extends Logging { persistenceLock.setUpdateTime(new Date) persistenceLock.setUpdator(lockedBy) try { - val isLocked: Boolean = lockManagerPersistence.lock(persistenceLock, timeout) - if(isLocked){ - info(labelContainer.getLabels + " successfully locked label" + persistenceLock.getLockObject) + val isLocked: Boolean = if (timeout > 0) { + lockManagerPersistence.lock(persistenceLock, timeout) + } else { + lockManagerPersistence.lock(persistenceLock, Long.MaxValue) + } + if(isLocked) { + info(labelContainer.getCurrentLabel + " successfully locked label" + persistenceLock.getLockObject) labelContainer.getLockedLabels.add(labelContainer.getCurrentLabel) } isLocked @@ -68,15 +72,15 @@ class ResourceLockService extends Logging { def unLock(labelContainer: RMLabelContainer): Unit = { val labelIterator = labelContainer.getLockedLabels.iterator - while(labelIterator.hasNext){ + while(labelIterator.hasNext) { val label = labelIterator.next - if(!StringUtils.isBlank(label.getStringValue)){ + if (!StringUtils.isBlank(label.getStringValue)) { val persistenceLock = new PersistenceLock persistenceLock.setLockObject(label.getStringValue) try { lockManagerPersistence.unlock(persistenceLock) info("unlocked " + persistenceLock.getLockObject) - }catch { + } catch { case t: Throwable => error(s"failed to unlock label [${persistenceLock.getLockObject}]", t) throw t @@ -86,10 +90,10 @@ class ResourceLockService extends Logging { } } - def clearTimeoutLock(timeout: Long) = { + def clearTimeoutLock(timeout: Long): Unit = { val currentTime = System.currentTimeMillis lockManagerPersistence.getAll.foreach{ lock => - if(currentTime - lock.getCreateTime.getTime > timeout){ + if (currentTime - lock.getCreateTime.getTime > timeout) { lockManagerPersistence.unlock(lock) warn("timeout force unlock " + lock.getLockObject) } diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/ResourceManager.scala b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/ResourceManager.scala index 8a2d45bbfb..1f9679cfb1 100644 --- a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/ResourceManager.scala +++ b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/ResourceManager.scala @@ -96,4 +96,5 @@ abstract class ResourceManager { */ def getResourceInfo(serviceInstances: Array[ServiceInstance]): ResourceInfo + } \ No newline at end of file diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/impl/DefaultReqResourceService.scala b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/impl/DefaultReqResourceService.scala index a6420cd815..f449d4125c 100644 --- a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/impl/DefaultReqResourceService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/impl/DefaultReqResourceService.scala @@ -28,7 +28,4 @@ class DefaultReqResourceService(labelResourceService: LabelResourceService) exte override val resourceType: ResourceType = ResourceType.Default - override def canRequest(labelContainer: RMLabelContainer, resource: NodeResource): Boolean = { - super.canRequest(labelContainer, resource) - } } diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/impl/DefaultResourceManager.scala b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/impl/DefaultResourceManager.scala index bc3ca89913..e4a5bd476b 100644 --- a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/impl/DefaultResourceManager.scala +++ b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/impl/DefaultResourceManager.scala @@ -18,13 +18,13 @@ package org.apache.linkis.resourcemanager.service.impl import java.util -import java.util.UUID +import java.util.{Date, UUID} import java.util.concurrent.TimeUnit - import com.google.common.collect.Lists import org.apache.linkis.common.ServiceInstance import org.apache.linkis.common.utils.{Logging, Utils} import org.apache.linkis.governance.common.conf.GovernanceCommonConf +import org.apache.linkis.manager.common.entity.enumeration.NodeStatus import org.apache.linkis.manager.common.entity.node.{AMEMNode, AMEngineNode, InfoRMNode} import org.apache.linkis.manager.common.entity.resource.{CommonNodeResource, NodeResource, Resource, ResourceType} import org.apache.linkis.manager.common.utils.ResourceUtils @@ -33,18 +33,21 @@ import org.apache.linkis.manager.label.entity.{Label, ResourceLabel} import org.apache.linkis.manager.label.entity.em.EMInstanceLabel import org.apache.linkis.manager.label.entity.engine.EngineInstanceLabel import org.apache.linkis.manager.label.service.NodeLabelService -import org.apache.linkis.manager.persistence.{NodeManagerPersistence, ResourceManagerPersistence} +import org.apache.linkis.manager.persistence.{LabelManagerPersistence, NodeManagerPersistence, NodeMetricManagerPersistence, ResourceManagerPersistence} import org.apache.linkis.resourcemanager._ import org.apache.linkis.resourcemanager.domain.RMLabelContainer -import org.apache.linkis.resourcemanager.exception.{RMErrorException, RMWarnException} +import org.apache.linkis.resourcemanager.exception.{RMErrorCode, RMErrorException, RMWarnException} import org.apache.linkis.resourcemanager.external.service.ExternalResourceService import org.apache.linkis.resourcemanager.protocol.{TimeoutEMEngineRequest, TimeoutEMEngineResponse} import org.apache.linkis.resourcemanager.service._ import org.apache.linkis.resourcemanager.utils.{RMConfiguration, RMUtils} import org.apache.linkis.rpc.Sender +import org.apache.linkis.server.BDPJettyServerHelper import org.springframework.beans.factory.InitializingBean import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component +import org.springframework.transaction.annotation.Transactional +import org.springframework.util.CollectionUtils import scala.collection.JavaConversions._ @@ -52,7 +55,7 @@ import scala.collection.JavaConversions._ class DefaultResourceManager extends ResourceManager with Logging with InitializingBean { @Autowired - var resourceManagerPersistence : ResourceManagerPersistence = _ + var resourceManagerPersistence: ResourceManagerPersistence = _ @Autowired var nodeManagerPersistence: NodeManagerPersistence = _ @@ -69,14 +72,22 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ @Autowired var resourceLogService: ResourceLogService = _ - /*@Autowired - var nodeHeartbeatMonitor: NodeHeartbeatMonitor = _*/ + @Autowired + var labelManagerPersistence: LabelManagerPersistence = _ + + @Autowired + private var nodeMetricManagerPersistence: NodeMetricManagerPersistence = _ @Autowired private var nodeLabelService: NodeLabelService = _ var requestResourceServices: Array[RequestResourceService] = _ + val gson = BDPJettyServerHelper.gson + + val labelFactory = LabelBuilderFactoryContext.getLabelBuilderFactory + + override def afterPropertiesSet(): Unit = { requestResourceServices = Array( new DefaultReqResourceService(labelResourceService), @@ -89,16 +100,14 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ resourceLockService.clearTimeoutLock(RMConfiguration.LOCK_RELEASE_TIMEOUT.getValue.toLong) } }, RMConfiguration.LOCK_RELEASE_CHECK_INTERVAL.getValue.toLong, RMConfiguration.LOCK_RELEASE_CHECK_INTERVAL.getValue.toLong, TimeUnit.MILLISECONDS) - // submit NodeHeartbeatMonitor job - //Utils.defaultScheduler.scheduleAtFixedRate(nodeHeartbeatMonitor, RMConfiguration.NODE_HEARTBEAT_INTERVAL.getValue.toLong, RMConfiguration.NODE_HEARTBEAT_INTERVAL.getValue.toLong, TimeUnit.MILLISECONDS) } /** - * The registration method is mainly used to notify all RM nodes (including the node) - * 该注册方法，主要是用于通知所有的RM节点（包括本节点） - * - * - */ + * The registration method is mainly used to notify all RM nodes (including the node) + * 该注册方法，主要是用于通知所有的RM节点（包括本节点） + * + * + */ override def register(serviceInstance: ServiceInstance, resource: NodeResource): Unit = { info(s"Start processing registration of ServiceInstance: ${serviceInstance}") val eMInstanceLabel = LabelBuilderFactoryContext.getLabelBuilderFactory.createLabel(classOf[EMInstanceLabel]) @@ -115,17 +124,17 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ case _ => } val emResource = labelResourceService.getLabelResource(eMInstanceLabel) - if(emResource != null){ + if (emResource != null) { warn(s"${serviceInstance} has been registered, now update resource.") - if(!emResource.getResourceType.equals(resource.getResourceType)){ - throw new RMErrorException(110019, s"${serviceInstance} has been registered in ${emResource.getResourceType}, cannot be updated to ${resource.getResourceType}") + if (!emResource.getResourceType.equals(resource.getResourceType)) { + throw new RMErrorException(RMErrorCode.LABEL_DUPLICATED.getCode, s"${serviceInstance} has been registered in ${emResource.getResourceType}, cannot be updated to ${resource.getResourceType}") } } // TODO get ID Label set label resource - Utils.tryCatch{ - labelResourceService.setLabelResource(eMInstanceLabel, resource) + Utils.tryCatch { + labelResourceService.setLabelResource(eMInstanceLabel, resource, eMInstanceLabel.getStringValue) resourceLogService.success(ChangeType.ECM_INIT, null, eMInstanceLabel) - }{ + } { case exception: Exception => { resourceLogService.failed(ChangeType.ECM_INIT, null, eMInstanceLabel, exception) } @@ -139,9 +148,9 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ } /** - * The registration method is mainly used to notify all RM nodes (including the node), and the instance is offline. - * 该注册方法，主要是用于通知所有的RM节点（包括本节点），下线该实例 - */ + * The registration method is mainly used to notify all RM nodes (including the node), and the instance is offline. + * 该注册方法，主要是用于通知所有的RM节点（包括本节点），下线该实例 + */ override def unregister(serviceInstance: ServiceInstance): Unit = { // TODO get ID Label val eMInstanceLabel = LabelBuilderFactoryContext.getLabelBuilderFactory.createLabel(classOf[EMInstanceLabel]) @@ -150,17 +159,8 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ val labelContainer = new RMLabelContainer(Lists.newArrayList(eMInstanceLabel)) Utils.tryFinally { - // No need to lock for ECM unregistration -// labelContainer.getLabels.toArray.foreach { -// case label: Label[_] => -// labelContainer.setCurrentLabel(label.asInstanceOf[Label[_]]) -// resourceLockService.tryLock(labelContainer) -// case _ => -// } - - // clear EM related engine resource records - Utils.tryCatch{ - nodeManagerPersistence.getEngineNodeByEM(serviceInstance).foreach{ node => + Utils.tryCatch { + nodeManagerPersistence.getEngineNodeByEM(serviceInstance).foreach { node => val engineInstanceLabel = LabelBuilderFactoryContext.getLabelBuilderFactory.createLabel(classOf[EngineInstanceLabel]) engineInstanceLabel.setInstance(node.getServiceInstance.getInstance) engineInstanceLabel.setServiceName(node.getServiceInstance.getApplicationName) @@ -170,7 +170,7 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ labelContainer.setCurrentLabel(eMInstanceLabel) resourceLockService.unLock(labelContainer) resourceLogService.success(ChangeType.ECM_CLEAR, null, eMInstanceLabel) - }{ + } { case exception: Exception => { resourceLogService.failed(ChangeType.ECM_CLEAR, null, eMInstanceLabel, exception) } @@ -178,21 +178,18 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ } } { - //5.Release lock(释放锁) - //resourceLockService.unLock(labelContainer) info(s"ECMResourceClear:${serviceInstance}, usedResource:${Resource.initResource(ResourceType.Default).toJson}") - info(s"finished processing unregistration of ${serviceInstance}") } } /** - * Request resources, if not successful, return directly - * 请求资源，如果不成功，直接返回 - * - * @param labels - * @param resource - * @return - */ + * Request resources, if not successful, return directly + * 请求资源，如果不成功，直接返回 + * + * @param labels + * @param resource + * @return + */ override def requestResource(labels: util.List[Label[_]], resource: NodeResource): ResultResource = { requestResource(labels, resource, -1) } @@ -207,15 +204,15 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ * @return */ override def requestResource(labels: util.List[Label[_]], resource: NodeResource, wait: Long): ResultResource = { - val labelContainer = labelResourceService.enrichLabels(new RMLabelContainer(labels)) + val labelContainer = labelResourceService.enrichLabels(labels) debug("start processing request resource for labels [" + labelContainer + "] and resource [" + resource + "]") // pre-check without lock val requestResourceService = getRequestResourceService(resource.getResourceType) try { - labelContainer.getResourceLabels.foreach{ label => + labelContainer.getResourceLabels.foreach { label => labelContainer.setCurrentLabel(label) - if (!requestResourceService.canRequest(labelContainer, resource)){ - return NotEnoughResource(s"Labels：$labels not enough resource") + if (!requestResourceService.canRequest(labelContainer, resource)) { + return NotEnoughResource(s"Labels:$labels not enough resource") } } } catch { @@ -224,53 +221,44 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ Utils.tryFinally { // lock labels - labelContainer.getResourceLabels.foreach { - case label: Label[_] => - labelContainer.setCurrentLabel(label) - val locked = if(wait <= 0){ - resourceLockService.tryLock(labelContainer) - } else { - resourceLockService.tryLock(labelContainer, wait) - } - if (!locked){ - return NotEnoughResource("Wait for lock time out") - } - - // check again with lock - try { - if (!requestResourceService.canRequest(labelContainer, resource)) - return NotEnoughResource(s"Labels：$labels not enough resource") - } catch { - case warn: RMWarnException => return NotEnoughResource(warn.getMessage) - } - case _ => + if (wait <= 0) { + tryLock(labelContainer) + } else { + tryLock(labelContainer, wait) } resource.setLockedResource(resource.getMinResource) - // lock label resources + val labelResourceList = new util.HashMap[String, NodeResource]() labelContainer.getResourceLabels.foreach(label => { + //check all resource of label + Utils.tryCatch { + if (!requestResourceService.canRequest(labelContainer, resource)) { + return NotEnoughResource("resource check failed") + } + } { + case exception: RMWarnException => return NotEnoughResource(exception.getDesc) + } val usedResource = labelResourceService.getLabelResource(label) - if(usedResource == null){ - info(s"Resource label: ${label.getStringValue} has no usedResource, please check, refuse request usedResource！" + - s"(资源标签:${label.getStringValue}),缺少对应的资源记录，可能是该标签的资源没有初始化，请及时检查，此次申请资源失败！") - throw new RMErrorException(110022, s"Resource label: ${label.getStringValue} has no usedResource, please check, refuse request usedResource！" + - s"(资源标签:${label.getStringValue}),缺少对应的资源记录，可能是该标签的资源没有初始化，请及时检查，此次申请资源失败！") + if (usedResource == null) { + val msg = s"Resource label: ${label.getStringValue} has no usedResource, please check, refuse request usedResource" + info(msg) + throw new RMErrorException(110022, msg) } labelResourceList.put(label.getStringValue, usedResource) }) labelContainer.getResourceLabels.foreach { case label: Label[_] => val labelResource = labelResourceList.get(label.getStringValue) - if(labelResource != null){ + if (labelResource != null) { labelResource.setLeftResource(labelResource.getLeftResource - resource.getLockedResource) labelResource.setLockedResource(labelResource.getLockedResource + resource.getLockedResource) - labelResourceService.setLabelResource(label, labelResource) - info(s"ResourceChanged:${label.getStringValue} --> ${labelResource}") + labelResourceService.setLabelResource(label, labelResource, labelContainer.getCombinedUserCreatorEngineTypeLabel.getStringValue) + logger.info(s"ResourceChanged:${label.getStringValue} --> ${labelResource}") + resourceCheck(label, labelResource) } case _ => } - // record engine locked resource val tickedId = UUID.randomUUID().toString val emNode = new AMEMNode @@ -285,13 +273,13 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ engineInstanceLabel.setServiceName(labelContainer.getEngineServiceName) engineInstanceLabel.setInstance(tickedId) - nodeLabelService.addLabelToNode(engineNode.getServiceInstance,engineInstanceLabel) + nodeLabelService.addLabelToNode(engineNode.getServiceInstance, engineInstanceLabel) - labelResourceService.setEngineConnLabelResource(engineInstanceLabel, resource) + labelResourceService.setEngineConnLabelResource(engineInstanceLabel, resource, labelContainer.getCombinedUserCreatorEngineTypeLabel.getStringValue) // fire timeout check scheduled job if (RMConfiguration.RM_WAIT_EVENT_TIME_OUT.getValue > 0) Utils.defaultScheduler.schedule( - new UnlockTimeoutResourceRunnable(labels, engineInstanceLabel), + new UnlockTimeoutResourceRunnable(labels, engineInstanceLabel, tickedId), RMConfiguration.RM_WAIT_EVENT_TIME_OUT.getValue, TimeUnit.MILLISECONDS ) @@ -303,6 +291,17 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ } } + def resourceCheck(label: Label[_], labelResource: NodeResource): Unit = { + if (labelResource != null && label != null) { + val resourceInit = Resource.initResource(labelResource.getResourceType) + if (labelResource.getLockedResource < resourceInit || + labelResource.getUsedResource < resourceInit || + labelResource.getLeftResource < resourceInit) { + logger.error(s"found error resource! resource label:${label.getStringValue}, resource:${labelResource}, please check!") + } + } + } + /** * When the resource is instantiated, the total amount of resources actually occupied is returned. * 当资源被实例化后，返回实际占用的资源总量 @@ -312,39 +311,40 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ * @param usedResource */ override def resourceUsed(labels: util.List[Label[_]], usedResource: NodeResource): Unit = { - val labelContainer = labelResourceService.enrichLabels(new RMLabelContainer(labels)) + val labelContainer = labelResourceService.enrichLabels(labels) var lockedResource: NodeResource = null try { lockedResource = labelResourceService.getLabelResource(labelContainer.getEngineInstanceLabel) } catch { - case e: NullPointerException => + case e: NullPointerException => error(s"EngineInstanceLabel [${labelContainer.getEngineInstanceLabel}] cause NullPointerException") throw e } - if (lockedResource == null) { - throw new RMErrorException(110021, s"No locked resource found by engine ${labelContainer.getEngineInstanceLabel}") + val nodeInstance = nodeManagerPersistence.getEngineNode(labelContainer.getEngineInstanceLabel.getServiceInstance) + if (nodeInstance == null) { + throw new RMErrorException(RMErrorCode.LABEL_RESOURCE_NOT_FOUND.getCode, s"No serviceInstance found by engine ${labelContainer.getEngineInstanceLabel}, current label resource ${lockedResource}") + } + if (lockedResource == null || lockedResource.getLockedResource <= Resource.initResource(lockedResource.getResourceType)) { + throw new RMErrorException(RMErrorCode.LABEL_RESOURCE_NOT_FOUND.getCode, s"No locked resource found by engine ${labelContainer.getEngineInstanceLabel}, current label resource ${lockedResource}") } - /*if (!lockedResource.getLockedResource.equalsTo(usedResource.getUsedResource)) { - throw new RMErrorException(110022, s"Locked ${lockedResource.getLockedResource}, but want to use ${usedResource.getUsedResource}") - }*/ - info(s"resourceUsed ready:${labelContainer.getEMInstanceLabel.getServiceInstance}, used resource ${lockedResource.getLockedResource}") + logger.info(s"resourceUsed ready:${labelContainer.getEMInstanceLabel.getServiceInstance}, used resource ${lockedResource.getLockedResource}") val addedResource = Resource.initResource(lockedResource.getResourceType) + lockedResource.getLockedResource Utils.tryFinally { // lock labels - labelContainer.getResourceLabels.foreach { - case label: Label[_] => - labelContainer.setCurrentLabel(label.asInstanceOf[Label[_]]) - resourceLockService.tryLock(labelContainer) - case _ => + tryLock(labelContainer) + // check again after lock resource + lockedResource = labelResourceService.getLabelResource(labelContainer.getEngineInstanceLabel) + if (lockedResource == null || lockedResource.getLockedResource <= Resource.initResource(lockedResource.getResourceType)) { + throw new RMErrorException(RMErrorCode.LABEL_RESOURCE_NOT_FOUND.getCode, s"No locked resource found by engine ${labelContainer.getEngineInstanceLabel}, current label resource ${lockedResource}") } labelContainer.getResourceLabels.foreach { case engineInstanceLabel: EngineInstanceLabel => - Utils.tryCatch{ + Utils.tryCatch { lockedResource.setUsedResource(lockedResource.getLockedResource) lockedResource.setLockedResource(Resource.getZeroResource(lockedResource.getLockedResource)) - labelResourceService.setLabelResource(engineInstanceLabel, lockedResource) + labelResourceService.setLabelResource(engineInstanceLabel, lockedResource, labelContainer.getCombinedUserCreatorEngineTypeLabel.getStringValue) resourceLogService.success(ChangeType.ENGINE_INIT, engineInstanceLabel) - }{ + } { case exception: Exception => { resourceLogService.failed(ChangeType.ENGINE_INIT, engineInstanceLabel, null, exception) throw exception @@ -352,36 +352,41 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ case _ => } case label: Label[_] => - Utils.tryCatch{ + Utils.tryCatch { val labelResource = labelResourceService.getLabelResource(label) - if(labelResource != null){ + if (labelResource != null) { labelResource.setLockedResource(labelResource.getLockedResource - addedResource) - if(null == labelResource.getUsedResource) labelResource.setUsedResource(Resource.initResource(labelResource.getResourceType)) + if (null == labelResource.getUsedResource) labelResource.setUsedResource(Resource.initResource(labelResource.getResourceType)) labelResource.setUsedResource(labelResource.getUsedResource + addedResource) - labelResourceService.setLabelResource(label, labelResource) - if(label.isInstanceOf[EMInstanceLabel]){ - resourceLogService.success(ChangeType.ECM_RESOURCE_ADD, null, label.asInstanceOf[EMInstanceLabel]) + labelResourceService.setLabelResource(label, labelResource, labelContainer.getCombinedUserCreatorEngineTypeLabel.getStringValue) + label match { + case emLabel: EMInstanceLabel => + resourceLogService.success(ChangeType.ECM_RESOURCE_ADD, null, emLabel) + case _ => } + resourceCheck(label, labelResource) } - }{ + } { case exception: Exception => { - if(label.isInstanceOf[EMInstanceLabel]){ - resourceLogService.failed(ChangeType.ECM_RESOURCE_ADD, null, label.asInstanceOf[EMInstanceLabel], exception) + label match { + case emLabel: EMInstanceLabel => + resourceLogService.failed(ChangeType.ECM_RESOURCE_ADD, null, emLabel, exception) + case _ => } } } case _ => } - }{ + } { resourceLockService.unLock(labelContainer) } } - def timeCheck(labelResource: NodeResource, usedResource: NodeResource) = { - if(labelResource.getCreateTime != null && usedResource.getCreateTime != null){ - if(labelResource.getCreateTime.getTime > usedResource.getCreateTime.getTime){ - throw new RMErrorException(10022,s"no need to clear this labelResource, labelResource:${labelResource} created time is after than usedResource:${usedResource}" + - s"无需清理该标签的资源，该标签资源的创建时间晚于已用资源的创建时间") + def timeCheck(labelResource: NodeResource, usedResource: NodeResource): Unit = { + if (labelResource.getCreateTime != null && usedResource.getCreateTime != null) { + if (labelResource.getCreateTime.getTime > usedResource.getCreateTime.getTime) { + throw new RMErrorException(10022, s"no need to clear this labelResource, labelResource:${labelResource} created time is after than usedResource:${usedResource}" + + s"无需清理该标签的资源,该标签资源的创建时间晚于已用资源的创建时间") } } } @@ -393,27 +398,28 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ * @param labels */ override def resourceReleased(labels: util.List[Label[_]]): Unit = { - val labelContainer = labelResourceService.enrichLabels(new RMLabelContainer(labels)) + val labelContainer = labelResourceService.enrichLabels(labels) val usedResource = labelResourceService.getLabelResource(labelContainer.getEngineInstanceLabel) - if(usedResource == null){ - throw new RMErrorException(110021, s"No used resource found by engine ${labelContainer.getEngineInstanceLabel}") + if (usedResource == null) { + throw new RMErrorException(RMErrorCode.LABEL_RESOURCE_NOT_FOUND.getCode, s"No used resource found by engine ${labelContainer.getEngineInstanceLabel}") } - info(s"resourceRelease ready:${labelContainer.getEngineInstanceLabel.getServiceInstance},current node resource${usedResource}") + logger.info(s"resourceRelease ready:${labelContainer.getEngineInstanceLabel.getServiceInstance},current node resource${usedResource}") Utils.tryFinally { // lock labels - labelContainer.getResourceLabels.foreach { - case label: Label[_] => - labelContainer.setCurrentLabel(label.asInstanceOf[Label[_]]) - resourceLockService.tryLock(labelContainer) - case _ => + tryLock(labelContainer, RMUtils.RM_RESOURCE_LOCK_WAIT_TIME.getValue) + + // To avoid concurrent problem, check resource again after lock label + val usedResource = labelResourceService.getLabelResource(labelContainer.getEngineInstanceLabel) + if (usedResource == null) { + throw new RMErrorException(RMErrorCode.LABEL_RESOURCE_NOT_FOUND.getCode, s"No used resource found by engine ${labelContainer.getEngineInstanceLabel}") } - val tmpLabel = labelContainer.getLabels.find(_.isInstanceOf[EngineInstanceLabel]).getOrElse(null) - if(tmpLabel != null){ + val tmpLabel = labelContainer.getLabels.find(_.isInstanceOf[EngineInstanceLabel]).orNull + if (tmpLabel != null) { val engineInstanceLabel = tmpLabel.asInstanceOf[EngineInstanceLabel] Utils.tryCatch { labelResourceService.removeResourceByLabel(engineInstanceLabel) - resourceLogService.success(ChangeType.ENGINE_CLEAR, engineInstanceLabel,null) - }{ + resourceLogService.success(ChangeType.ENGINE_CLEAR, engineInstanceLabel, null) + } { case exception: Exception => { resourceLogService.failed(ChangeType.ENGINE_CLEAR, engineInstanceLabel, null, exception) throw exception @@ -423,27 +429,32 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ } labelContainer.getResourceLabels.foreach { case label: Label[_] => - Utils.tryCatch{ + Utils.tryCatch { val labelResource = labelResourceService.getLabelResource(label) - if(labelResource != null){ - timeCheck(labelResource, usedResource) - if(null != usedResource.getUsedResource){ + if (labelResource != null) { + if (label.isInstanceOf[EMInstanceLabel]) timeCheck(labelResource, usedResource) + if (null != usedResource.getUsedResource) { labelResource.setUsedResource(labelResource.getUsedResource - usedResource.getUsedResource) labelResource.setLeftResource(labelResource.getLeftResource + usedResource.getUsedResource) } - if(null != usedResource.getLockedResource){ + if (null != usedResource.getLockedResource) { labelResource.setLockedResource(labelResource.getLockedResource - usedResource.getLockedResource) labelResource.setLeftResource(labelResource.getLeftResource + usedResource.getLockedResource) } - labelResourceService.setLabelResource(label, labelResource) - if(label.isInstanceOf[EMInstanceLabel]){ - resourceLogService.success(ChangeType.ECM_Resource_MINUS, null, label.asInstanceOf[EMInstanceLabel]) + labelResourceService.setLabelResource(label, labelResource, labelContainer.getCombinedUserCreatorEngineTypeLabel.getStringValue) + label match { + case emLabel: EMInstanceLabel => + resourceLogService.success(ChangeType.ECM_Resource_MINUS, null, emLabel) + case _ => } + resourceCheck(label, labelResource) } - }{ + } { case exception: Exception => { - if(label.isInstanceOf[EMInstanceLabel]){ - resourceLogService.failed(ChangeType.ECM_Resource_MINUS, null, label.asInstanceOf[EMInstanceLabel], exception) + label match { + case emLabel: EMInstanceLabel => + resourceLogService.failed(ChangeType.ECM_Resource_MINUS, null, emLabel, exception) + case _ => } } case _ => @@ -464,20 +475,19 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ */ - override def getResourceInfo(serviceInstances: Array[ServiceInstance]): ResourceInfo = { val resourceInfo = new ResourceInfo(Lists.newArrayList()) serviceInstances.map { serviceInstance => val rmNode = new InfoRMNode var aggregatedResource: NodeResource = null serviceInstance.getApplicationName match { - case GovernanceCommonConf.ENGINE_CONN_SPRING_NAME.getValue =>{ + case GovernanceCommonConf.ENGINE_CONN_SPRING_NAME.getValue => { val engineInstanceLabel = LabelBuilderFactoryContext.getLabelBuilderFactory.createLabel(classOf[EngineInstanceLabel]) engineInstanceLabel.setServiceName(serviceInstance.getApplicationName) engineInstanceLabel.setInstance(serviceInstance.getInstance) aggregatedResource = labelResourceService.getLabelResource(engineInstanceLabel) } - case GovernanceCommonConf.ENGINE_CONN_MANAGER_SPRING_NAME.getValue =>{ + case GovernanceCommonConf.ENGINE_CONN_MANAGER_SPRING_NAME.getValue => { val emInstanceLabel = LabelBuilderFactoryContext.getLabelBuilderFactory.createLabel(classOf[EMInstanceLabel]) emInstanceLabel.setServiceName(serviceInstance.getApplicationName) emInstanceLabel.setInstance(serviceInstance.getInstance) @@ -496,83 +506,51 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ requestResourceService.getOrElse(requestResourceServices.find(_.resourceType == ResourceType.Default).get) } - class UnlockTimeoutResourceRunnable(labels: util.List[Label[_]], engineInstanceLabel: EngineInstanceLabel) extends Runnable { + class UnlockTimeoutResourceRunnable(labels: util.List[Label[_]], engineInstanceLabel: EngineInstanceLabel, ticketId: String) extends Runnable { override def run(): Unit = { - info(s"check locked resource of ${engineInstanceLabel}") - val resource = labelResourceService.getLabelResource(engineInstanceLabel) - if(resource == null){ - info(s"${engineInstanceLabel} has no resource") + logger.info(s"check locked resource of ${engineInstanceLabel}, ticketId: $ticketId") + val persistResource = resourceManagerPersistence.getNodeResourceByTicketId(ticketId) + if (null == persistResource) { + logger.info(s" ticketId $ticketId relation resource not exists") + return } - info(s"${engineInstanceLabel} has resource") - if(resource != null - && resource.getLockedResource != null - && resource.getLockedResource > Resource.getZeroResource(resource.getLockedResource)){ - info(s"start to unlock timeout resource for ${engineInstanceLabel}") - val labelContainer = labelResourceService.enrichLabels(new RMLabelContainer(labels)) - val timeoutEMEngineRequest = new TimeoutEMEngineRequest - timeoutEMEngineRequest.setTicketId(engineInstanceLabel.getInstance()) - timeoutEMEngineRequest.setUser(labelContainer.getUserCreatorLabel.getUser) - val timeoutEMEngineResponse = Sender.getSender(labelContainer.getEMInstanceLabel.getServiceInstance).ask(timeoutEMEngineRequest).asInstanceOf[TimeoutEMEngineResponse] - if(timeoutEMEngineResponse.getCanReleaseResource){ - Utils.tryFinally { - // lock labels - labelContainer.getResourceLabels.foreach { - case label: Label[_] => - labelContainer.setCurrentLabel(label.asInstanceOf[Label[_]]) - resourceLockService.tryLock(labelContainer) - case _ => - } - - labelContainer.getResourceLabels.foreach { - case engineInstanceLabel: EngineInstanceLabel => - Utils.tryCatch{ - labelResourceService.removeResourceByLabel(engineInstanceLabel) - resourceLogService.success(ChangeType.ENGINE_CLEAR, engineInstanceLabel, null) - }{ - case exception: Exception => { - resourceLogService.failed(ChangeType.ENGINE_CLEAR,engineInstanceLabel, null, exception) - throw exception - } - case _ => - } - case emInstanceLabel: EMInstanceLabel => - Utils.tryCatch{ - val labelResource = labelResourceService.getLabelResource(emInstanceLabel) - if(labelResource != null){ - labelResource.setLockedResource(labelResource.getLockedResource - resource.getLockedResource) - labelResource.setLeftResource(labelResource.getLeftResource + resource.getLockedResource) - labelResourceService.setLabelResource(emInstanceLabel, labelResource) - resourceLogService.success(ChangeType.ECM_Resource_MINUS, null, emInstanceLabel) - }else{ - resourceLogService.failed(ChangeType.ECM_Resource_MINUS, null, emInstanceLabel) - } - }{ - case exception: Exception => { - resourceLogService.failed(ChangeType.ECM_Resource_MINUS, null, emInstanceLabel, exception) - } - case _ => + val usedResource = ResourceUtils.fromPersistenceResource(persistResource) + if (usedResource != null + && usedResource.getLockedResource != null + && usedResource.getLockedResource > Resource.getZeroResource(usedResource.getLockedResource)) { + val resourceLabels = resourceManagerPersistence.getLabelsByTicketId(ticketId) + resourceLabels.foreach { label => + LabelBuilderFactoryContext.getLabelBuilderFactory.createLabel[Label[_]](label.getLabelKey, label.getStringValue) match { + case engineLabel: EngineInstanceLabel => + if (ticketId.equals(engineLabel.getInstance())) { + logger.warn(s"ticketId $ticketId lock resource timeout, now to clear resource") + resourceReleased(nodeLabelService.getNodeLabels(engineLabel.getServiceInstance)) + } else { + val node = new AMEngineNode() + node.setServiceInstance(engineLabel.getServiceInstance) + val metrics = nodeMetricManagerPersistence.getNodeMetrics(node) + if (null == metrics || NodeStatus.isAvailable(NodeStatus.values()(metrics.getStatus))) { + logger.warn(s"serviceInstance ${engineLabel.getServiceInstance} lock resource timeout, clear resource") + resourceReleased(nodeLabelService.getNodeLabels(engineLabel.getServiceInstance)) + } else { + askAgainAfter(-1) } - case _ => - } - } { - resourceLockService.unLock(labelContainer) - info(s"Timeout resource unlocked for ${engineInstanceLabel}") + } + case _ => } - } else { - askAgainAfter(timeoutEMEngineResponse.getNextAskInterval) } } info(s"finished check locked resource of ${engineInstanceLabel}") } - private def askAgainAfter(interval: Long): Unit ={ - val realInterval = if (interval == null) RMConfiguration.RM_WAIT_EVENT_TIME_OUT.getValue else interval + private def askAgainAfter(interval: Long): Unit = { + val realInterval = if (interval <= 0) RMConfiguration.RM_WAIT_EVENT_TIME_OUT.getValue else interval Utils.defaultScheduler.schedule( - new UnlockTimeoutResourceRunnable(labels, engineInstanceLabel), + new UnlockTimeoutResourceRunnable(labels, engineInstanceLabel, ticketId), realInterval, TimeUnit.MILLISECONDS ) - info(s"delayed resource unlocked for ${engineInstanceLabel}") + logger.info(s"delayed resource unlocked for ${engineInstanceLabel}") } } @@ -580,4 +558,17 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ //TODO } + + private def tryLock(labelContainer: RMLabelContainer, timeOut: Long = -1): Unit = { + labelContainer.getResourceLabels.foreach { + case label: Label[_] => + labelContainer.setCurrentLabel(label.asInstanceOf[Label[_]]) + val locked = resourceLockService.tryLock(labelContainer, timeOut) + if (!locked) { + throw new RMWarnException(110022, s"try to lock resource label ${labelContainer.getCurrentLabel} over $timeOut ms, please wait a moment and try again!") + } + case _ => + } + } + } diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/impl/DriverAndYarnReqResourceService.scala b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/impl/DriverAndYarnReqResourceService.scala index 0a1311a553..715fe6db31 100644 --- a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/impl/DriverAndYarnReqResourceService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/impl/DriverAndYarnReqResourceService.scala @@ -18,9 +18,9 @@ package org.apache.linkis.resourcemanager.service.impl import org.apache.linkis.manager.common.entity.resource.ResourceType.DriverAndYarn -import org.apache.linkis.manager.common.entity.resource.{DriverAndYarnResource, NodeResource, ResourceSerializer, ResourceType} +import org.apache.linkis.manager.common.entity.resource.{DriverAndYarnResource, NodeResource, Resource, ResourceSerializer, ResourceType, YarnResource} import org.apache.linkis.resourcemanager.domain.RMLabelContainer -import org.apache.linkis.resourcemanager.exception.RMWarnException +import org.apache.linkis.resourcemanager.exception.{RMErrorCode, RMWarnException} import org.apache.linkis.resourcemanager.external.service.ExternalResourceService import org.apache.linkis.resourcemanager.external.yarn.YarnResourceIdentifier import org.apache.linkis.resourcemanager.service.{LabelResourceService, RequestResourceService} @@ -33,7 +33,7 @@ class DriverAndYarnReqResourceService(labelResourceService: LabelResourceService override val resourceType: ResourceType = DriverAndYarn override def canRequest(labelContainer: RMLabelContainer, resource: NodeResource): Boolean = { - if(! super.canRequest(labelContainer, resource)) { + if (!super.canRequest(labelContainer, resource)) { return false } val requestedDriverAndYarnResource = resource.getMaxResource.asInstanceOf[DriverAndYarnResource] @@ -41,14 +41,30 @@ class DriverAndYarnReqResourceService(labelResourceService: LabelResourceService val yarnIdentifier = new YarnResourceIdentifier(requestedYarnResource.queueName) val providedYarnResource = externalResourceService.getResource(ResourceType.Yarn, labelContainer, yarnIdentifier) val (maxCapacity, usedCapacity) = (providedYarnResource.getMaxResource, providedYarnResource.getUsedResource) - info(s"This queue:${requestedYarnResource.queueName} used resource:$usedCapacity and max resource: $maxCapacity") + logger.debug(s"This queue: ${requestedYarnResource.queueName} used resource:$usedCapacity and max resource: $maxCapacity") val queueLeftResource = maxCapacity - usedCapacity - info(s"queue: ${requestedYarnResource.queueName} left $queueLeftResource, this request requires: $requestedYarnResource") - if(queueLeftResource < requestedYarnResource ){ - info(s"用户:${labelContainer.getUserCreatorLabel.getUser} 请求的队列资源$requestedYarnResource 大于队列剩余资源$queueLeftResource") - // TODO sendAlert(moduleInstance, user, creator, requestResource, queueLeftResource, queueLeftResource) - val notEnoughMessage = generateNotEnoughMessage(requestedYarnResource, queueLeftResource) + logger.info(s"queue: ${requestedYarnResource.queueName} left $queueLeftResource, this request requires: $requestedYarnResource") + if (queueLeftResource < requestedYarnResource) { + logger.info(s"user: ${labelContainer.getUserCreatorLabel.getUser} request queue resource $requestedYarnResource > left resource $queueLeftResource") + val notEnoughMessage = generateQueueNotEnoughMessage(requestedYarnResource, queueLeftResource) throw new RMWarnException(notEnoughMessage._1, notEnoughMessage._2) } else true } + + def generateQueueNotEnoughMessage(requestResource: Resource, availableResource: Resource) : (Int, String) = { + requestResource match { + case yarn: YarnResource => + val yarnAvailable = availableResource.asInstanceOf[YarnResource] + if(yarn.queueCores > yarnAvailable.queueCores) { + (RMErrorCode.CLUSTER_QUEUE_CPU_INSUFFICIENT.getCode, RMErrorCode.CLUSTER_QUEUE_CPU_INSUFFICIENT.getMessage) + } else if (yarn.queueMemory > yarnAvailable.queueMemory) { + (RMErrorCode.CLUSTER_QUEUE_MEMORY_INSUFFICIENT.getCode, RMErrorCode.CLUSTER_QUEUE_MEMORY_INSUFFICIENT.getMessage) + } else { + (RMErrorCode.CLUSTER_QUEUE_INSTANCES_INSUFFICIENT.getCode, RMErrorCode.CLUSTER_QUEUE_INSTANCES_INSUFFICIENT.getMessage) + } + case _ => + (RMErrorCode.CLUSTER_QUEUE_MEMORY_INSUFFICIENT.getCode, RMErrorCode.CLUSTER_QUEUE_MEMORY_INSUFFICIENT.getMessage) + } + } + } diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/impl/LabelResourceServiceImpl.scala b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/impl/LabelResourceServiceImpl.scala index 450ebf86b6..82e6ab5e05 100644 --- a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/impl/LabelResourceServiceImpl.scala +++ b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/impl/LabelResourceServiceImpl.scala @@ -18,19 +18,19 @@ package org.apache.linkis.resourcemanager.service.impl import org.apache.linkis.common.utils.Logging -import org.apache.linkis.manager.common.entity.persistence.{PersistenceLabel, PersistenceResource} +import org.apache.linkis.manager.common.entity.persistence.PersistenceResource import org.apache.linkis.manager.common.entity.resource.NodeResource import org.apache.linkis.manager.common.utils.ResourceUtils import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext -import org.apache.linkis.manager.label.entity.{CombinedLabel, Label} +import org.apache.linkis.manager.label.entity.Label import org.apache.linkis.manager.label.service.ResourceLabelService -import org.apache.linkis.manager.label.utils.LabelUtils import org.apache.linkis.manager.persistence.{LabelManagerPersistence, ResourceManagerPersistence} import org.apache.linkis.resourcemanager.domain.RMLabelContainer import org.apache.linkis.resourcemanager.service.LabelResourceService import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component +import java.util import scala.collection.JavaConversions._ @@ -52,29 +52,23 @@ class LabelResourceServiceImpl extends LabelResourceService with Logging { resourceLabelService.getResourceByLabel(label) } - override def setLabelResource(label: Label[_], nodeResource: NodeResource): Unit = { - resourceLabelService.setResourceToLabel(label, nodeResource) + override def setLabelResource(label: Label[_], nodeResource: NodeResource, source: String): Unit = { + resourceLabelService.setResourceToLabel(label, nodeResource, source) } override def getResourcesByUser(user: String): Array[NodeResource] = { resourceManagerPersistence.getResourceByUser(user).map(ResourceUtils.fromPersistenceResource).toArray } - override def enrichLabels(labelContainer: RMLabelContainer): RMLabelContainer = { - new RMLabelContainer(labelContainer.getLabels) + override def enrichLabels(labels: util.List[Label[_]]): RMLabelContainer = { + new RMLabelContainer(labels) } override def removeResourceByLabel(label: Label[_]): Unit = { resourceLabelService.removeResourceByLabel(label) } - /** - * 方法同 setLabelResource 只适用于启动引擎申请资源后设置engineConn资源 - * - * @param label - * @param nodeResource - */ - override def setEngineConnLabelResource(label: Label[_], nodeResource: NodeResource): Unit = resourceLabelService.setEngineConnResourceToLabel(label, nodeResource) + override def setEngineConnLabelResource(label: Label[_], nodeResource: NodeResource, source: String): Unit = resourceLabelService.setEngineConnResourceToLabel(label, nodeResource, source) override def getLabelsByResource(resource: PersistenceResource): Array[Label[_]] = { labelManagerPersistence.getLabelByResource(resource).map{ label => diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/impl/ResourceLogService.scala b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/impl/ResourceLogService.scala index 9c58a8ee0d..2f4d0410d1 100644 --- a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/impl/ResourceLogService.scala +++ b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/service/impl/ResourceLogService.scala @@ -19,7 +19,7 @@ package org.apache.linkis.resourcemanager.service.impl import org.apache.linkis.common.utils.Logging import org.apache.linkis.manager.common.entity.resource.{Resource, ResourceType} -import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.{CombinedLabel, Label} import org.apache.linkis.manager.label.entity.em.EMInstanceLabel import org.apache.linkis.manager.label.entity.engine.EngineInstanceLabel import org.apache.linkis.resourcemanager.service.LabelResourceService @@ -32,7 +32,9 @@ case class ResourceLogService() extends Logging{ @Autowired var labelResourceService: LabelResourceService = _ - private def printLog(changeType: String, status: String, engineLabel: EngineInstanceLabel = null, ecmLabel: EMInstanceLabel = null) :String = { + + + private def printLog(changeType: String, status: String, engineLabel: EngineInstanceLabel = null, ecmLabel: EMInstanceLabel = null): String = { val logString = new StringBuilder(changeType + " ") logString ++= (status + ", ") if (engineLabel != null) { @@ -67,12 +69,6 @@ case class ResourceLogService() extends Logging{ case ChangeType.ENGINE_CLEAR => { printLog(changeType, ChangeType.FAILED, engineLabel, ecmLabel) } - case ChangeType.ENGINE_RESOURCE_ADD => { - printLog(changeType, ChangeType.FAILED, engineLabel, ecmLabel) - } - case ChangeType.ENGINE_RESOURCE_MINUS => { - printLog(changeType, ChangeType.FAILED, engineLabel, ecmLabel) - } case ChangeType.ECM_INIT => { printLog(changeType, ChangeType.FAILED, null, ecmLabel) } @@ -87,9 +83,9 @@ case class ResourceLogService() extends Logging{ } case _ => " " } - if(exception != null){ + if (exception != null) { error(log, exception) - }else{ + } else { error(log) } } @@ -103,12 +99,6 @@ case class ResourceLogService() extends Logging{ case ChangeType.ENGINE_CLEAR => { printLog(changeType, ChangeType.SUCCESS, engineLabel, ecmLabel) } - case ChangeType.ENGINE_RESOURCE_ADD => { - printLog(changeType, ChangeType.SUCCESS, engineLabel, ecmLabel) - } - case ChangeType.ENGINE_RESOURCE_MINUS => { - printLog(changeType, ChangeType.SUCCESS, engineLabel, ecmLabel) - } case ChangeType.ECM_INIT => { printLog(changeType, ChangeType.SUCCESS, null, ecmLabel) } @@ -121,31 +111,44 @@ case class ResourceLogService() extends Logging{ case ChangeType.ECM_Resource_MINUS => { printLog(changeType, ChangeType.SUCCESS, engineLabel, ecmLabel) } - case _ =>" " + case _ => " " } info(log) } } + + def printUsedResourceNode(nodeLabel: EngineInstanceLabel, source: CombinedLabel): Unit = { + printNode(nodeLabel, source) + } + + def printReleaseResourceNode(nodeLabel: EngineInstanceLabel, source: CombinedLabel): Unit = { + printNode(nodeLabel, source) + } + + def printNode(nodeLabel: EngineInstanceLabel, source: CombinedLabel): Unit = { + val log = s"${nodeLabel.getInstance()}\t${source.getStringValue}" + info(log) + } } object ChangeType { + val ENGINE_REQUEST = "EngineResourceRequest" + val ENGINE_INIT = "EngineResourceInit" val ENGINE_CLEAR = "EngineResourceClear" - val ENGINE_RESOURCE_ADD = "EngineResourceAdd" - - val ENGINE_RESOURCE_MINUS = "EngineResourceMinus" - val ECM_INIT = "ECMResourceInit" - val ECM_CLEAR = "ECMResourceClear" + val ECM_RESOURCE_LOCK = "ECMResourceLock" val ECM_RESOURCE_ADD = "ECMResourceAdd" val ECM_Resource_MINUS = "ECMResourceMinus" + val ECM_CLEAR = "ECMResourceClear" + val SUCCESS = "success" val FAILED = "failed" diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/utils/RMUtils.scala b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/utils/RMUtils.scala index d1420e3c30..298ac8b6df 100644 --- a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/utils/RMUtils.scala +++ b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/utils/RMUtils.scala @@ -21,16 +21,41 @@ import org.apache.linkis.common.conf.{CommonVars, TimeType} import org.apache.linkis.common.utils.Logging import org.apache.linkis.manager.common.entity.persistence.PersistenceResource import org.apache.linkis.manager.common.entity.resource._ +import org.apache.linkis.manager.common.serializer.NodeResourceSerializer +import org.apache.linkis.manager.label.entity.engine.EngineType +import org.apache.linkis.server.BDPJettyServerHelper import org.json4s.DefaultFormats import org.json4s.jackson.Serialization.{read, write} +import scala.collection.JavaConverters.asScalaBufferConverter + object RMUtils extends Logging { - implicit val formats = DefaultFormats + ResourceSerializer + lazy val GSON = BDPJettyServerHelper.gson + implicit val formats = DefaultFormats + ResourceSerializer + NodeResourceSerializer + val mapper = BDPJettyServerHelper.jacksonJson val MANAGER_KILL_ENGINE_EAIT = CommonVars("wds.linkis.manager.rm.kill.engine.wait", new TimeType("30s")) - val RM_REQUEST_ENABLE = CommonVars("wds.linkis.manager.rm.request.enable", false) + val RM_REQUEST_ENABLE = CommonVars("wds.linkis.manager.rm.request.enable", true) + + val RM_RESOURCE_LOCK_WAIT_TIME = CommonVars("wds.linkis.manager.rm.lock.wait", 5 * 60 * 1000) + + val RM_DEBUG_ENABLE = CommonVars("wds.linkis.manager.rm.debug.enable", false) + + val RM_DEBUG_LOG_PATH = CommonVars("wds.linkis.manager.rm.debug.log.path", "file:///tmp/linkis/rmLog") + + val EXTERNAL_RESOURCE_REFRESH_TIME = CommonVars("wds.linkis.manager.rm.external.resource.regresh.time", new TimeType("30m")) + + val GOVERNANCE_STATION_ADMIN = CommonVars("wds.linkis.governance.station.admin", "hadoop") + + val COMBINED_USERCREATOR_ENGINETYPE = "combined_userCreator_engineType" + + val ENGINE_TYPE = CommonVars.apply("wds.linkis.configuration.engine.type", EngineType.getAllEngineTypes.asScala.mkString(",")) + + val AM_SERVICE_NAME = "linkis-cg-linkismanager" + + def deserializeResource(plainResource: String): Resource = { read[Resource](plainResource) @@ -40,6 +65,7 @@ object RMUtils extends Logging { write(resource) } + def toPersistenceResource(nodeResource: NodeResource) : PersistenceResource = { val persistenceResource = new PersistenceResource if(nodeResource.getMaxResource != null) persistenceResource.setMaxResource(serializeResource(nodeResource.getMaxResource)) @@ -51,19 +77,10 @@ object RMUtils extends Logging { persistenceResource } - /* def fromPersistenceResource(persistenceResource: PersistenceResource) : CommonNodeResource = { - val nodeResource = new CommonNodeResource - if(persistenceResource.getMaxResource != null) nodeResource.setMaxResource(deserializeResource(persistenceResource.getMaxResource)) - if(persistenceResource.getMinResource != null) nodeResource.setMinResource(deserializeResource(persistenceResource.getMinResource)) - if(persistenceResource.getLockedResource != null) nodeResource.setLockedResource(deserializeResource(persistenceResource.getLockedResource)) - if(persistenceResource.getExpectedResource != null) nodeResource.setMaxResource(deserializeResource(persistenceResource.getExpectedResource)) - if(persistenceResource.getLeftResource != null) nodeResource.setLeftResource(deserializeResource(persistenceResource.getLeftResource)) - nodeResource.setResourceType(ResourceType.valueOf(persistenceResource.getResourceType)) - nodeResource - }*/ + def aggregateNodeResource(firstNodeResource: NodeResource, secondNodeResource: NodeResource) : CommonNodeResource = { - if(firstNodeResource != null && secondNodeResource != null){ + if (firstNodeResource != null && secondNodeResource != null) { val aggregatedNodeResource = new CommonNodeResource aggregatedNodeResource.setResourceType(firstNodeResource.getResourceType) aggregatedNodeResource.setMaxResource(aggregateResource(firstNodeResource.getMaxResource, secondNodeResource.getMaxResource)) @@ -73,7 +90,7 @@ object RMUtils extends Logging { aggregatedNodeResource.setLeftResource(aggregateResource(firstNodeResource.getLeftResource, secondNodeResource.getLeftResource)) return aggregatedNodeResource } - if(firstNodeResource == null && secondNodeResource == null){ + if (firstNodeResource == null && secondNodeResource == null) { return null } if(firstNodeResource == null) { @@ -83,7 +100,7 @@ object RMUtils extends Logging { } } - def aggregateResource(firstResource: Resource, secondResource: Resource) : Resource = { + def aggregateResource(firstResource: Resource, secondResource: Resource): Resource = { (firstResource, secondResource) match { case (null, null) => null case (null, secondResource) => secondResource diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/utils/UserConfiguration.scala b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/utils/UserConfiguration.scala index 2055b3fc74..4e1005e320 100644 --- a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/utils/UserConfiguration.scala +++ b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/main/scala/org/apache/linkis/resourcemanager/utils/UserConfiguration.scala @@ -17,11 +17,8 @@ package org.apache.linkis.resourcemanager.utils -import java.util - import org.apache.linkis.common.conf.Configuration import org.apache.linkis.common.utils.{Logging, Utils} - import org.apache.linkis.governance.common.protocol.conf.{RequestQueryEngineConfigWithGlobalConfig, RequestQueryGlobalConfig, ResponseQueryConfig} import org.apache.linkis.manager.common.entity.resource._ import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext @@ -32,6 +29,8 @@ import org.apache.linkis.resourcemanager.exception.RMWarnException import org.apache.linkis.resourcemanager.utils.RMConfiguration._ import org.apache.linkis.rpc.RPCMapCache +import java.util + object UserConfiguration extends Logging { private val labelFactory = LabelBuilderFactoryContext.getLabelBuilderFactory @@ -50,7 +49,6 @@ object UserConfiguration extends Logging { override protected def createRequest(labelTuple: (UserCreatorLabel, EngineTypeLabel)): CacheableProtocol = { RequestQueryEngineConfigWithGlobalConfig(labelTuple._1, labelTuple._2) } - override protected def createMap(any: Any): util.Map[String, String] = any match { case response: ResponseQueryConfig => response.getKeyAndValue } @@ -73,7 +71,7 @@ object UserConfiguration extends Logging { (userCreateLabel, engineType) } - def getUserConfiguredResource(resourceType: ResourceType, userCreatorLabel: UserCreatorLabel, engineTypeLabel: EngineTypeLabel) = { + def getUserConfiguredResource(resourceType: ResourceType, userCreatorLabel: UserCreatorLabel, engineTypeLabel: EngineTypeLabel): Resource = { Utils.tryAndWarn{ val userCreatorAvailableResource = generateResource(resourceType, engineMapCache.getCacheMap(userCreatorLabel, engineTypeLabel)) info(s"${userCreatorLabel.getUser} on creator ${userCreatorLabel.getCreator} available engine ${engineTypeLabel.getEngineType} resource:$userCreatorAvailableResource") diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/AlertTest.scala b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/AlertTest.scala deleted file mode 100644 index c899b57112..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/test/scala/com/webank/bdp/dataworkcloud/resourcemanager/notify/AlertTest.scala +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.webank.bdp.dataworkcloud.resourcemanager.notify - -object AlertTest { - def main(args: Array[String]): Unit = { -// val service = new DefaultReqResourceService(null, null, null, null, null) -// service.sendAlert(ServiceInstance("", ""), null, null, null, null,null) - - } -} diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/test/scala/com/webank/test/TestMain.scala b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/test/scala/com/webank/test/TestMain.scala deleted file mode 100644 index 9f5076d3b1..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/test/scala/com/webank/test/TestMain.scala +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.webank.test - - -import org.apache.linkis.manager.common.entity.resource.{ResourceSerializer, YarnResource} -import org.json4s.JsonAST.JObject -import org.json4s.JsonDSL._ -import org.json4s.{CustomSerializer, DefaultFormats, Extraction} - - - -class ResultResource - -case class NotEnoughResource(reason: String = null) extends ResultResource - -class AvailableResource(val ticketId: String) extends ResultResource - -case class UserResultResource(ticketId: String, user: String) extends ResultResource - -object ResultResourceSerializer1 extends CustomSerializer[ResultResource](implicit formats => ( { - case JObject(List(("AvailableResource", JObject(List(("ticketId", ticketId)))))) => new AvailableResource(ticketId.extract[String]) -}, { - case r: AvailableResource => ("AvailableResource", ("ticketId", Extraction.decompose(r.ticketId))) -})) - -object ResultResourceSerializer2 extends CustomSerializer[ResultResource](implicit formats => ( { - case JObject(List(("NotEnoughResource", JObject(List(("reason", reason)))))) => NotEnoughResource(reason.extract[String]) - // case JObject(List(("ticketId", ticketId))) => new AvailableResource(ticketId.extract[String]) - case JObject(List(("UserResultResource", JObject(List(("ticketId", ticketId), ("user", user)))))) => - UserResultResource(ticketId.extract[String], user.extract[String]) -}, { - case r: NotEnoughResource => ("NotEnoughResource", ("reason", Extraction.decompose(r.reason))) - case r: AvailableResource => ("ticketId", Extraction.decompose(r.ticketId)) - case r: UserResultResource => ("UserResultResource", ("ticketId", r.ticketId) ~ ("user", r.user)) -})) - -object TestMain { - implicit val formats = DefaultFormats + ResultResourceSerializer1 + ResourceSerializer - - //implicit val formats = DefaultFormats + ResourceSerializer + ModuleResourceInfoSerializer + ResultResourceSerializer1 + ModuleInstanceSerializer + ModuleInfoSerializer - def main(args: Array[String]): Unit = { - val yarnResource1 = new YarnResource(100, 10, 0, "ide") - val yarnResource2 = new YarnResource(1, 1, 0, "ide") - println(yarnResource1 > yarnResource2) - } -} diff --git a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/test/scala/com/webank/test/TestResource.scala b/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/test/scala/com/webank/test/TestResource.scala deleted file mode 100644 index 061f1c6871..0000000000 --- a/linkis-computation-governance/linkis-manager/linkis-resource-manager/src/test/scala/com/webank/test/TestResource.scala +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.webank.test - -import com.fasterxml.jackson.databind.ObjectMapper -import org.apache.linkis.manager.common.entity.resource.{Resource, ResourceSerializer, ResourceType} -import org.apache.linkis.manager.common.serializer.NodeResourceSerializer -import org.apache.linkis.server.Message -import org.json4s.DefaultFormats -import org.json4s.jackson.Serialization.write - -import scala.collection.mutable - - -object TestResource { - - implicit val formats = DefaultFormats + ResourceSerializer + NodeResourceSerializer - val mapper = new ObjectMapper() - - def main(args: Array[String]): Unit = { - val message = Message.ok("") - val creatorToApplicationList = new mutable.HashMap[String, mutable.HashMap[String, Any]] - val applicationList = new mutable.HashMap[String, Any] - creatorToApplicationList.put("IDE", applicationList) - applicationList.put("engineInstances", new mutable.ArrayBuffer[Any]()) - applicationList.put("usedResource", Resource.initResource(ResourceType.LoadInstance)) - applicationList.put("maxResource", Resource.initResource(ResourceType.LoadInstance)) - val engineInstance = new mutable.HashMap[String, Any] - engineInstance.put("creator", "") - engineInstance.put("engineType", "") - engineInstance.put("instance", "") - engineInstance.put("label", "") - //engineInstance.put("resource", new CommonNodeResource) - engineInstance.put("status", "") - // TODO engineInstance.setStartTime(node) - applicationList.get("engineInstances").get.asInstanceOf[mutable.ArrayBuffer[Any]].append(engineInstance) - val a = write(creatorToApplicationList) - val b = write(engineInstance) - val c = write(applicationList) - val d = mapper.readTree(a) - } - - // def main(args: Array[String]): Unit = { - // val resource1 = new DriverAndYarnResource(new LoadInstanceResource(100,100,0), - // new YarnResource(101440 * 1024l * 1024l,50,2,"q05")) - // val resource2 = new DriverAndYarnResource(new LoadInstanceResource(50,50,0), - // new YarnResource(103424 * 1024l * 1024l,2,0,"q05")) - // - // println((resource1 - resource2).toString) - // } - // def minus(resource1:Resource,resource2:Resource) = { - // resource1 - resource2 - // } -}