From 333ec179e6b2af75059338e200ad4f157a369bbc Mon Sep 17 00:00:00 2001 From: R-Rajkumar Date: Mon, 22 Sep 2014 13:38:12 +0530 Subject: [PATCH 1/2] cluster monitors hierarchy redesigned and docker cluster monitor improved --- .../stratos/autoscaler/AutoscalerContext.java | 106 +-- .../autoscaler/KubernetesClusterContext.java | 639 ++++++++++++++++- .../autoscaler/MemberStatsContext.java | 9 - .../controller/CloudControllerClient.java | 26 + .../AutoscalerHealthStatEventReceiver.java | 647 ++++++++++++------ .../AutoscalerTopologyEventReceiver.java | 483 +++++++------ .../monitor/AbstractClusterMonitor.java | 127 ++++ .../monitor/ClusterMonitorFactory.java | 336 +++++++++ .../monitor/ContainerClusterMonitor.java | 38 + .../monitor/DockerServiceClusterMonitor.java | 156 +++++ .../monitor/KubernetesClusterMonitor.java | 186 ----- ...ractMonitor.java => VMClusterMonitor.java} | 105 +-- ...erMonitor.java => VMLbClusterMonitor.java} | 71 +- ...itor.java => VMServiceClusterMonitor.java} | 138 ++-- .../autoscaler/util/AutoscalerUtil.java | 574 ++++++++-------- .../stratos/common/enums/ClusterType.java | 5 + 16 files changed, 2417 insertions(+), 1229 deletions(-) create mode 100644 components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/AbstractClusterMonitor.java create mode 100644 components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/ClusterMonitorFactory.java create mode 100644 components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/ContainerClusterMonitor.java create mode 100644 components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/DockerServiceClusterMonitor.java delete mode 100644 components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/KubernetesClusterMonitor.java rename components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/{AbstractMonitor.java => VMClusterMonitor.java} (59%) rename components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/{LbClusterMonitor.java => VMLbClusterMonitor.java} (65%) rename components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/{ClusterMonitor.java => VMServiceClusterMonitor.java} (72%) create mode 100644 components/org.apache.stratos.common/src/main/java/org/apache/stratos/common/enums/ClusterType.java diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/AutoscalerContext.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/AutoscalerContext.java index e3eb598ec5..18003d85da 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/AutoscalerContext.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/AutoscalerContext.java @@ -20,15 +20,13 @@ */ package org.apache.stratos.autoscaler; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.stratos.autoscaler.monitor.ClusterMonitor; -import org.apache.stratos.autoscaler.monitor.KubernetesClusterMonitor; -import org.apache.stratos.autoscaler.monitor.LbClusterMonitor; - import java.util.HashMap; import java.util.Map; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.stratos.autoscaler.monitor.AbstractClusterMonitor; + /** * This class is there for accumulating cluster details which are not there in Topology */ @@ -37,20 +35,14 @@ public class AutoscalerContext { private static final Log log = LogFactory.getLog(AutoscalerContext.class); private AutoscalerContext() { try { - setMonitors(new HashMap()); - setLbMonitors(new HashMap()); - setKubernetesClusterMonitors(new HashMap()); + setClusterMonitors(new HashMap()); } catch (Exception e) { log.error("Rule evaluateMinCheck error", e); } } - // Map - private Map monitors; - // Map - private Map lbMonitors; - // Map - private Map kubernetesClusterMonitors; + // Map + private Map clusterMonitors; private static class Holder { private static final AutoscalerContext INSTANCE = new AutoscalerContext(); @@ -60,86 +52,32 @@ public static AutoscalerContext getInstance() { return Holder.INSTANCE; } - public void addMonitor(ClusterMonitor monitor) { - monitors.put(monitor.getClusterId(), monitor); - } - - public ClusterMonitor getMonitor(String clusterId) { - return monitors.get(clusterId); - } - - public boolean monitorExist(String clusterId) { - return monitors.containsKey(clusterId); - } - - public boolean lbMonitorExist(String clusterId) { - return lbMonitors.containsKey(clusterId); - } - - public LbClusterMonitor getLBMonitor(String clusterId) { - return lbMonitors.get(clusterId); - } - - public ClusterMonitor removeMonitor(String clusterId) { - if(!monitorExist(clusterId)) { - log.fatal("Cluster monitor not found for cluster id: "+clusterId); - return null; - } - log.info("Removed monitor [cluster id]: " + clusterId); - return monitors.remove(clusterId); - } - public LbClusterMonitor removeLbMonitor(String clusterId) { - if(!lbMonitorExist(clusterId)) { - log.fatal("LB monitor not found for cluster id: "+clusterId); - return null; - } - log.info("Removed LB monitor [cluster id]: " + clusterId); - return lbMonitors.remove(clusterId); - } - - public Map getMonitors() { - return monitors; - } - - - public void setMonitors(Map monitors) { - this.monitors = monitors; - } - - public void setLbMonitors(Map monitors) { - this.lbMonitors = monitors; - } - - public void addLbMonitor(LbClusterMonitor monitor) { - lbMonitors.put(monitor.getClusterId(), monitor); - } - - public void addKubernetesClusterMonitor(KubernetesClusterMonitor kubernetesClusterMonitor) { - kubernetesClusterMonitors.put(kubernetesClusterMonitor.getClusterId(), kubernetesClusterMonitor); + public void addClusterMonitor(AbstractClusterMonitor clusterMonitor) { + clusterMonitors.put(clusterMonitor.getClusterId(), clusterMonitor); } - public KubernetesClusterMonitor getKubernetesClusterMonitor(String clusterId) { - return kubernetesClusterMonitors.get(clusterId); + public AbstractClusterMonitor getClusterMonitor(String clusterId) { + return clusterMonitors.get(clusterId); } - public boolean kubernetesClusterMonitorExist(String clusterId) { - return kubernetesClusterMonitors.containsKey(clusterId); + public boolean clusterMonitorExist(String clusterId) { + return clusterMonitors.containsKey(clusterId); } - public Map getKubernetesClusterMonitors() { - return kubernetesClusterMonitors; + public Map getClusterMonitors() { + return clusterMonitors; } - public void setKubernetesClusterMonitors(Map kubernetesClusterMonitors) { - this.kubernetesClusterMonitors = kubernetesClusterMonitors; + public void setClusterMonitors(Map clusterMonitors) { + this.clusterMonitors = clusterMonitors; } - public KubernetesClusterMonitor removeKubernetesClusterMonitor(String clusterId) { - if(!kubernetesClusterMonitorExist(clusterId)) { - log.fatal("Kubernetes cluster monitor not found for cluster id: "+clusterId); + public AbstractClusterMonitor removeClusterMonitor(String clusterId) { + if(!clusterMonitorExist(clusterId)) { + log.fatal("ClusterMonitor not found for cluster id: "+clusterId); return null; } - log.info("Removed KubernetesClusterMonitor [cluster id]: " + clusterId); - return kubernetesClusterMonitors.remove(clusterId); + log.info("Removed ClusterMonitor [cluster id]: " + clusterId); + return clusterMonitors.remove(clusterId); } } diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/KubernetesClusterContext.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/KubernetesClusterContext.java index f369ac96a0..e1d5e305c3 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/KubernetesClusterContext.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/KubernetesClusterContext.java @@ -1,20 +1,647 @@ package org.apache.stratos.autoscaler; import java.io.Serializable; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.configuration.XMLConfiguration; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.stratos.autoscaler.policy.model.LoadAverage; +import org.apache.stratos.autoscaler.policy.model.MemoryConsumption; +import org.apache.stratos.autoscaler.policy.model.RequestsInFlight; +import org.apache.stratos.autoscaler.util.ConfUtil; +import org.apache.stratos.cloud.controller.stub.pojo.MemberContext; public class KubernetesClusterContext implements Serializable{ private static final long serialVersionUID = 808741789615481596L; - String kubernetesClusterID; + private static final Log log = LogFactory.getLog(KubernetesClusterContext.class); + + private String kubernetesClusterId; + private String serviceName; + + private int minReplicas; + private int maxReplicas; + private int currentReplicas = 0; + + // properties + private Properties properties; + + // 15 mints as the default + private long expiryTime = 900000; + // pending members + private List pendingMembers; + private int pendingMembersFailureCount = 0; + private final int PENDING_MEMBER_FAILURE_THRESHOLD = 5; + + // 1 day as default + private long obsoltedMemberExpiryTime = 1*24*60*60*1000; + + // members to be terminated + private Map obsoletedMembers; + + // active members + private List activeMembers; + + // termination pending members, member is added to this when Autoscaler send grace fully shut down event + private List terminationPendingMembers; + + //Keep statistics come from CEP + private Map memberStatsContexts; - public KubernetesClusterContext(String kubernetesClusterID){ - this.kubernetesClusterID = kubernetesClusterID; + //Following information will keep events details + private RequestsInFlight requestsInFlight; + private MemoryConsumption memoryConsumption; + private LoadAverage loadAverage; + + //boolean values to keep whether the requests in flight parameters are reset or not + private boolean rifReset = false, averageRifReset = false, gradientRifReset = false, secondDerivativeRifRest = false; + //boolean values to keep whether the memory consumption parameters are reset or not + private boolean memoryConsumptionReset = false, averageMemoryConsumptionReset = false, + gradientMemoryConsumptionReset = false, secondDerivativeMemoryConsumptionRest = false; + //boolean values to keep whether the load average parameters are reset or not + private boolean loadAverageReset = false, averageLoadAverageReset = false, gradientLoadAverageReset = false, + secondDerivativeLoadAverageRest = false; + + public KubernetesClusterContext(String kubernetesClusterId){ + this.kubernetesClusterId = kubernetesClusterId; + this.pendingMembers = new ArrayList(); + this.activeMembers = new ArrayList(); + this.terminationPendingMembers = new ArrayList(); + this.obsoletedMembers = new ConcurrentHashMap(); + this.memberStatsContexts = new ConcurrentHashMap(); + this.requestsInFlight = new RequestsInFlight(); + this.loadAverage = new LoadAverage(); + this.memoryConsumption = new MemoryConsumption(); + + // check if a different value has been set for expiryTime + XMLConfiguration conf = ConfUtil.getInstance(null).getConfiguration(); + expiryTime = conf.getLong("autoscaler.member.expiryTimeout", 900000); + if (log.isDebugEnabled()) { + log.debug("Member expiry time is set to: " + expiryTime); + } + + Thread th = new Thread(new PendingMemberWatcher(this)); + th.start(); + Thread th2 = new Thread(new ObsoletedMemberWatcher(this)); + th2.start(); } public String getKubernetesClusterID() { - return kubernetesClusterID; + return kubernetesClusterId; + } + public void setKubernetesClusterID(String kubernetesClusterId) { + this.kubernetesClusterId = kubernetesClusterId; + } + + public List getPendingMembers() { + return pendingMembers; + } + + public void setPendingMembers(List pendingMembers) { + this.pendingMembers = pendingMembers; + } + + public int getActiveMemberCount() { + return activeMembers.size(); + } + + public void setActiveMembers(List activeMembers) { + this.activeMembers = activeMembers; + } + + public int getMinReplicas() { + return minReplicas; + } + + public void setMinReplicas(int minReplicas) { + this.minReplicas = minReplicas; + } + + public int getMaxReplicas() { + return maxReplicas; + } + + public void setMaxReplicas(int maxReplicas) { + this.maxReplicas = maxReplicas; + } + + public int getCurrentReplicas() { + return currentReplicas; + } + + public void setCurrentReplicas(int currentReplicas) { + this.currentReplicas = currentReplicas; + } + + public void addPendingMember(MemberContext ctxt) { + this.pendingMembers.add(ctxt); + } + + public boolean removePendingMember(String id) { + if (id == null) { + return false; + } + for (Iterator iterator = pendingMembers.iterator(); iterator.hasNext();) { + MemberContext pendingMember = (MemberContext) iterator.next(); + if (id.equals(pendingMember.getMemberId())) { + iterator.remove(); + return true; + } + + } + + return false; + } + + public void movePendingMemberToActiveMembers(String memberId) { + if (memberId == null) { + return; + } + Iterator iterator = pendingMembers.listIterator(); + while (iterator.hasNext()) { + MemberContext pendingMember = iterator.next(); + if (pendingMember == null) { + iterator.remove(); + continue; + } + if (memberId.equals(pendingMember.getMemberId())) { + // member is activated + // remove from pending list + iterator.remove(); + // add to the activated list + this.activeMembers.add(pendingMember); + pendingMembersFailureCount = 0; + if (log.isDebugEnabled()) { + log.debug(String.format( + "Pending member is removed and added to the " + + "activated member list. [Member Id] %s", + memberId)); + } + break; + } + } + } + + public void moveActiveMemberToTerminationPendingMembers(String memberId) { + if (memberId == null) { + return; + } + Iterator iterator = activeMembers.listIterator(); + while (iterator.hasNext()) { + MemberContext activeMember = iterator.next(); + if (activeMember == null) { + iterator.remove(); + continue; + } + if (memberId.equals(activeMember.getMemberId())) { + // member is activated + // remove from pending list + iterator.remove(); + // add to the activated list + this.terminationPendingMembers.add(activeMember); + if (log.isDebugEnabled()) { + log.debug(String + .format("Active member is removed and added to the " + + "termination pending member list. [Member Id] %s", + memberId)); + } + break; + } + } + } + + public void addActiveMember(MemberContext ctxt) { + this.activeMembers.add(ctxt); + } + + public void removeActiveMember(MemberContext ctxt) { + this.activeMembers.remove(ctxt); + } + + public boolean removeTerminationPendingMember(String memberId) { + boolean terminationPendingMemberAvailable = false; + for (MemberContext memberContext : terminationPendingMembers) { + if (memberContext.getMemberId().equals(memberId)) { + terminationPendingMemberAvailable = true; + terminationPendingMembers.remove(memberContext); + break; + } + } + return terminationPendingMemberAvailable; + } + + public long getObsoltedMemberExpiryTime() { + return obsoltedMemberExpiryTime; + } + + public void setObsoltedMemberExpiryTime(long obsoltedMemberExpiryTime) { + this.obsoltedMemberExpiryTime = obsoltedMemberExpiryTime; + } + + public void addObsoleteMember(MemberContext ctxt) { + this.obsoletedMembers.put(ctxt.getMemberId(), ctxt); + } + + public boolean removeObsoleteMember(String memberId) { + if (this.obsoletedMembers.remove(memberId) == null) { + return false; + } + return true; + } + + public long getExpiryTime() { + return expiryTime; + } + + public void setExpiryTime(long expiryTime) { + this.expiryTime = expiryTime; + } + + public Map getObsoletedMembers() { + return obsoletedMembers; + } + + public void setObsoletedMembers(Map obsoletedMembers) { + this.obsoletedMembers = obsoletedMembers; + } + + public Map getMemberStatsContexts() { + return memberStatsContexts; + } + + public MemberStatsContext getMemberStatsContext(String memberId) { + return memberStatsContexts.get(memberId); + } + + public void addMemberStatsContext(MemberStatsContext ctxt) { + this.memberStatsContexts.put(ctxt.getMemberId(), ctxt); + } + + public void removeMemberStatsContext(String memberId) { + this.memberStatsContexts.remove(memberId); + } + + public Properties getProperties() { + return properties; + } + + public void setProperties(Properties properties) { + this.properties = properties; + } + + public String getServiceName() { + return serviceName; + } + + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + public List getTerminationPendingMembers() { + return terminationPendingMembers; + } + + public void setTerminationPendingMembers( + List terminationPendingMembers) { + this.terminationPendingMembers = terminationPendingMembers; } - public void setKubernetesClusterID(String kubernetesClusterID) { - this.kubernetesClusterID = kubernetesClusterID; + + public int getTotalMemberCount() { + return activeMembers.size() + pendingMembers.size() + + terminationPendingMembers.size(); + } + + public int getNonTerminatedMemberCount() { + return activeMembers.size() + pendingMembers.size() + + terminationPendingMembers.size(); + } + + public List getActiveMembers() { + return activeMembers; + } + + public boolean removeActiveMemberById(String memberId) { + boolean removeActiveMember = false; + synchronized (activeMembers) { + Iterator iterator = activeMembers.listIterator(); + while (iterator.hasNext()) { + MemberContext memberContext = iterator.next(); + if (memberId.equals(memberContext.getMemberId())) { + iterator.remove(); + removeActiveMember = true; + + break; + } + } + } + return removeActiveMember; + } + + public boolean activeMemberExist(String memberId) { + + for (MemberContext memberContext : activeMembers) { + if (memberId.equals(memberContext.getMemberId())) { + return true; + } + } + return false; + } + + private class PendingMemberWatcher implements Runnable { + private KubernetesClusterContext ctxt; + + public PendingMemberWatcher(KubernetesClusterContext ctxt) { + this.ctxt = ctxt; + } + + @Override + public void run() { + + while (true) { + long expiryTime = ctxt.getExpiryTime(); + List pendingMembers = ctxt.getPendingMembers(); + + synchronized (pendingMembers) { + Iterator iterator = pendingMembers + .listIterator(); + while (iterator.hasNext()) { + MemberContext pendingMember = iterator.next(); + + if (pendingMember == null) { + continue; + } + long pendingTime = System.currentTimeMillis() + - pendingMember.getInitTime(); + if (pendingTime >= expiryTime) { + + iterator.remove(); + log.info("Pending state of member: " + + pendingMember.getMemberId() + + " is expired. " + + "Adding as an obsoleted member."); + // member should be terminated + ctxt.addObsoleteMember(pendingMember); + pendingMembersFailureCount++; + if (pendingMembersFailureCount > PENDING_MEMBER_FAILURE_THRESHOLD) { + setExpiryTime(expiryTime * 2);// Doubles the + // expiry time + // after the + // threshold of + // failure + // exceeded + // TODO Implement an alerting system: + // STRATOS-369 + } + } + } + } + + try { + // TODO find a constant + Thread.sleep(15000); + } catch (InterruptedException ignore) { + } + } + } + + } + + private class ObsoletedMemberWatcher implements Runnable { + private KubernetesClusterContext ctxt; + + public ObsoletedMemberWatcher(KubernetesClusterContext ctxt) { + this.ctxt = ctxt; + } + + @Override + public void run() { + while (true) { + + long obsoltedMemberExpiryTime = ctxt + .getObsoltedMemberExpiryTime(); + Map obsoletedMembers = ctxt + .getObsoletedMembers(); + Iterator> iterator = obsoletedMembers + .entrySet().iterator(); + + while (iterator.hasNext()) { + Map.Entry pairs = iterator.next(); + MemberContext obsoleteMember = (MemberContext) pairs + .getValue(); + if (obsoleteMember == null) { + continue; + } + long obsoleteTime = System.currentTimeMillis() + - obsoleteMember.getInitTime(); + if (obsoleteTime >= obsoltedMemberExpiryTime) { + iterator.remove(); + } + } + try { + // TODO find a constant + Thread.sleep(15000); + } catch (InterruptedException ignore) { + } + } + } + } + + public float getAverageRequestsInFlight() { + return requestsInFlight.getAverage(); + } + + public void setAverageRequestsInFlight(float averageRequestsInFlight) { + requestsInFlight.setAverage(averageRequestsInFlight); + averageRifReset = true; + if (secondDerivativeRifRest && gradientRifReset) { + rifReset = true; + if (log.isDebugEnabled()) { + log.debug(String + .format("Requests in flights stats are reset, ready to do scale check [kub cluster] %s", + this.kubernetesClusterId)); + } + } + } + + public float getRequestsInFlightSecondDerivative() { + return requestsInFlight.getSecondDerivative(); + } + + public void setRequestsInFlightSecondDerivative( + float requestsInFlightSecondDerivative) { + requestsInFlight.setSecondDerivative(requestsInFlightSecondDerivative); + secondDerivativeRifRest = true; + if (averageRifReset && gradientRifReset) { + rifReset = true; + if (log.isDebugEnabled()) { + log.debug(String + .format("Requests in flights stats are reset, ready to do scale check [kub cluster] %s", + this.kubernetesClusterId)); + } + } + } + + public float getRequestsInFlightGradient() { + return requestsInFlight.getGradient(); + } + + public void setRequestsInFlightGradient(float requestsInFlightGradient) { + requestsInFlight.setGradient(requestsInFlightGradient); + gradientRifReset = true; + if (secondDerivativeRifRest && averageRifReset) { + rifReset = true; + if (log.isDebugEnabled()) { + log.debug(String + .format("Requests in flights stats are reset, ready to do scale check [kub cluster] %s", + this.kubernetesClusterId)); + } + } + } + + public boolean isRifReset() { + return rifReset; + } + + public void setRifReset(boolean rifReset) { + this.rifReset = rifReset; + this.averageRifReset = rifReset; + this.gradientRifReset = rifReset; + this.secondDerivativeRifRest = rifReset; + } + + public float getAverageMemoryConsumption() { + return memoryConsumption.getAverage(); + } + + public void setAverageMemoryConsumption(float averageMemoryConsumption) { + memoryConsumption.setAverage(averageMemoryConsumption); + averageMemoryConsumptionReset = true; + if (secondDerivativeMemoryConsumptionRest + && gradientMemoryConsumptionReset) { + memoryConsumptionReset = true; + if (log.isDebugEnabled()) { + log.debug(String + .format("Memory consumption stats are reset, ready to do scale check [kub cluster] %s", + this.kubernetesClusterId)); + } + } + } + + public float getMemoryConsumptionSecondDerivative() { + return memoryConsumption.getSecondDerivative(); + } + + public void setMemoryConsumptionSecondDerivative( + float memoryConsumptionSecondDerivative) { + memoryConsumption + .setSecondDerivative(memoryConsumptionSecondDerivative); + secondDerivativeMemoryConsumptionRest = true; + if (averageMemoryConsumptionReset && gradientMemoryConsumptionReset) { + memoryConsumptionReset = true; + if (log.isDebugEnabled()) { + log.debug(String + .format("Memory consumption stats are reset, ready to do scale check [kub cluster] %s", + this.kubernetesClusterId)); + } + } + } + + public float getMemoryConsumptionGradient() { + return memoryConsumption.getGradient(); + } + + public void setMemoryConsumptionGradient(float memoryConsumptionGradient) { + memoryConsumption.setGradient(memoryConsumptionGradient); + gradientMemoryConsumptionReset = true; + if (secondDerivativeMemoryConsumptionRest + && averageMemoryConsumptionReset) { + memoryConsumptionReset = true; + if (log.isDebugEnabled()) { + log.debug(String + .format("Memory consumption stats are reset, ready to do scale check [kub cluster] %s", + this.kubernetesClusterId)); + } + } + } + + public boolean isMemoryConsumptionReset() { + return memoryConsumptionReset; + } + + public void setMemoryConsumptionReset(boolean memoryConsumptionReset) { + this.memoryConsumptionReset = memoryConsumptionReset; + this.averageMemoryConsumptionReset = memoryConsumptionReset; + this.gradientMemoryConsumptionReset = memoryConsumptionReset; + this.secondDerivativeMemoryConsumptionRest = memoryConsumptionReset; + } + + + public float getAverageLoadAverage() { + return loadAverage.getAverage(); + } + + public void setAverageLoadAverage(float averageLoadAverage) { + loadAverage.setAverage(averageLoadAverage); + averageLoadAverageReset = true; + if (secondDerivativeLoadAverageRest && gradientLoadAverageReset) { + loadAverageReset = true; + if (log.isDebugEnabled()) { + log.debug(String + .format("Load average stats are reset, ready to do scale check [kub cluster] %s", + this.kubernetesClusterId)); + } + } + } + + public float getLoadAverageSecondDerivative() { + return loadAverage.getSecondDerivative(); + } + + public void setLoadAverageSecondDerivative(float loadAverageSecondDerivative) { + loadAverage.setSecondDerivative(loadAverageSecondDerivative); + secondDerivativeLoadAverageRest = true; + if (averageLoadAverageReset && gradientLoadAverageReset) { + loadAverageReset = true; + if (log.isDebugEnabled()) { + log.debug(String + .format("Load average stats are reset, ready to do scale check [kub cluster] %s", + this.kubernetesClusterId)); + } + } + } + + public float getLoadAverageGradient() { + return loadAverage.getGradient(); + } + + public void setLoadAverageGradient(float loadAverageGradient) { + loadAverage.setGradient(loadAverageGradient); + gradientLoadAverageReset = true; + if (secondDerivativeLoadAverageRest && averageLoadAverageReset) { + loadAverageReset = true; + if (log.isDebugEnabled()) { + log.debug(String + .format("Load average stats are reset, ready to do scale check [kub cluster] %s", + this.kubernetesClusterId)); + } + } + } + + public boolean isLoadAverageReset() { + return loadAverageReset; + } + + public void setLoadAverageReset(boolean loadAverageReset) { + this.loadAverageReset = loadAverageReset; + this.averageLoadAverageReset = loadAverageReset; + this.gradientLoadAverageReset = loadAverageReset; + this.secondDerivativeLoadAverageRest = loadAverageReset; } } diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/MemberStatsContext.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/MemberStatsContext.java index 9877fe8f6e..ac8b61ae81 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/MemberStatsContext.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/MemberStatsContext.java @@ -45,20 +45,11 @@ public void setMemberId(String memberId) { this.memberId = memberId; } - public LoadAverage getLoadAverage() { return loadAverage; } -// -// public void setLoadAverage(LoadAverage loadAverage) { -// this.loadAverage = loadAverage; -// } public MemoryConsumption getMemoryConsumption() { return memoryConsumption; } -// -// public void setMemoryConsumption(MemoryConsumption memoryConsumption) { -// this.memoryConsumption = memoryConsumption; -// } } diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/client/cloud/controller/CloudControllerClient.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/client/cloud/controller/CloudControllerClient.java index 69c9f5e355..3f900b666d 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/client/cloud/controller/CloudControllerClient.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/client/cloud/controller/CloudControllerClient.java @@ -276,5 +276,31 @@ public synchronized MemberContext createContainer(String kubernetesClusterId, St throw new SpawningException(e.getMessage(), e); } } + + public synchronized void terminateContainer(String memberId) throws TerminationException { + try { + if(log.isInfoEnabled()) { + log.info(String.format("Terminating container via cloud controller: [member] %s", memberId)); + } + long startTime = System.currentTimeMillis(); + stub.terminateInstance(memberId); + if(log.isDebugEnabled()) { + long endTime = System.currentTimeMillis(); + log.debug(String.format("Service call terminateContainer() returned in %dms", (endTime - startTime))); + } + } catch (RemoteException e) { + String msg = e.getMessage(); + log.error(msg, e); + throw new TerminationException(msg, e); + } catch (CloudControllerServiceInvalidMemberExceptionException e) { + String msg = e.getFaultMessage().getInvalidMemberException().getMessage(); + log.error(msg, e); + throw new TerminationException(msg, e); + } catch (CloudControllerServiceInvalidCartridgeTypeExceptionException e) { + String msg = e.getFaultMessage().getInvalidCartridgeTypeException().getMessage(); + log.error(msg, e); + throw new TerminationException(msg, e); + } + } } diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/message/receiver/health/AutoscalerHealthStatEventReceiver.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/message/receiver/health/AutoscalerHealthStatEventReceiver.java index 52952c342c..2efcef0a49 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/message/receiver/health/AutoscalerHealthStatEventReceiver.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/message/receiver/health/AutoscalerHealthStatEventReceiver.java @@ -21,14 +21,19 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.stratos.autoscaler.AutoscalerContext; +import org.apache.stratos.autoscaler.KubernetesClusterContext; import org.apache.stratos.autoscaler.MemberStatsContext; import org.apache.stratos.autoscaler.NetworkPartitionContext; import org.apache.stratos.autoscaler.PartitionContext; import org.apache.stratos.autoscaler.client.cloud.controller.CloudControllerClient; import org.apache.stratos.autoscaler.exception.TerminationException; -import org.apache.stratos.autoscaler.monitor.AbstractMonitor; +import org.apache.stratos.autoscaler.monitor.AbstractClusterMonitor; +import org.apache.stratos.autoscaler.monitor.ContainerClusterMonitor; +import org.apache.stratos.autoscaler.monitor.DockerServiceClusterMonitor; +import org.apache.stratos.autoscaler.monitor.VMClusterMonitor; import org.apache.stratos.autoscaler.policy.model.LoadAverage; import org.apache.stratos.autoscaler.policy.model.MemoryConsumption; +import org.apache.stratos.common.enums.ClusterType; import org.apache.stratos.messaging.domain.topology.Cluster; import org.apache.stratos.messaging.domain.topology.Member; import org.apache.stratos.messaging.domain.topology.Service; @@ -94,30 +99,42 @@ protected void onEvent(org.apache.stratos.messaging.event.Event event) { clusterId, networkPartitionId, floatValue)); } AutoscalerContext asCtx = AutoscalerContext.getInstance(); - AbstractMonitor monitor; + AbstractClusterMonitor monitor; - if(asCtx.monitorExist(clusterId)){ - monitor = asCtx.getMonitor(clusterId); - }else if(asCtx.lbMonitorExist(clusterId)){ - monitor = asCtx.getLBMonitor(clusterId); - }else{ + if(asCtx.clusterMonitorExist(clusterId)){ + monitor = asCtx.getClusterMonitor(clusterId); + } else { if(log.isDebugEnabled()){ log.debug(String.format("A cluster monitor is not found in autoscaler context [cluster] %s", clusterId)); } return; } - if(null != monitor){ - NetworkPartitionContext networkPartitionContext = monitor.getNetworkPartitionCtxt(networkPartitionId); - if(null != networkPartitionContext){ - networkPartitionContext.setAverageLoadAverage(floatValue); - } else { - if(log.isDebugEnabled()) { - log.debug(String.format("Network partition context is not available for :" + - " [network partition] %s", networkPartitionId)); + + if(monitor.getClusterType() == ClusterType.VMServiceCluster + || monitor.getClusterType() == ClusterType.VMLbCluster) { + + if(null != monitor){ + NetworkPartitionContext networkPartitionContext = ((VMClusterMonitor) monitor).getNetworkPartitionCtxt(networkPartitionId); + if(null != networkPartitionContext){ + networkPartitionContext.setAverageLoadAverage(floatValue); + } else { + if(log.isDebugEnabled()) { + log.debug(String.format("Network partition context is not available for :" + + " [network partition] %s", networkPartitionId)); + } } } + } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { + KubernetesClusterContext kubernetesClusterContext = ((DockerServiceClusterMonitor) monitor).getKubernetesClusterCtxt(); + if (null != kubernetesClusterContext) { + kubernetesClusterContext.setAverageLoadAverage(floatValue); + } else { + if(log.isDebugEnabled()) { + log.debug(String.format("Kubernetes cluster context is not available for :" + + " [cluster] %s", clusterId)); + } + } } - } }); @@ -136,32 +153,43 @@ protected void onEvent(org.apache.stratos.messaging.event.Event event) { clusterId, networkPartitionId, floatValue)); } AutoscalerContext asCtx = AutoscalerContext.getInstance(); - AbstractMonitor monitor; + AbstractClusterMonitor monitor; - if(asCtx.monitorExist(clusterId)){ - monitor = asCtx.getMonitor(clusterId); - }else if(asCtx.lbMonitorExist(clusterId)){ - monitor = asCtx.getLBMonitor(clusterId); - }else{ + if(asCtx.clusterMonitorExist(clusterId)){ + monitor = asCtx.getClusterMonitor(clusterId); + } else { if(log.isDebugEnabled()){ log.debug(String.format("A cluster monitor is not found in autoscaler context [cluster] %s", clusterId)); } return; } - - if(null != monitor){ - NetworkPartitionContext networkPartitionContext = monitor.getNetworkPartitionCtxt(networkPartitionId); - if(null != networkPartitionContext){ - networkPartitionContext.setAverageMemoryConsumption(floatValue); - } else { - if(log.isDebugEnabled()) { - log.debug(String.format("Network partition context is not available for :" + - " [network partition] %s", networkPartitionId)); + + if(monitor.getClusterType() == ClusterType.VMServiceCluster + || monitor.getClusterType() == ClusterType.VMLbCluster) { + + if(null != monitor){ + NetworkPartitionContext networkPartitionContext = ((VMClusterMonitor) monitor).getNetworkPartitionCtxt(networkPartitionId); + if(null != networkPartitionContext){ + networkPartitionContext.setAverageMemoryConsumption(floatValue); + } else { + if(log.isDebugEnabled()) { + log.debug(String.format("Network partition context is not available for :" + + " [network partition] %s", networkPartitionId)); + } } } + } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { + KubernetesClusterContext kubernetesClusterContext = ((DockerServiceClusterMonitor) monitor).getKubernetesClusterCtxt(); + if (null != kubernetesClusterContext) { + kubernetesClusterContext.setAverageMemoryConsumption(floatValue); + } else { + if(log.isDebugEnabled()) { + log.debug(String.format("Kubernetes cluster context is not available for :" + + " [cluster] %s", clusterId)); + } + } } } - }); healthStatEventReceiver.addEventListener(new AverageRequestsInFlightEventListener() { @Override @@ -178,28 +206,41 @@ protected void onEvent(org.apache.stratos.messaging.event.Event event) { clusterId, networkPartitionId, floatValue)); } AutoscalerContext asCtx = AutoscalerContext.getInstance(); - AbstractMonitor monitor; + AbstractClusterMonitor monitor; - if(asCtx.monitorExist(clusterId)){ - monitor = asCtx.getMonitor(clusterId); - }else if(asCtx.lbMonitorExist(clusterId)){ - monitor = asCtx.getLBMonitor(clusterId); - }else{ + if(asCtx.clusterMonitorExist(clusterId)){ + monitor = asCtx.getClusterMonitor(clusterId); + } else { if(log.isDebugEnabled()){ log.debug(String.format("A cluster monitor is not found in autoscaler context [cluster] %s", clusterId)); } return; } - if(null != monitor){ - NetworkPartitionContext networkPartitionContext = monitor.getNetworkPartitionCtxt(networkPartitionId); - if(null != networkPartitionContext){ - networkPartitionContext.setAverageRequestsInFlight(floatValue); - } else { - if(log.isDebugEnabled()) { - log.debug(String.format("Network partition context is not available for :" + - " [network partition] %s", networkPartitionId)); + + if(monitor.getClusterType() == ClusterType.VMServiceCluster + || monitor.getClusterType() == ClusterType.VMLbCluster) { + + if(null != monitor){ + NetworkPartitionContext networkPartitionContext = ((VMClusterMonitor) monitor).getNetworkPartitionCtxt(networkPartitionId); + if(null != networkPartitionContext){ + networkPartitionContext.setAverageRequestsInFlight(floatValue); + } else { + if(log.isDebugEnabled()) { + log.debug(String.format("Network partition context is not available for :" + + " [network partition] %s", networkPartitionId)); + } } } + } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { + KubernetesClusterContext kubernetesClusterContext = ((DockerServiceClusterMonitor) monitor).getKubernetesClusterCtxt(); + if (null != kubernetesClusterContext) { + kubernetesClusterContext.setAverageRequestsInFlight(floatValue); + } else { + if(log.isDebugEnabled()) { + log.debug(String.format("Kubernetes cluster context is not available for :" + + " [cluster] %s", clusterId)); + } + } } } @@ -218,28 +259,41 @@ protected void onEvent(org.apache.stratos.messaging.event.Event event) { clusterId, networkPartitionId, floatValue)); } AutoscalerContext asCtx = AutoscalerContext.getInstance(); - AbstractMonitor monitor; + AbstractClusterMonitor monitor; - if(asCtx.monitorExist(clusterId)){ - monitor = asCtx.getMonitor(clusterId); - }else if(asCtx.lbMonitorExist(clusterId)){ - monitor = asCtx.getLBMonitor(clusterId); - }else{ + if(asCtx.clusterMonitorExist(clusterId)){ + monitor = asCtx.getClusterMonitor(clusterId); + } else { if(log.isDebugEnabled()){ log.debug(String.format("A cluster monitor is not found in autoscaler context [cluster] %s", clusterId)); } return; } - if(null != monitor){ - NetworkPartitionContext networkPartitionContext = monitor.getNetworkPartitionCtxt(networkPartitionId); - if(null != networkPartitionContext){ - networkPartitionContext.setLoadAverageGradient(floatValue); - } else { - if(log.isDebugEnabled()) { - log.debug(String.format("Network partition context is not available for :" + - " [network partition] %s", networkPartitionId)); + + if(monitor.getClusterType() == ClusterType.VMServiceCluster + || monitor.getClusterType() == ClusterType.VMLbCluster){ + + if(null != monitor){ + NetworkPartitionContext networkPartitionContext = ((VMClusterMonitor) monitor).getNetworkPartitionCtxt(networkPartitionId); + if(null != networkPartitionContext){ + networkPartitionContext.setLoadAverageGradient(floatValue); + } else { + if(log.isDebugEnabled()) { + log.debug(String.format("Network partition context is not available for :" + + " [network partition] %s", networkPartitionId)); + } } } + } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { + KubernetesClusterContext kubernetesClusterContext = ((DockerServiceClusterMonitor) monitor).getKubernetesClusterCtxt(); + if (null != kubernetesClusterContext) { + kubernetesClusterContext.setLoadAverageGradient(floatValue); + } else { + if(log.isDebugEnabled()) { + log.debug(String.format("Kubernetes cluster context is not available for :" + + " [cluster] %s", clusterId)); + } + } } } @@ -259,28 +313,41 @@ protected void onEvent(org.apache.stratos.messaging.event.Event event) { clusterId, networkPartitionId, floatValue)); } AutoscalerContext asCtx = AutoscalerContext.getInstance(); - AbstractMonitor monitor; + AbstractClusterMonitor monitor; - if(asCtx.monitorExist(clusterId)){ - monitor = asCtx.getMonitor(clusterId); - }else if(asCtx.lbMonitorExist(clusterId)){ - monitor = asCtx.getLBMonitor(clusterId); - }else{ + if(asCtx.clusterMonitorExist(clusterId)){ + monitor = asCtx.getClusterMonitor(clusterId); + } else { if(log.isDebugEnabled()){ log.debug(String.format("A cluster monitor is not found in autoscaler context [cluster] %s", clusterId)); } return; - }; - if(null != monitor){ - NetworkPartitionContext networkPartitionContext = monitor.getNetworkPartitionCtxt(networkPartitionId); - if(null != networkPartitionContext){ - networkPartitionContext.setMemoryConsumptionGradient(floatValue); - } else { - if(log.isDebugEnabled()) { - log.debug(String.format("Network partition context is not available for :" + - " [network partition] %s", networkPartitionId)); + } + + if(monitor.getClusterType() == ClusterType.VMServiceCluster + || monitor.getClusterType() == ClusterType.VMLbCluster){ + + if(null != monitor){ + NetworkPartitionContext networkPartitionContext = ((VMClusterMonitor) monitor).getNetworkPartitionCtxt(networkPartitionId); + if(null != networkPartitionContext){ + networkPartitionContext.setMemoryConsumptionGradient(floatValue); + } else { + if(log.isDebugEnabled()) { + log.debug(String.format("Network partition context is not available for :" + + " [network partition] %s", networkPartitionId)); + } } } + } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { + KubernetesClusterContext kubernetesClusterContext = ((DockerServiceClusterMonitor) monitor).getKubernetesClusterCtxt(); + if (null != kubernetesClusterContext) { + kubernetesClusterContext.setMemoryConsumptionGradient(floatValue); + } else { + if(log.isDebugEnabled()) { + log.debug(String.format("Kubernetes cluster context is not available for :" + + " [cluster] %s", clusterId)); + } + } } } @@ -299,28 +366,41 @@ protected void onEvent(org.apache.stratos.messaging.event.Event event) { clusterId, networkPartitionId, floatValue)); } AutoscalerContext asCtx = AutoscalerContext.getInstance(); - AbstractMonitor monitor; + AbstractClusterMonitor monitor; - if(asCtx.monitorExist(clusterId)){ - monitor = asCtx.getMonitor(clusterId); - }else if(asCtx.lbMonitorExist(clusterId)){ - monitor = asCtx.getLBMonitor(clusterId); - }else{ + if(asCtx.clusterMonitorExist(clusterId)){ + monitor = asCtx.getClusterMonitor(clusterId); + } else { if(log.isDebugEnabled()){ log.debug(String.format("A cluster monitor is not found in autoscaler context [cluster] %s", clusterId)); } return; } - if(null != monitor){ - NetworkPartitionContext networkPartitionContext = monitor.getNetworkPartitionCtxt(networkPartitionId); - if(null != networkPartitionContext){ - networkPartitionContext.setRequestsInFlightGradient(floatValue); - } else { - if(log.isDebugEnabled()) { - log.debug(String.format("Network partition context is not available for :" + - " [network partition] %s", networkPartitionId)); + + if(monitor.getClusterType() == ClusterType.VMServiceCluster + || monitor.getClusterType() == ClusterType.VMLbCluster){ + + if(null != monitor){ + NetworkPartitionContext networkPartitionContext = ((VMClusterMonitor) monitor).getNetworkPartitionCtxt(networkPartitionId); + if(null != networkPartitionContext){ + networkPartitionContext.setRequestsInFlightGradient(floatValue); + } else { + if(log.isDebugEnabled()) { + log.debug(String.format("Network partition context is not available for :" + + " [network partition] %s", networkPartitionId)); + } } } + } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { + KubernetesClusterContext kubernetesClusterContext = ((DockerServiceClusterMonitor) monitor).getKubernetesClusterCtxt(); + if (null != kubernetesClusterContext) { + kubernetesClusterContext.setRequestsInFlightGradient(floatValue); + } else { + if(log.isDebugEnabled()) { + log.debug(String.format("Kubernetes cluster context is not available for :" + + " [cluster] %s", clusterId)); + } + } } } @@ -461,28 +541,41 @@ protected void onEvent(org.apache.stratos.messaging.event.Event event) { clusterId, networkPartitionId, floatValue)); } AutoscalerContext asCtx = AutoscalerContext.getInstance(); - AbstractMonitor monitor; + AbstractClusterMonitor monitor; - if(asCtx.monitorExist(clusterId)){ - monitor = asCtx.getMonitor(clusterId); - }else if(asCtx.lbMonitorExist(clusterId)){ - monitor = asCtx.getLBMonitor(clusterId); - }else{ + if(asCtx.clusterMonitorExist(clusterId)){ + monitor = asCtx.getClusterMonitor(clusterId); + } else { if(log.isDebugEnabled()){ log.debug(String.format("A cluster monitor is not found in autoscaler context [cluster] %s", clusterId)); } return; } - if(null != monitor){ - NetworkPartitionContext networkPartitionContext = monitor.getNetworkPartitionCtxt(networkPartitionId); - if(null != networkPartitionContext){ - networkPartitionContext.setLoadAverageSecondDerivative(floatValue); - } else { - if(log.isDebugEnabled()) { - log.debug(String.format("Network partition context is not available for :" + - " [network partition] %s", networkPartitionId)); + + if(monitor.getClusterType() == ClusterType.VMServiceCluster + || monitor.getClusterType() == ClusterType.VMLbCluster){ + + if(null != monitor){ + NetworkPartitionContext networkPartitionContext = ((VMClusterMonitor) monitor).getNetworkPartitionCtxt(networkPartitionId); + if(null != networkPartitionContext){ + networkPartitionContext.setLoadAverageSecondDerivative(floatValue); + } else { + if(log.isDebugEnabled()) { + log.debug(String.format("Network partition context is not available for :" + + " [network partition] %s", networkPartitionId)); + } } } + } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { + KubernetesClusterContext kubernetesClusterContext = ((DockerServiceClusterMonitor) monitor).getKubernetesClusterCtxt(); + if (null != kubernetesClusterContext) { + kubernetesClusterContext.setLoadAverageSecondDerivative(floatValue); + } else { + if(log.isDebugEnabled()) { + log.debug(String.format("Kubernetes cluster context is not available for :" + + " [cluster] %s", clusterId)); + } + } } } @@ -502,30 +595,42 @@ protected void onEvent(org.apache.stratos.messaging.event.Event event) { clusterId, networkPartitionId, floatValue)); } AutoscalerContext asCtx = AutoscalerContext.getInstance(); - AbstractMonitor monitor; + AbstractClusterMonitor monitor; - if(asCtx.monitorExist(clusterId)){ - monitor = asCtx.getMonitor(clusterId); - }else if(asCtx.lbMonitorExist(clusterId)){ - monitor = asCtx.getLBMonitor(clusterId); - }else{ + if(asCtx.clusterMonitorExist(clusterId)){ + monitor = asCtx.getClusterMonitor(clusterId); + } else { if(log.isDebugEnabled()){ log.debug(String.format("A cluster monitor is not found in autoscaler context [cluster] %s", clusterId)); } return; } - if(null != monitor){ - NetworkPartitionContext networkPartitionContext = monitor.getNetworkPartitionCtxt(networkPartitionId); - if(null != networkPartitionContext){ - networkPartitionContext.setMemoryConsumptionSecondDerivative(floatValue); - } else { - if(log.isDebugEnabled()) { - log.debug(String.format("Network partition context is not available for :" + - " [network partition] %s", networkPartitionId)); + + if(monitor.getClusterType() == ClusterType.VMServiceCluster + || monitor.getClusterType() == ClusterType.VMLbCluster){ + + if(null != monitor){ + NetworkPartitionContext networkPartitionContext = ((VMClusterMonitor) monitor).getNetworkPartitionCtxt(networkPartitionId); + if(null != networkPartitionContext){ + networkPartitionContext.setMemoryConsumptionSecondDerivative(floatValue); + } else { + if(log.isDebugEnabled()) { + log.debug(String.format("Network partition context is not available for :" + + " [network partition] %s", networkPartitionId)); + } } } + } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { + KubernetesClusterContext kubernetesClusterContext = ((DockerServiceClusterMonitor) monitor).getKubernetesClusterCtxt(); + if (null != kubernetesClusterContext) { + kubernetesClusterContext.setMemoryConsumptionSecondDerivative(floatValue); + } else { + if(log.isDebugEnabled()) { + log.debug(String.format("Kubernetes cluster context is not available for :" + + " [cluster] %s", clusterId)); + } + } } - } }); @@ -542,31 +647,43 @@ protected void onEvent(org.apache.stratos.messaging.event.Event event) { clusterId, networkPartitionId, floatValue)); } AutoscalerContext asCtx = AutoscalerContext.getInstance(); - AbstractMonitor monitor; + AbstractClusterMonitor monitor; - if(asCtx.monitorExist(clusterId)){ - monitor = asCtx.getMonitor(clusterId); - }else if(asCtx.lbMonitorExist(clusterId)){ - monitor = asCtx.getLBMonitor(clusterId); - }else{ + if(asCtx.clusterMonitorExist(clusterId)){ + monitor = asCtx.getClusterMonitor(clusterId); + } else { if(log.isDebugEnabled()){ log.debug(String.format("A cluster monitor is not found in autoscaler context [cluster] %s", clusterId)); } return; } - if(null != monitor){ - NetworkPartitionContext networkPartitionContext = monitor.getNetworkPartitionCtxt(networkPartitionId); - if(null != networkPartitionContext){ - networkPartitionContext.setRequestsInFlightSecondDerivative(floatValue); - } else { - if(log.isDebugEnabled()) { - log.debug(String.format("Network partition context is not available for :" + - " [network partition] %s", networkPartitionId)); + + if(monitor.getClusterType() == ClusterType.VMServiceCluster + || monitor.getClusterType() == ClusterType.VMLbCluster){ + + if(null != monitor){ + NetworkPartitionContext networkPartitionContext = ((VMClusterMonitor) monitor).getNetworkPartitionCtxt(networkPartitionId); + if(null != networkPartitionContext){ + networkPartitionContext.setRequestsInFlightSecondDerivative(floatValue); + } else { + if(log.isDebugEnabled()) { + log.debug(String.format("Network partition context is not available for :" + + " [network partition] %s", networkPartitionId)); + } } } + } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { + KubernetesClusterContext kubernetesClusterContext = ((DockerServiceClusterMonitor) monitor).getKubernetesClusterCtxt(); + if (null != kubernetesClusterContext) { + kubernetesClusterContext.setRequestsInFlightSecondDerivative(floatValue); + } else { + if(log.isDebugEnabled()) { + log.debug(String.format("Kubernetes cluster context is not available for :" + + " [cluster] %s", clusterId)); + } + } } } - }); } @@ -584,38 +701,61 @@ private LoadAverage findLoadAverage(String memberId) { String clusterId = member.getClusterId(); AutoscalerContext asCtx = AutoscalerContext.getInstance(); - AbstractMonitor monitor; + AbstractClusterMonitor monitor; - if(asCtx.monitorExist(clusterId)){ - monitor = asCtx.getMonitor(clusterId); - }else if(asCtx.lbMonitorExist(clusterId)){ - monitor = asCtx.getLBMonitor(clusterId); - }else{ + if(asCtx.clusterMonitorExist(clusterId)){ + monitor = asCtx.getClusterMonitor(clusterId); + } else { if(log.isDebugEnabled()){ log.debug(String.format("A cluster monitor is not found in autoscaler context [cluster] %s", clusterId)); } return null; } - String networkPartitionId = findNetworkPartitionId(memberId); - MemberStatsContext memberStatsContext = monitor.getNetworkPartitionCtxt(networkPartitionId) - .getPartitionCtxt(member.getPartitionId()) - .getMemberStatsContext(memberId); - if(null == memberStatsContext){ - if(log.isDebugEnabled()) { - log.debug(String.format("Member context is not available for : [member] %s", memberId)); + + if(monitor.getClusterType() == ClusterType.VMServiceCluster + || monitor.getClusterType() == ClusterType.VMLbCluster){ + + String networkPartitionId = findNetworkPartitionId(memberId); + MemberStatsContext memberStatsContext = ((VMClusterMonitor) monitor).getNetworkPartitionCtxt(networkPartitionId) + .getPartitionCtxt(member.getPartitionId()) + .getMemberStatsContext(memberId); + if(null == memberStatsContext){ + if(log.isDebugEnabled()) { + log.debug(String.format("Member context is not available for : [member] %s", memberId)); + } + return null; } - return null; - } - else if(!member.isActive()){ - if(log.isDebugEnabled()){ - log.debug(String.format("Member activated event has not received for the member %s. Therefore ignoring" + - " the health stat", memberId)); + else if(!member.isActive()){ + if(log.isDebugEnabled()){ + log.debug(String.format("Member activated event has not received for the member %s. Therefore ignoring" + + " the health stat", memberId)); + } + return null; } - return null; + + LoadAverage loadAverage = memberStatsContext.getLoadAverage(); + return loadAverage; + } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { + MemberStatsContext memberStatsContext = ((ContainerClusterMonitor) monitor).getKubernetesClusterCtxt().getMemberStatsContext(memberId); + if(null == memberStatsContext){ + if(log.isDebugEnabled()) { + log.debug(String.format("Member context is not available for : [member] %s", memberId)); + } + return null; + } + else if(!member.isActive()){ + if(log.isDebugEnabled()){ + log.debug(String.format("Member activated event has not received for the member %s. Therefore ignoring" + + " the health stat", memberId)); + } + return null; + } + + LoadAverage loadAverage = memberStatsContext.getLoadAverage(); + return loadAverage; } - LoadAverage loadAverage = memberStatsContext.getLoadAverage(); - return loadAverage; + return null; } private MemoryConsumption findMemoryConsumption(String memberId) { @@ -628,37 +768,61 @@ private MemoryConsumption findMemoryConsumption(String memberId) { } return null; } - AbstractMonitor monitor = AutoscalerContext.getInstance().getMonitor(member.getClusterId()); - if(null == monitor){ + + AutoscalerContext asCtx = AutoscalerContext.getInstance(); + AbstractClusterMonitor monitor; - monitor = AutoscalerContext.getInstance().getLBMonitor(member.getClusterId()); - if(null == monitor){ - if(log.isDebugEnabled()) { - log.debug(String.format("Cluster monitor is not available for : [member] %s", memberId)); - } + if(asCtx.clusterMonitorExist(member.getClusterId())){ + monitor = asCtx.getClusterMonitor(member.getClusterId()); + } else { + if(log.isDebugEnabled()){ + log.debug(String.format("A cluster monitor is not found in autoscaler context [cluster] %s", member.getClusterId())); } return null; } - String networkPartitionId = findNetworkPartitionId(memberId); - MemberStatsContext memberStatsContext = monitor.getNetworkPartitionCtxt(networkPartitionId) - .getPartitionCtxt(member.getPartitionId()) - .getMemberStatsContext(memberId); - if(null == memberStatsContext){ - if(log.isDebugEnabled()) { - log.debug(String.format("Member context is not available for : [member] %s", memberId)); + if(monitor.getClusterType() == ClusterType.VMServiceCluster + || monitor.getClusterType() == ClusterType.VMLbCluster){ + + String networkPartitionId = findNetworkPartitionId(memberId); + MemberStatsContext memberStatsContext = ((VMClusterMonitor) monitor).getNetworkPartitionCtxt(networkPartitionId) + .getPartitionCtxt(member.getPartitionId()) + .getMemberStatsContext(memberId); + if(null == memberStatsContext){ + if(log.isDebugEnabled()) { + log.debug(String.format("Member context is not available for : [member] %s", memberId)); + } + return null; + }else if(!member.isActive()){ + if(log.isDebugEnabled()){ + log.debug(String.format("Member activated event has not received for the member %s. Therefore ignoring" + + " the health stat", memberId)); + } + return null; } - return null; - }else if(!member.isActive()){ - if(log.isDebugEnabled()){ - log.debug(String.format("Member activated event has not received for the member %s. Therefore ignoring" + - " the health stat", memberId)); + MemoryConsumption memoryConsumption = memberStatsContext.getMemoryConsumption(); + + return memoryConsumption; + } else if (monitor.getClusterType() == ClusterType.DockerServiceCluster) { + MemberStatsContext memberStatsContext = ((ContainerClusterMonitor) monitor).getKubernetesClusterCtxt().getMemberStatsContext(memberId); + if(null == memberStatsContext){ + if(log.isDebugEnabled()) { + log.debug(String.format("Member context is not available for : [member] %s", memberId)); + } + return null; + }else if(!member.isActive()){ + if(log.isDebugEnabled()){ + log.debug(String.format("Member activated event has not received for the member %s. Therefore ignoring" + + " the health stat", memberId)); + } + return null; } - return null; - } - MemoryConsumption memoryConsumption = memberStatsContext.getMemoryConsumption(); + MemoryConsumption memoryConsumption = memberStatsContext.getMemoryConsumption(); - return memoryConsumption; + return memoryConsumption; + } + + return null; } private String findNetworkPartitionId(String memberId) { @@ -692,63 +856,100 @@ private Member findMember(String memberId) { private void handleMemberFaultEvent(String clusterId, String memberId) { try { AutoscalerContext asCtx = AutoscalerContext.getInstance(); - AbstractMonitor monitor; + AbstractClusterMonitor monitor; - if(asCtx.monitorExist(clusterId)){ - monitor = asCtx.getMonitor(clusterId); - }else if(asCtx.lbMonitorExist(clusterId)){ - monitor = asCtx.getLBMonitor(clusterId); - }else{ + if(asCtx.clusterMonitorExist(clusterId)){ + monitor = asCtx.getClusterMonitor(clusterId); + } else { if(log.isDebugEnabled()){ log.debug(String.format("A cluster monitor is not found in autoscaler context [cluster] %s", clusterId)); } return; - } - - NetworkPartitionContext nwPartitionCtxt; - try{ - TopologyManager.acquireReadLock(); - Member member = findMember(memberId); + } + + if(monitor.getClusterType() == ClusterType.VMServiceCluster + || monitor.getClusterType() == ClusterType.VMLbCluster){ - if(null == member){ - return; - } - if(!member.isActive()){ + NetworkPartitionContext nwPartitionCtxt; + try{ + TopologyManager.acquireReadLock(); + Member member = findMember(memberId); + + if(null == member){ + return; + } + if(!member.isActive()){ + if(log.isDebugEnabled()){ + log.debug(String.format("Member activated event has not received for the member %s. Therefore ignoring" + + " the member fault health stat", memberId)); + } + return; + } + + nwPartitionCtxt = ((VMClusterMonitor) monitor).getNetworkPartitionCtxt(member); + + }finally{ + TopologyManager.releaseReadLock(); + } + // start a new member in the same Partition + String partitionId = ((VMClusterMonitor) monitor).getPartitionOfMember(memberId); + PartitionContext partitionCtxt = nwPartitionCtxt.getPartitionCtxt(partitionId); + + if(!partitionCtxt.activeMemberExist(memberId)){ if(log.isDebugEnabled()){ - log.debug(String.format("Member activated event has not received for the member %s. Therefore ignoring" + - " the member fault health stat", memberId)); + log.debug(String.format("Could not find the active member in partition context, [member] %s ", memberId)); } return; } - - nwPartitionCtxt = monitor.getNetworkPartitionCtxt(member); - - }finally{ - TopologyManager.releaseReadLock(); - } - // start a new member in the same Partition - String partitionId = monitor.getPartitionOfMember(memberId); - PartitionContext partitionCtxt = nwPartitionCtxt.getPartitionCtxt(partitionId); - - if(!partitionCtxt.activeMemberExist(memberId)){ - if(log.isDebugEnabled()){ - log.debug(String.format("Could not find the active member in partition context, [member] %s ", memberId)); + // terminate the faulty member + CloudControllerClient ccClient = CloudControllerClient.getInstance(); + ccClient.terminate(memberId); + + // remove from active member list + partitionCtxt.removeActiveMemberById(memberId); + + if (log.isInfoEnabled()) { + log.info(String.format("Faulty member is terminated and removed from the active members list: [member] %s [partition] %s [cluster] %s ", + memberId, partitionId, clusterId)); + } + } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { + try{ + TopologyManager.acquireReadLock(); + Member member = findMember(memberId); + + if(null == member){ + return; + } + if(!member.isActive()){ + if(log.isDebugEnabled()){ + log.debug(String.format("Member activated event has not received for the member %s. Therefore ignoring" + + " the member fault health stat", memberId)); + } + return; + } + }finally{ + TopologyManager.releaseReadLock(); } - return; - } - // terminate the faulty member - CloudControllerClient ccClient = CloudControllerClient.getInstance(); - ccClient.terminate(memberId); - - // remove from active member list - partitionCtxt.removeActiveMemberById(memberId); - - if (log.isInfoEnabled()) { - log.info(String.format("Faulty member is terminated and removed from the active members list: [member] %s [partition] %s [cluster] %s ", - memberId, partitionId, clusterId)); + + KubernetesClusterContext kubernetesClusterContext = ((ContainerClusterMonitor) monitor).getKubernetesClusterCtxt(); + if (!kubernetesClusterContext.activeMemberExist(memberId)) { + if(log.isDebugEnabled()){ + log.debug(String.format("Could not find the active member in partition context, [member] %s ", memberId)); + } + return; + } + //terminate the faulty member + CloudControllerClient.getInstance().terminateContainer(memberId); + //remove from active member list + kubernetesClusterContext.removeActiveMemberById(memberId); + + if (log.isInfoEnabled()) { + log.info(String.format("Faulty member is terminated and removed from the active members list: [member] %s [kub cluster] %s [cluster] %s ", + memberId, kubernetesClusterContext.getKubernetesClusterID(), clusterId)); + } + } - - + } catch (TerminationException e) { log.error(e); } diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/message/receiver/topology/AutoscalerTopologyEventReceiver.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/message/receiver/topology/AutoscalerTopologyEventReceiver.java index cf37885aae..6525eba913 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/message/receiver/topology/AutoscalerTopologyEventReceiver.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/message/receiver/topology/AutoscalerTopologyEventReceiver.java @@ -19,34 +19,54 @@ package org.apache.stratos.autoscaler.message.receiver.topology; +import java.util.List; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.stratos.autoscaler.*; +import org.apache.stratos.autoscaler.AutoscalerContext; +import org.apache.stratos.autoscaler.KubernetesClusterContext; +import org.apache.stratos.autoscaler.MemberStatsContext; +import org.apache.stratos.autoscaler.NetworkPartitionContext; +import org.apache.stratos.autoscaler.NetworkPartitionLbHolder; +import org.apache.stratos.autoscaler.PartitionContext; import org.apache.stratos.autoscaler.client.cloud.controller.CloudControllerClient; import org.apache.stratos.autoscaler.deployment.policy.DeploymentPolicy; import org.apache.stratos.autoscaler.exception.PartitionValidationException; import org.apache.stratos.autoscaler.exception.PolicyValidationException; import org.apache.stratos.autoscaler.exception.TerminationException; -import org.apache.stratos.autoscaler.monitor.AbstractMonitor; -import org.apache.stratos.autoscaler.monitor.ClusterMonitor; -import org.apache.stratos.autoscaler.monitor.KubernetesClusterMonitor; -import org.apache.stratos.autoscaler.monitor.LbClusterMonitor; +import org.apache.stratos.autoscaler.monitor.AbstractClusterMonitor; +import org.apache.stratos.autoscaler.monitor.ClusterMonitorFactory; +import org.apache.stratos.autoscaler.monitor.ContainerClusterMonitor; +import org.apache.stratos.autoscaler.monitor.VMClusterMonitor; import org.apache.stratos.autoscaler.partition.PartitionManager; import org.apache.stratos.autoscaler.policy.PolicyManager; import org.apache.stratos.autoscaler.rule.AutoscalerRuleEvaluator; -import org.apache.stratos.autoscaler.util.AutoscalerUtil; +import org.apache.stratos.common.enums.ClusterType; import org.apache.stratos.messaging.domain.topology.Cluster; import org.apache.stratos.messaging.domain.topology.Service; import org.apache.stratos.messaging.event.Event; -import org.apache.stratos.messaging.event.topology.*; -import org.apache.stratos.messaging.listener.topology.*; +import org.apache.stratos.messaging.event.topology.ClusterCreatedEvent; +import org.apache.stratos.messaging.event.topology.ClusterMaintenanceModeEvent; +import org.apache.stratos.messaging.event.topology.ClusterRemovedEvent; +import org.apache.stratos.messaging.event.topology.MemberActivatedEvent; +import org.apache.stratos.messaging.event.topology.MemberMaintenanceModeEvent; +import org.apache.stratos.messaging.event.topology.MemberReadyToShutdownEvent; +import org.apache.stratos.messaging.event.topology.MemberTerminatedEvent; +import org.apache.stratos.messaging.listener.topology.ClusterCreatedEventListener; +import org.apache.stratos.messaging.listener.topology.ClusterMaintenanceModeEventListener; +import org.apache.stratos.messaging.listener.topology.ClusterRemovedEventListener; +import org.apache.stratos.messaging.listener.topology.CompleteTopologyEventListener; +import org.apache.stratos.messaging.listener.topology.MemberActivatedEventListener; +import org.apache.stratos.messaging.listener.topology.MemberMaintenanceListener; +import org.apache.stratos.messaging.listener.topology.MemberReadyToShutdownEventListener; +import org.apache.stratos.messaging.listener.topology.MemberStartedEventListener; +import org.apache.stratos.messaging.listener.topology.MemberTerminatedEventListener; +import org.apache.stratos.messaging.listener.topology.ServiceRemovedEventListener; import org.apache.stratos.messaging.message.receiver.topology.TopologyEventReceiver; import org.apache.stratos.messaging.message.receiver.topology.TopologyManager; import org.drools.runtime.StatefulKnowledgeSession; import org.drools.runtime.rule.FactHandle; -import java.util.List; - /** * Autoscaler topology receiver. */ @@ -116,42 +136,60 @@ protected void onEvent(Event event) { try { MemberReadyToShutdownEvent memberReadyToShutdownEvent = (MemberReadyToShutdownEvent)event; AutoscalerContext asCtx = AutoscalerContext.getInstance(); - AbstractMonitor monitor; + AbstractClusterMonitor monitor; String clusterId = memberReadyToShutdownEvent.getClusterId(); String memberId = memberReadyToShutdownEvent.getMemberId(); - if(asCtx.monitorExist(clusterId)){ - monitor = asCtx.getMonitor(clusterId); - }else if(asCtx.lbMonitorExist(clusterId)){ - monitor = asCtx.getLBMonitor(clusterId); - }else{ + if(asCtx.clusterMonitorExist(clusterId)) { + monitor = asCtx.getClusterMonitor(clusterId); + } else { if(log.isDebugEnabled()){ log.debug(String.format("A cluster monitor is not found in autoscaler context [cluster] %s", clusterId)); } return; } + + TopologyManager.acquireReadLock(); + + if(monitor.getClusterType() == ClusterType.VMServiceCluster + || monitor.getClusterType() == ClusterType.VMLbCluster) { + + NetworkPartitionContext nwPartitionCtxt; + nwPartitionCtxt = ((VMClusterMonitor) monitor).getNetworkPartitionCtxt(memberReadyToShutdownEvent.getNetworkPartitionId()); - NetworkPartitionContext nwPartitionCtxt; - nwPartitionCtxt = monitor.getNetworkPartitionCtxt(memberReadyToShutdownEvent.getNetworkPartitionId()); - - // start a new member in the same Partition - String partitionId = monitor.getPartitionOfMember(memberId); - PartitionContext partitionCtxt = nwPartitionCtxt.getPartitionCtxt(partitionId); - + // start a new member in the same Partition + String partitionId = ((VMClusterMonitor) monitor).getPartitionOfMember(memberId); + PartitionContext partitionCtxt = nwPartitionCtxt.getPartitionCtxt(partitionId); - // terminate the shutdown ready member - CloudControllerClient ccClient = CloudControllerClient.getInstance(); - ccClient.terminate(memberId); - // remove from active member list - partitionCtxt.removeActiveMemberById(memberId); + // terminate the shutdown ready member + CloudControllerClient ccClient = CloudControllerClient.getInstance(); + ccClient.terminate(memberId); - if (log.isInfoEnabled()) { - log.info(String.format("Member is terminated and removed from the active members list: [member] %s [partition] %s [cluster] %s ", - memberId, partitionId, clusterId)); + // remove from active member list + partitionCtxt.removeActiveMemberById(memberId); + + if (log.isInfoEnabled()) { + log.info(String.format("Member is terminated and removed from the active members list: [member] %s [partition] %s [cluster] %s ", + memberId, partitionId, clusterId)); + } + } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { + KubernetesClusterContext kubernetesClusterContext = ((ContainerClusterMonitor) monitor).getKubernetesClusterCtxt(); + //terminate the shutdown ready container + CloudControllerClient.getInstance().terminateContainer(memberId); + //remove from active member list + kubernetesClusterContext.removeActiveMemberById(memberId); + + if (log.isInfoEnabled()) { + log.info(String.format("Member is terminated and removed from the active members list: [member] %s [kub cluster] %s [cluster] %s ", + memberId, kubernetesClusterContext.getKubernetesClusterID(), clusterId)); + } } + } catch (TerminationException e) { log.error(e); + } finally { + TopologyManager.releaseReadLock(); } } @@ -185,12 +223,8 @@ protected void onEvent(Event event) { TopologyManager.acquireReadLock(); Service service = TopologyManager.getTopology().getService(e.getServiceName()); Cluster cluster = service.getCluster(e.getClusterId()); - if(AutoscalerContext.getInstance().kubernetesClusterMonitorExist(cluster.getClusterId())) { - AutoscalerContext.getInstance().getKubernetesClusterMonitor(e.getClusterId()).setStatus(e.getStatus()); - } else if(AutoscalerContext.getInstance().monitorExist((cluster.getClusterId()))) { - AutoscalerContext.getInstance().getMonitor(e.getClusterId()).setStatus(e.getStatus()); - } else if (AutoscalerContext.getInstance().lbMonitorExist((cluster.getClusterId()))) { - AutoscalerContext.getInstance().getLBMonitor(e.getClusterId()).setStatus(e.getStatus()); + if(AutoscalerContext.getInstance().clusterMonitorExist(cluster.getClusterId())) { + AutoscalerContext.getInstance().getClusterMonitor(e.getClusterId()).setStatus(e.getStatus()); } else { log.error("cluster monitor not exists for the cluster: " + cluster.toString()); } @@ -213,19 +247,9 @@ protected void onEvent(Event event) { String clusterId = e.getClusterId(); String deploymentPolicy = e.getDeploymentPolicy(); - AbstractMonitor monitor = null; - KubernetesClusterMonitor kubernetesClusterMonitor = null; + AbstractClusterMonitor monitor = null; - if (e.isKubernetesCluster()) { - - kubernetesClusterMonitor = - AutoscalerContext.getInstance().removeKubernetesClusterMonitor(clusterId); - if(kubernetesClusterMonitor != null) { - // destroy drools sessions - log.info(String.format("Kubernetes cluster monitor has been removed successfully: [cluster] %s ", - clusterId)); - } - } else if (e.isLbCluster()) { + if (e.isLbCluster()) { DeploymentPolicy depPolicy = PolicyManager.getInstance().getDeploymentPolicy(deploymentPolicy); if (depPolicy != null) { List lbHolders = PartitionManager.getInstance() @@ -248,13 +272,9 @@ protected void onEvent(Event event) { } } - monitor = AutoscalerContext.getInstance() - .removeLbMonitor(clusterId); - - } else { - monitor = AutoscalerContext.getInstance() - .removeMonitor(clusterId); } + + monitor = AutoscalerContext.getInstance().removeClusterMonitor(clusterId); // runTerminateAllRule(monitor); if (monitor != null) { @@ -289,43 +309,73 @@ protected void onEvent(Event event) { String networkPartitionId = e.getNetworkPartitionId(); String clusterId = e.getClusterId(); String partitionId = e.getPartitionId(); - AbstractMonitor monitor; + String memberId = e.getMemberId(); + AbstractClusterMonitor monitor; + + AutoscalerContext asCtx = AutoscalerContext.getInstance(); - if (AutoscalerContext.getInstance().monitorExist(clusterId)) { - monitor = AutoscalerContext.getInstance().getMonitor(clusterId); + if(asCtx.clusterMonitorExist(clusterId)) { + monitor = asCtx.getClusterMonitor(clusterId); } else { - //This is LB member - monitor = AutoscalerContext.getInstance().getLBMonitor(clusterId); + if(log.isDebugEnabled()){ + log.debug(String.format("A cluster monitor is not found in autoscaler context [cluster] %s", clusterId)); + } + return; } + + if(monitor.getClusterType() == ClusterType.VMServiceCluster + || monitor.getClusterType() == ClusterType.VMLbCluster) { + + NetworkPartitionContext networkPartitionContext = ((VMClusterMonitor) monitor).getNetworkPartitionCtxt(networkPartitionId); - NetworkPartitionContext networkPartitionContext = monitor.getNetworkPartitionCtxt(networkPartitionId); + PartitionContext partitionContext = networkPartitionContext.getPartitionCtxt(partitionId); + partitionContext.removeMemberStatsContext(memberId); - PartitionContext partitionContext = networkPartitionContext.getPartitionCtxt(partitionId); - String memberId = e.getMemberId(); - partitionContext.removeMemberStatsContext(memberId); + if (partitionContext.removeTerminationPendingMember(memberId)) { + if (log.isDebugEnabled()) { + log.debug(String.format("Member is removed from termination pending members list: [member] %s", memberId)); + } + } else if (partitionContext.removePendingMember(memberId)) { + if (log.isDebugEnabled()) { + log.debug(String.format("Member is removed from pending members list: [member] %s", memberId)); + } + } else if (partitionContext.removeActiveMemberById(memberId)) { + log.warn(String.format("Member is in the wrong list and it is removed from active members list", memberId)); + } else if (partitionContext.removeObsoleteMember(memberId)){ + log.warn(String.format("Member's obsolated timeout has been expired and it is removed from obsolated members list", memberId)); + } else { + log.warn(String.format("Member is not available in any of the list active, pending and termination pending", memberId)); + } - if (partitionContext.removeTerminationPendingMember(memberId)) { - if (log.isDebugEnabled()) { - log.debug(String.format("Member is removed from termination pending members list: [member] %s", memberId)); + if (log.isInfoEnabled()) { + log.info(String.format("Member stat context has been removed successfully: [member] %s", memberId)); } - } else if (partitionContext.removePendingMember(memberId)) { - if (log.isDebugEnabled()) { - log.debug(String.format("Member is removed from pending members list: [member] %s", memberId)); + } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { + + KubernetesClusterContext kubernetesClusterContext = ((ContainerClusterMonitor) monitor).getKubernetesClusterCtxt(); + kubernetesClusterContext.removeMemberStatsContext(memberId); + + if (kubernetesClusterContext.removeTerminationPendingMember(memberId)) { + if (log.isDebugEnabled()) { + log.debug(String.format("Member is removed from termination pending members list: [member] %s", memberId)); + } + } else if (kubernetesClusterContext.removePendingMember(memberId)) { + if (log.isDebugEnabled()) { + log.debug(String.format("Member is removed from pending members list: [member] %s", memberId)); + } + } else if (kubernetesClusterContext.removeActiveMemberById(memberId)) { + log.warn(String.format("Member is in the wrong list and it is removed from active members list", memberId)); + } else if (kubernetesClusterContext.removeObsoleteMember(memberId)){ + log.warn(String.format("Member's obsolated timeout has been expired and it is removed from obsolated members list", memberId)); + } else { + log.warn(String.format("Member is not available in any of the list active, pending and termination pending", memberId)); } - } else if (partitionContext.removeActiveMemberById(memberId)) { - log.warn(String.format("Member is in the wrong list and it is removed from active members list", memberId)); - } else if (partitionContext.removeObsoleteMember(memberId)){ - log.warn(String.format("Member's obsolated timeout has been expired and it is removed from obsolated members list", memberId)); - } else { - log.warn(String.format("Member is not available in any of the list active, pending and termination pending", memberId)); - } - if (log.isInfoEnabled()) { - log.info(String.format("Member stat context has been removed successfully: [member] %s", memberId)); + if (log.isInfoEnabled()) { + log.info(String.format("Member stat context has been removed successfully: [member] %s", memberId)); + } } -// partitionContext.decrementCurrentActiveMemberCount(1); - - + } catch (Exception e) { log.error("Error processing event", e); } finally { @@ -347,24 +397,37 @@ protected void onEvent(Event event) { String partitionId = e.getPartitionId(); String networkPartitionId = e.getNetworkPartitionId(); - PartitionContext partitionContext; String clusterId = e.getClusterId(); - AbstractMonitor monitor; - - if (AutoscalerContext.getInstance().monitorExist(clusterId)) { - monitor = AutoscalerContext.getInstance().getMonitor(clusterId); - partitionContext = monitor.getNetworkPartitionCtxt(networkPartitionId).getPartitionCtxt(partitionId); + AbstractClusterMonitor monitor; + + AutoscalerContext asCtx = AutoscalerContext.getInstance(); + if(asCtx.clusterMonitorExist(clusterId)) { + monitor = asCtx.getClusterMonitor(clusterId); } else { - monitor = AutoscalerContext.getInstance().getLBMonitor(clusterId); - partitionContext = monitor.getNetworkPartitionCtxt(networkPartitionId).getPartitionCtxt(partitionId); - } - partitionContext.addMemberStatsContext(new MemberStatsContext(memberId)); - if (log.isInfoEnabled()) { - log.info(String.format("Member stat context has been added successfully: [member] %s", memberId)); + if(log.isDebugEnabled()){ + log.debug(String.format("A cluster monitor is not found in autoscaler context [cluster] %s", clusterId)); + } + return; } -// partitionContext.incrementCurrentActiveMemberCount(1); - partitionContext.movePendingMemberToActiveMembers(memberId); - + + if (monitor.getClusterType() == ClusterType.VMServiceCluster) { + PartitionContext partitionContext; + partitionContext = ((VMClusterMonitor) monitor).getNetworkPartitionCtxt(networkPartitionId).getPartitionCtxt(partitionId); + partitionContext.addMemberStatsContext(new MemberStatsContext(memberId)); + if (log.isInfoEnabled()) { + log.info(String.format("Member stat context has been added successfully: [member] %s", memberId)); + } + partitionContext.movePendingMemberToActiveMembers(memberId); + } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { + KubernetesClusterContext kubernetesClusterContext; + kubernetesClusterContext = ((ContainerClusterMonitor) monitor).getKubernetesClusterCtxt(); + kubernetesClusterContext.addMemberStatsContext(new MemberStatsContext(memberId)); + if (log.isInfoEnabled()) { + log.info(String.format("Member stat context has been added successfully: [member] %s", memberId)); + } + kubernetesClusterContext.movePendingMemberToActiveMembers(memberId); + } + } catch (Exception e) { log.error("Error processing event", e); } finally { @@ -377,42 +440,59 @@ protected void onEvent(Event event) { @Override protected void onEvent(Event event) { try { + TopologyManager.acquireReadLock(); + MemberReadyToShutdownEvent memberReadyToShutdownEvent = (MemberReadyToShutdownEvent)event; AutoscalerContext asCtx = AutoscalerContext.getInstance(); - AbstractMonitor monitor; + AbstractClusterMonitor monitor; String clusterId = memberReadyToShutdownEvent.getClusterId(); String memberId = memberReadyToShutdownEvent.getMemberId(); - if(asCtx.monitorExist(clusterId)){ - monitor = asCtx.getMonitor(clusterId); - }else if(asCtx.lbMonitorExist(clusterId)){ - monitor = asCtx.getLBMonitor(clusterId); - }else{ + if(asCtx.clusterMonitorExist(clusterId)) { + monitor = asCtx.getClusterMonitor(clusterId); + } else { if(log.isDebugEnabled()){ log.debug(String.format("A cluster monitor is not found in autoscaler context [cluster] %s", clusterId)); } return; } - NetworkPartitionContext nwPartitionCtxt; - nwPartitionCtxt = monitor.getNetworkPartitionCtxt(memberReadyToShutdownEvent.getNetworkPartitionId()); + if(monitor.getClusterType() == ClusterType.VMServiceCluster + || monitor.getClusterType() == ClusterType.VMLbCluster) { + + NetworkPartitionContext nwPartitionCtxt; + nwPartitionCtxt = ((VMClusterMonitor) monitor).getNetworkPartitionCtxt(memberReadyToShutdownEvent.getNetworkPartitionId()); - // start a new member in the same Partition - String partitionId = monitor.getPartitionOfMember(memberId); - PartitionContext partitionCtxt = nwPartitionCtxt.getPartitionCtxt(partitionId); + // start a new member in the same Partition + String partitionId = ((VMClusterMonitor) monitor).getPartitionOfMember(memberId); + PartitionContext partitionCtxt = nwPartitionCtxt.getPartitionCtxt(partitionId); - // terminate the shutdown ready member - CloudControllerClient ccClient = CloudControllerClient.getInstance(); - ccClient.terminate(memberId); + // terminate the shutdown ready member + CloudControllerClient ccClient = CloudControllerClient.getInstance(); + ccClient.terminate(memberId); - // remove from active member list - partitionCtxt.removeActiveMemberById(memberId); + // remove from active member list + partitionCtxt.removeActiveMemberById(memberId); - if (log.isInfoEnabled()) { - log.info(String.format("Member is terminated and removed from the active members list: [member] %s [partition] %s [cluster] %s ", - memberId, partitionId, clusterId)); + if (log.isInfoEnabled()) { + log.info(String.format("Member is terminated and removed from the active members list: [member] %s [partition] %s [cluster] %s ", + memberId, partitionId, clusterId)); + } + } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { + KubernetesClusterContext kubernetesClusterContext; + kubernetesClusterContext = ((ContainerClusterMonitor) monitor).getKubernetesClusterCtxt(); + // terminate the shutdown ready member + CloudControllerClient.getInstance().terminateContainer(memberId); + // remove from active member list + kubernetesClusterContext.removeActiveMemberById(memberId); + + if (log.isInfoEnabled()) { + log.info(String.format("Member is terminated and removed from the active members list: [member] %s [kub cluster] %s [cluster] %s ", + memberId, kubernetesClusterContext.getKubernetesClusterID(), clusterId)); + } } + } catch (TerminationException e) { log.error(e); } @@ -433,22 +513,38 @@ protected void onEvent(Event event) { String partitionId = e.getPartitionId(); String networkPartitionId = e.getNetworkPartitionId(); - PartitionContext partitionContext; String clusterId = e.getClusterId(); - AbstractMonitor monitor; - - if (AutoscalerContext.getInstance().monitorExist(clusterId)) { - monitor = AutoscalerContext.getInstance().getMonitor(clusterId); - partitionContext = monitor.getNetworkPartitionCtxt(networkPartitionId).getPartitionCtxt(partitionId); + AbstractClusterMonitor monitor; + + AutoscalerContext asCtx = AutoscalerContext.getInstance(); + if (asCtx.clusterMonitorExist(clusterId)) { + monitor = AutoscalerContext.getInstance().getClusterMonitor(clusterId); } else { - monitor = AutoscalerContext.getInstance().getLBMonitor(clusterId); - partitionContext = monitor.getNetworkPartitionCtxt(networkPartitionId).getPartitionCtxt(partitionId); + if(log.isDebugEnabled()){ + log.debug(String.format("A cluster monitor is not found in autoscaler context [cluster] %s", clusterId)); + } + return; } - partitionContext.addMemberStatsContext(new MemberStatsContext(memberId)); - if (log.isDebugEnabled()) { - log.debug(String.format("Member has been moved as pending termination: [member] %s", memberId)); + + if(monitor.getClusterType() == ClusterType.VMServiceCluster + || monitor.getClusterType() == ClusterType.VMLbCluster) { + + PartitionContext partitionContext; + partitionContext = ((VMClusterMonitor) monitor).getNetworkPartitionCtxt(networkPartitionId).getPartitionCtxt(partitionId); + partitionContext.addMemberStatsContext(new MemberStatsContext(memberId)); + if (log.isDebugEnabled()) { + log.debug(String.format("Member has been moved as pending termination: [member] %s", memberId)); + } + partitionContext.moveActiveMemberToTerminationPendingMembers(memberId); + } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { + KubernetesClusterContext kubernetesClusterContext; + kubernetesClusterContext = ((ContainerClusterMonitor) monitor).getKubernetesClusterCtxt(); + kubernetesClusterContext.addMemberStatsContext(new MemberStatsContext(memberId)); + if (log.isDebugEnabled()) { + log.debug(String.format("Member has been moved as pending termination: [member] %s", memberId)); + } + kubernetesClusterContext.moveActiveMemberToTerminationPendingMembers(memberId); } - partitionContext.moveActiveMemberToTerminationPendingMembers(memberId); } catch (Exception e) { log.error("Error processing event", e); @@ -480,64 +576,15 @@ protected void onEvent(Event event) { }); } - private class LBClusterMonitorAdder implements Runnable { - private Cluster cluster; - - public LBClusterMonitorAdder(Cluster cluster) { - this.cluster = cluster; - } - - public void run() { - LbClusterMonitor monitor = null; - int retries = 5; - boolean success = false; - do { - try { - Thread.sleep(5000); - } catch (InterruptedException e1) { - } - try { - monitor = AutoscalerUtil.getLBClusterMonitor(cluster); - success = true; - - } catch (PolicyValidationException e) { - String msg = "LB Cluster monitor creation failed for cluster: " + cluster.getClusterId(); - log.debug(msg, e); - retries--; - - } catch (PartitionValidationException e) { - String msg = "LB Cluster monitor creation failed for cluster: " + cluster.getClusterId(); - log.debug(msg, e); - retries--; - } - } while (!success && retries <= 0); - - if (monitor == null) { - String msg = "LB Cluster monitor creation failed, even after retrying for 5 times, " - + "for cluster: " + cluster.getClusterId(); - log.error(msg); - throw new RuntimeException(msg); - } - - Thread th = new Thread(monitor); - th.start(); - AutoscalerContext.getInstance().addLbMonitor(monitor); - if (log.isInfoEnabled()) { - log.info(String.format("LB Cluster monitor has been added successfully: [cluster] %s", - cluster.getClusterId())); - } - } - } - private class ClusterMonitorAdder implements Runnable { private Cluster cluster; - + private String clusterMonitorType; public ClusterMonitorAdder(Cluster cluster) { this.cluster = cluster; } public void run() { - ClusterMonitor monitor = null; + AbstractClusterMonitor monitor = null; int retries = 5; boolean success = false; do { @@ -547,68 +594,23 @@ public void run() { } try { - monitor = AutoscalerUtil.getClusterMonitor(cluster); + monitor = ClusterMonitorFactory.getMonitor(cluster); success = true; - + clusterMonitorType = monitor.getClusterType().name(); } catch (PolicyValidationException e) { - String msg = "Cluster monitor creation failed for cluster: " + cluster.getClusterId(); + String msg = clusterMonitorType +" monitor creation failed for cluster: " + cluster.getClusterId(); log.debug(msg, e); retries--; } catch (PartitionValidationException e) { - String msg = "Cluster monitor creation failed for cluster: " + cluster.getClusterId(); - log.debug(msg, e); - retries--; - } - } while (!success && retries != 0); - - if (monitor == null) { - String msg = "Cluster monitor creation failed, even after retrying for 5 times, " - + "for cluster: " + cluster.getClusterId(); - log.error(msg); - throw new RuntimeException(msg); - } - - Thread th = new Thread(monitor); - th.start(); - AutoscalerContext.getInstance().addMonitor(monitor); - if (log.isInfoEnabled()) { - log.info(String.format("Cluster monitor has been added successfully: [cluster] %s", - cluster.getClusterId())); - } - } - } - - private class KubernetesClusterMonitorAdder implements Runnable { - private Cluster cluster; - - public KubernetesClusterMonitorAdder(Cluster cluster) { - this.cluster = cluster; - } - - public void run() { - KubernetesClusterMonitor monitor = null; - int retries = 5; - boolean success = false; - do { - try { - Thread.sleep(5000); - } catch (InterruptedException e1) { - } - - try { - monitor = AutoscalerUtil.getKubernetesClusterMonitor(cluster); - success = true; - - } catch (Exception e) { - String msg = "Kubernetes cluster monitor creation failed for cluster: " + cluster.getClusterId(); + String msg = clusterMonitorType +" monitor creation failed for cluster: " + cluster.getClusterId(); log.debug(msg, e); retries--; } } while (!success && retries != 0); if (monitor == null) { - String msg = "Kubernetes cluster monitor creation failed, even after retrying for 5 times, " + String msg = clusterMonitorType +" monitor creation failed, even after retrying for 5 times, " + "for cluster: " + cluster.getClusterId(); log.error(msg); throw new RuntimeException(msg); @@ -616,16 +618,16 @@ public void run() { Thread th = new Thread(monitor); th.start(); - AutoscalerContext.getInstance().addKubernetesClusterMonitor(monitor); + AutoscalerContext.getInstance().addClusterMonitor(monitor); if (log.isInfoEnabled()) { - log.info(String.format("Kubernetes cluster monitor has been added successfully: [cluster] %s", - cluster.getClusterId())); + log.info(String.format("%s monitor has been added successfully: [cluster] %s", + clusterMonitorType, cluster.getClusterId())); } } } - + @SuppressWarnings("unused") - private void runTerminateAllRule(AbstractMonitor monitor) { + private void runTerminateAllRule(VMClusterMonitor monitor) { FactHandle terminateAllFactHandle = null; @@ -648,16 +650,9 @@ public void terminate() { protected synchronized void startClusterMonitor(Cluster cluster) { Thread th = null; - if (cluster.isKubernetesCluster() - && !AutoscalerContext.getInstance().kubernetesClusterMonitorExist(cluster.getClusterId())) { - th = new Thread(new KubernetesClusterMonitorAdder(cluster)); - } else if (cluster.isLbCluster() - && !AutoscalerContext.getInstance().lbMonitorExist(cluster.getClusterId())) { - th = new Thread(new LBClusterMonitorAdder(cluster)); - } else if (!cluster.isLbCluster() && !cluster.isKubernetesCluster() - && !AutoscalerContext.getInstance().monitorExist(cluster.getClusterId())) { - th = new Thread(new ClusterMonitorAdder(cluster)); - } + if (!AutoscalerContext.getInstance().clusterMonitorExist(cluster.getClusterId())) { + th = new Thread(new ClusterMonitorAdder(cluster)); + } if (th != null) { th.start(); try { diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/AbstractClusterMonitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/AbstractClusterMonitor.java new file mode 100644 index 0000000000..00796f110e --- /dev/null +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/AbstractClusterMonitor.java @@ -0,0 +1,127 @@ +package org.apache.stratos.autoscaler.monitor; + +import org.apache.stratos.autoscaler.rule.AutoscalerRuleEvaluator; +import org.apache.stratos.common.enums.ClusterType; +import org.apache.stratos.messaging.domain.topology.ClusterStatus; +import org.drools.runtime.StatefulKnowledgeSession; +import org.drools.runtime.rule.FactHandle; + +public abstract class AbstractClusterMonitor implements Runnable{ + + private String clusterId; + private String serviceId; + private ClusterType clusterType; + private ClusterStatus status; + private int monitorInterval; + + protected FactHandle minCheckFactHandle; + protected FactHandle scaleCheckFactHandle; + private StatefulKnowledgeSession minCheckKnowledgeSession; + private StatefulKnowledgeSession scaleCheckKnowledgeSession; + private boolean isDestroyed; + + private AutoscalerRuleEvaluator autoscalerRuleEvaluator; + + protected AbstractClusterMonitor(String clusterId, String serviceId, ClusterType clusterType, + AutoscalerRuleEvaluator autoscalerRuleEvaluator) { + + super(); + this.clusterId = clusterId; + this.serviceId = serviceId; + this.clusterType = clusterType; + this.autoscalerRuleEvaluator = autoscalerRuleEvaluator; + this.scaleCheckKnowledgeSession = autoscalerRuleEvaluator.getScaleCheckStatefulSession(); + this.minCheckKnowledgeSession = autoscalerRuleEvaluator.getMinCheckStatefulSession(); + } + + protected abstract void readConfigurations(); + protected abstract void monitor(); + public abstract void destroy(); + + public String getClusterId() { + return clusterId; + } + + public void setClusterId(String clusterId) { + this.clusterId = clusterId; + } + + public void setStatus(ClusterStatus status) { + this.status = status; + } + + public ClusterType getClusterType() { + return clusterType; + } + + public ClusterStatus getStatus() { + return status; + } + + public String getServiceId() { + return serviceId; + } + + public void setServiceId(String serviceId) { + this.serviceId = serviceId; + } + + public int getMonitorInterval() { + return monitorInterval; + } + + public void setMonitorInterval(int monitorInterval) { + this.monitorInterval = monitorInterval; + } + + public FactHandle getMinCheckFactHandle() { + return minCheckFactHandle; + } + + public void setMinCheckFactHandle(FactHandle minCheckFactHandle) { + this.minCheckFactHandle = minCheckFactHandle; + } + + public FactHandle getScaleCheckFactHandle() { + return scaleCheckFactHandle; + } + + public void setScaleCheckFactHandle(FactHandle scaleCheckFactHandle) { + this.scaleCheckFactHandle = scaleCheckFactHandle; + } + + public StatefulKnowledgeSession getMinCheckKnowledgeSession() { + return minCheckKnowledgeSession; + } + + public void setMinCheckKnowledgeSession( + StatefulKnowledgeSession minCheckKnowledgeSession) { + this.minCheckKnowledgeSession = minCheckKnowledgeSession; + } + + public StatefulKnowledgeSession getScaleCheckKnowledgeSession() { + return scaleCheckKnowledgeSession; + } + + public void setScaleCheckKnowledgeSession( + StatefulKnowledgeSession scaleCheckKnowledgeSession) { + this.scaleCheckKnowledgeSession = scaleCheckKnowledgeSession; + } + + public boolean isDestroyed() { + return isDestroyed; + } + + public void setDestroyed(boolean isDestroyed) { + this.isDestroyed = isDestroyed; + } + + public AutoscalerRuleEvaluator getAutoscalerRuleEvaluator() { + return autoscalerRuleEvaluator; + } + + public void setAutoscalerRuleEvaluator( + AutoscalerRuleEvaluator autoscalerRuleEvaluator) { + this.autoscalerRuleEvaluator = autoscalerRuleEvaluator; + } +} diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/ClusterMonitorFactory.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/ClusterMonitorFactory.java new file mode 100644 index 0000000000..489078e9ec --- /dev/null +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/ClusterMonitorFactory.java @@ -0,0 +1,336 @@ +package org.apache.stratos.autoscaler.monitor; + +import java.util.Map; +import java.util.Random; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.stratos.autoscaler.KubernetesClusterContext; +import org.apache.stratos.autoscaler.MemberStatsContext; +import org.apache.stratos.autoscaler.NetworkPartitionContext; +import org.apache.stratos.autoscaler.NetworkPartitionLbHolder; +import org.apache.stratos.autoscaler.PartitionContext; +import org.apache.stratos.autoscaler.client.cloud.controller.CloudControllerClient; +import org.apache.stratos.autoscaler.deployment.policy.DeploymentPolicy; +import org.apache.stratos.autoscaler.exception.PartitionValidationException; +import org.apache.stratos.autoscaler.exception.PolicyValidationException; +import org.apache.stratos.autoscaler.partition.PartitionGroup; +import org.apache.stratos.autoscaler.partition.PartitionManager; +import org.apache.stratos.autoscaler.policy.PolicyManager; +import org.apache.stratos.autoscaler.policy.model.AutoscalePolicy; +import org.apache.stratos.cloud.controller.stub.deployment.partition.Partition; +import org.apache.stratos.cloud.controller.stub.pojo.MemberContext; +import org.apache.stratos.cloud.controller.stub.pojo.Properties; +import org.apache.stratos.cloud.controller.stub.pojo.Property; +import org.apache.stratos.common.constants.StratosConstants; +import org.apache.stratos.messaging.domain.topology.Cluster; +import org.apache.stratos.messaging.domain.topology.ClusterStatus; +import org.apache.stratos.messaging.domain.topology.Member; +import org.apache.stratos.messaging.domain.topology.MemberStatus; +import org.apache.stratos.messaging.util.Constants; + +public class ClusterMonitorFactory { + + private static final Log log = LogFactory.getLog(ClusterMonitorFactory.class); + + public static AbstractClusterMonitor getMonitor(Cluster cluster) throws PolicyValidationException, PartitionValidationException { + + AbstractClusterMonitor clusterMonitor; + if(cluster.isKubernetesCluster()){ + clusterMonitor = getDockerServiceClusterMonitor(cluster); + } else if (cluster.isLbCluster()){ + clusterMonitor = getVMLbClusterMonitor(cluster); + } else { + clusterMonitor = getVMServiceClusterMonitor(cluster); + } + + return clusterMonitor; + } + + private static VMServiceClusterMonitor getVMServiceClusterMonitor(Cluster cluster) throws PolicyValidationException, PartitionValidationException { + // FIXME fix the following code to correctly update + // AutoscalerContext context = AutoscalerContext.getInstance(); + if (null == cluster) { + return null; + } + + String autoscalePolicyName = cluster.getAutoscalePolicyName(); + String deploymentPolicyName = cluster.getDeploymentPolicyName(); + + if (log.isDebugEnabled()) { + log.debug("Deployment policy name: " + deploymentPolicyName); + log.debug("Autoscaler policy name: " + autoscalePolicyName); + } + + AutoscalePolicy policy = + PolicyManager.getInstance() + .getAutoscalePolicy(autoscalePolicyName); + DeploymentPolicy deploymentPolicy = + PolicyManager.getInstance() + .getDeploymentPolicy(deploymentPolicyName); + + if (deploymentPolicy == null) { + String msg = "Deployment Policy is null. Policy name: " + deploymentPolicyName; + log.error(msg); + throw new PolicyValidationException(msg); + } + + Partition[] allPartitions = deploymentPolicy.getAllPartitions(); + if (allPartitions == null) { + String msg = + "Deployment Policy's Partitions are null. Policy name: " + + deploymentPolicyName; + log.error(msg); + throw new PolicyValidationException(msg); + } + + CloudControllerClient.getInstance().validateDeploymentPolicy(cluster.getServiceName(), deploymentPolicy); + + VMServiceClusterMonitor clusterMonitor = + new VMServiceClusterMonitor(cluster.getClusterId(), + cluster.getServiceName(), + deploymentPolicy, policy); + clusterMonitor.setStatus(ClusterStatus.Created); + + for (PartitionGroup partitionGroup: deploymentPolicy.getPartitionGroups()){ + + NetworkPartitionContext networkPartitionContext = new NetworkPartitionContext(partitionGroup.getId(), + partitionGroup.getPartitionAlgo(), partitionGroup.getPartitions()); + + for(Partition partition: partitionGroup.getPartitions()){ + PartitionContext partitionContext = new PartitionContext(partition); + partitionContext.setServiceName(cluster.getServiceName()); + partitionContext.setProperties(cluster.getProperties()); + partitionContext.setNetworkPartitionId(partitionGroup.getId()); + + for (Member member: cluster.getMembers()){ + String memberId = member.getMemberId(); + if(member.getPartitionId().equalsIgnoreCase(partition.getId())){ + MemberContext memberContext = new MemberContext(); + memberContext.setClusterId(member.getClusterId()); + memberContext.setMemberId(memberId); + memberContext.setPartition(partition); + memberContext.setProperties(convertMemberPropsToMemberContextProps(member.getProperties())); + + if(MemberStatus.Activated.equals(member.getStatus())){ + partitionContext.addActiveMember(memberContext); +// networkPartitionContext.increaseMemberCountOfPartition(partition.getNetworkPartitionId(), 1); +// partitionContext.incrementCurrentActiveMemberCount(1); + + } else if(MemberStatus.Created.equals(member.getStatus()) || MemberStatus.Starting.equals(member.getStatus())){ + partitionContext.addPendingMember(memberContext); + +// networkPartitionContext.increaseMemberCountOfPartition(partition.getNetworkPartitionId(), 1); + } else if(MemberStatus.Suspended.equals(member.getStatus())){ +// partitionContext.addFaultyMember(memberId); + } + partitionContext.addMemberStatsContext(new MemberStatsContext(memberId)); + if(log.isInfoEnabled()){ + log.info(String.format("Member stat context has been added: [member] %s", memberId)); + } + } + + } + networkPartitionContext.addPartitionContext(partitionContext); + if(log.isInfoEnabled()){ + log.info(String.format("Partition context has been added: [partition] %s", + partitionContext.getPartitionId())); + } + } + + clusterMonitor.addNetworkPartitionCtxt(networkPartitionContext); + if(log.isInfoEnabled()){ + log.info(String.format("Network partition context has been added: [network partition] %s", + networkPartitionContext.getId())); + } + } + + + // find lb reference type + java.util.Properties props = cluster.getProperties(); + + if(props.containsKey(Constants.LOAD_BALANCER_REF)) { + String value = props.getProperty(Constants.LOAD_BALANCER_REF); + clusterMonitor.setLbReferenceType(value); + if(log.isDebugEnabled()) { + log.debug("Set the lb reference type: "+value); + } + } + + // set hasPrimary property + // hasPrimary is true if there are primary members available in that cluster + clusterMonitor.setHasPrimary(Boolean.parseBoolean(cluster.getProperties().getProperty(Constants.IS_PRIMARY))); + + log.info("Cluster monitor created: "+clusterMonitor.toString()); + return clusterMonitor; + } + + private static Properties convertMemberPropsToMemberContextProps( + java.util.Properties properties) { + Properties props = new Properties(); + for (Map.Entry e : properties.entrySet() ) { + Property prop = new Property(); + prop.setName((String)e.getKey()); + prop.setValue((String)e.getValue()); + props.addProperties(prop); + } + return props; + } + + + private static VMLbClusterMonitor getVMLbClusterMonitor(Cluster cluster) throws PolicyValidationException, PartitionValidationException { + // FIXME fix the following code to correctly update + // AutoscalerContext context = AutoscalerContext.getInstance(); + if (null == cluster) { + return null; + } + + String autoscalePolicyName = cluster.getAutoscalePolicyName(); + String deploymentPolicyName = cluster.getDeploymentPolicyName(); + + if (log.isDebugEnabled()) { + log.debug("Deployment policy name: " + deploymentPolicyName); + log.debug("Autoscaler policy name: " + autoscalePolicyName); + } + + AutoscalePolicy policy = + PolicyManager.getInstance() + .getAutoscalePolicy(autoscalePolicyName); + DeploymentPolicy deploymentPolicy = + PolicyManager.getInstance() + .getDeploymentPolicy(deploymentPolicyName); + + if (deploymentPolicy == null) { + String msg = "Deployment Policy is null. Policy name: " + deploymentPolicyName; + log.error(msg); + throw new PolicyValidationException(msg); + } + + String clusterId = cluster.getClusterId(); + VMLbClusterMonitor clusterMonitor = + new VMLbClusterMonitor(clusterId, + cluster.getServiceName(), + deploymentPolicy, policy); + clusterMonitor.setStatus(ClusterStatus.Created); + // partition group = network partition context + for (PartitionGroup partitionGroup : deploymentPolicy.getPartitionGroups()) { + + NetworkPartitionLbHolder networkPartitionLbHolder = + PartitionManager.getInstance() + .getNetworkPartitionLbHolder(partitionGroup.getId()); +// PartitionManager.getInstance() +// .getNetworkPartitionLbHolder(partitionGroup.getId()); + // FIXME pick a random partition + Partition partition = + partitionGroup.getPartitions()[new Random().nextInt(partitionGroup.getPartitions().length)]; + PartitionContext partitionContext = new PartitionContext(partition); + partitionContext.setServiceName(cluster.getServiceName()); + partitionContext.setProperties(cluster.getProperties()); + partitionContext.setNetworkPartitionId(partitionGroup.getId()); + partitionContext.setMinimumMemberCount(1);//Here it hard codes the minimum value as one for LB cartridge partitions + + NetworkPartitionContext networkPartitionContext = new NetworkPartitionContext(partitionGroup.getId(), + partitionGroup.getPartitionAlgo(), partitionGroup.getPartitions()) ; + for (Member member : cluster.getMembers()) { + String memberId = member.getMemberId(); + if (member.getNetworkPartitionId().equalsIgnoreCase(networkPartitionContext.getId())) { + MemberContext memberContext = new MemberContext(); + memberContext.setClusterId(member.getClusterId()); + memberContext.setMemberId(memberId); + memberContext.setPartition(partition); + + if (MemberStatus.Activated.equals(member.getStatus())) { + partitionContext.addActiveMember(memberContext); +// networkPartitionContext.increaseMemberCountOfPartition(partition.getNetworkPartitionId(), 1); +// partitionContext.incrementCurrentActiveMemberCount(1); + } else if (MemberStatus.Created.equals(member.getStatus()) || + MemberStatus.Starting.equals(member.getStatus())) { + partitionContext.addPendingMember(memberContext); +// networkPartitionContext.increaseMemberCountOfPartition(partition.getNetworkPartitionId(), 1); + } else if (MemberStatus.Suspended.equals(member.getStatus())) { +// partitionContext.addFaultyMember(memberId); + } + + partitionContext.addMemberStatsContext(new MemberStatsContext(memberId)); + if(log.isInfoEnabled()){ + log.info(String.format("Member stat context has been added: [member] %s", memberId)); + } + } + + } + networkPartitionContext.addPartitionContext(partitionContext); + + // populate lb cluster id in network partition context. + java.util.Properties props = cluster.getProperties(); + + // get service type of load balanced cluster + String loadBalancedServiceType = props.getProperty(Constants.LOAD_BALANCED_SERVICE_TYPE); + + if(props.containsKey(Constants.LOAD_BALANCER_REF)) { + String value = props.getProperty(Constants.LOAD_BALANCER_REF); + + if (value.equals(org.apache.stratos.messaging.util.Constants.DEFAULT_LOAD_BALANCER)) { + networkPartitionLbHolder.setDefaultLbClusterId(clusterId); + + } else if (value.equals(org.apache.stratos.messaging.util.Constants.SERVICE_AWARE_LOAD_BALANCER)) { + String serviceName = cluster.getServiceName(); + // TODO: check if this is correct + networkPartitionLbHolder.addServiceLB(serviceName, clusterId); + + if (loadBalancedServiceType != null && !loadBalancedServiceType.isEmpty()) { + networkPartitionLbHolder.addServiceLB(loadBalancedServiceType, clusterId); + if (log.isDebugEnabled()) { + log.debug("Added cluster id " + clusterId + " as the LB cluster id for service type " + loadBalancedServiceType); + } + } + } + } + + clusterMonitor.addNetworkPartitionCtxt(networkPartitionContext); + } + + log.info("LB Cluster monitor created: "+clusterMonitor.toString()); + return clusterMonitor; + } + + private static DockerServiceClusterMonitor getDockerServiceClusterMonitor(Cluster cluster) { + + if (null == cluster) { + return null; + } + + String autoscalePolicyName = cluster.getAutoscalePolicyName(); + if (log.isDebugEnabled()) { + log.debug("Autoscaler policy name: " + autoscalePolicyName); + } + + AutoscalePolicy policy = PolicyManager.getInstance().getAutoscalePolicy(autoscalePolicyName); + java.util.Properties props = cluster.getProperties(); + String kubernetesHostClusterID = props.getProperty(StratosConstants.KUBERNETES_CLUSTER_ID); + KubernetesClusterContext kubernetesClusterCtxt = new KubernetesClusterContext(kubernetesHostClusterID); + + DockerServiceClusterMonitor dockerClusterMonitor = new DockerServiceClusterMonitor( + kubernetesClusterCtxt, + cluster.getClusterId(), + cluster.getServiceName(), + policy); + + dockerClusterMonitor.setStatus(ClusterStatus.Created); + + // find lb reference type + if(props.containsKey(Constants.LOAD_BALANCER_REF)) { + String value = props.getProperty(Constants.LOAD_BALANCER_REF); + dockerClusterMonitor.setLbReferenceType(value); + if(log.isDebugEnabled()) { + log.debug("Set the lb reference type: "+value); + } + } + +// // set hasPrimary property +// // hasPrimary is true if there are primary members available in that cluster +// dockerClusterMonitor.setHasPrimary(Boolean.parseBoolean(props.getProperty(Constants.IS_PRIMARY))); + + log.info("Docker cluster monitor created: "+ dockerClusterMonitor.toString()); + return dockerClusterMonitor; + } +} diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/ContainerClusterMonitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/ContainerClusterMonitor.java new file mode 100644 index 0000000000..f9b9047f77 --- /dev/null +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/ContainerClusterMonitor.java @@ -0,0 +1,38 @@ +package org.apache.stratos.autoscaler.monitor; + +import org.apache.stratos.autoscaler.KubernetesClusterContext; +import org.apache.stratos.autoscaler.policy.model.AutoscalePolicy; +import org.apache.stratos.autoscaler.rule.AutoscalerRuleEvaluator; +import org.apache.stratos.common.enums.ClusterType; + +public abstract class ContainerClusterMonitor extends AbstractClusterMonitor { + + private KubernetesClusterContext kubernetesClusterCtxt; + protected AutoscalePolicy autoscalePolicy; + + protected ContainerClusterMonitor(String clusterId, String serviceId, ClusterType clusterType, + KubernetesClusterContext kubernetesClusterContext, + AutoscalerRuleEvaluator autoscalerRuleEvaluator, AutoscalePolicy autoscalePolicy){ + + super(clusterId, serviceId, clusterType, autoscalerRuleEvaluator); + this.kubernetesClusterCtxt = kubernetesClusterContext; + this.autoscalePolicy = autoscalePolicy; + } + + public KubernetesClusterContext getKubernetesClusterCtxt() { + return kubernetesClusterCtxt; + } + + public void setKubernetesClusterCtxt( + KubernetesClusterContext kubernetesClusterCtxt) { + this.kubernetesClusterCtxt = kubernetesClusterCtxt; + } + + public AutoscalePolicy getAutoscalePolicy() { + return autoscalePolicy; + } + + public void setAutoscalePolicy(AutoscalePolicy autoscalePolicy) { + this.autoscalePolicy = autoscalePolicy; + } +} diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/DockerServiceClusterMonitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/DockerServiceClusterMonitor.java new file mode 100644 index 0000000000..ca39b6ab42 --- /dev/null +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/DockerServiceClusterMonitor.java @@ -0,0 +1,156 @@ +package org.apache.stratos.autoscaler.monitor; + +import java.util.Properties; + +import org.apache.commons.configuration.XMLConfiguration; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.stratos.autoscaler.KubernetesClusterContext; +import org.apache.stratos.autoscaler.client.cloud.controller.CloudControllerClient; +import org.apache.stratos.autoscaler.policy.model.AutoscalePolicy; +import org.apache.stratos.autoscaler.rule.AutoscalerRuleEvaluator; +import org.apache.stratos.autoscaler.util.AutoScalerConstants; +import org.apache.stratos.autoscaler.util.ConfUtil; +import org.apache.stratos.cloud.controller.stub.pojo.MemberContext; +import org.apache.stratos.common.constants.StratosConstants; +import org.apache.stratos.common.enums.ClusterType; +import org.apache.stratos.messaging.domain.topology.ClusterStatus; +import org.apache.stratos.messaging.message.receiver.topology.TopologyManager; + +public final class DockerServiceClusterMonitor extends ContainerClusterMonitor{ + + private static final Log log = LogFactory.getLog(DockerServiceClusterMonitor.class); + + private String lbReferenceType; + private int numberOfReplicasInServiceCluster = 0; + int retryInterval = 60000; + + public DockerServiceClusterMonitor(KubernetesClusterContext kubernetesClusterCtxt, + String serviceClusterID, String serviceId, AutoscalePolicy autoscalePolicy) { + super(serviceClusterID, serviceId, ClusterType.DockerServiceCluster, kubernetesClusterCtxt, + new AutoscalerRuleEvaluator(), autoscalePolicy); + readConfigurations(); + } + + @Override + public void run() { + try { + // TODO make this configurable, + // this is the delay the min check of normal cluster monitor to wait + // until LB monitor is added + Thread.sleep(60000); + } catch (InterruptedException ignore) { + } + + while (!isDestroyed()) { + if (log.isDebugEnabled()) { + log.debug("Kubernetes cluster monitor is running.. " + this.toString()); + } + try { + if (!ClusterStatus.In_Maintenance.equals(getStatus())) { + monitor(); + } else { + if (log.isDebugEnabled()) { + log.debug("Kubernetes cluster monitor is suspended as the cluster is in " + + ClusterStatus.In_Maintenance + " mode......"); + } + } + } catch (Exception e) { + log.error("Kubernetes cluster monitor: Monitor failed." + this.toString(), + e); + } + try { + Thread.sleep(getMonitorInterval()); + } catch (InterruptedException ignore) { + } + } + } + + @Override + protected void monitor() { + + // is container created successfully? + boolean success = false; + String kubernetesClusterId = getKubernetesClusterCtxt().getKubernetesClusterID(); + + try { + TopologyManager.acquireReadLock(); + Properties props = TopologyManager.getTopology().getService(getServiceId()).getCluster(getClusterId()).getProperties(); + int minReplicas = Integer.parseInt(props.getProperty(StratosConstants.KUBERNETES_MIN_REPLICAS)); + + int nonTerminatedMembers = getKubernetesClusterCtxt().getActiveMembers().size() + getKubernetesClusterCtxt().getPendingMembers().size(); + + if (nonTerminatedMembers == 0) { + + while (success) { + try { + + MemberContext memberContext = CloudControllerClient.getInstance().createContainer(kubernetesClusterId, getClusterId()); + if(null != memberContext) { + getKubernetesClusterCtxt().addPendingMember(memberContext); + success = true; + numberOfReplicasInServiceCluster = minReplicas; + if(log.isDebugEnabled()){ + log.debug(String.format("Pending member added, [member] %s [kub cluster] %s", + memberContext.getMemberId(), getKubernetesClusterCtxt().getKubernetesClusterID())); + } + } else { + if (log.isDebugEnabled()) { + log.debug("Returned member context is null, did not add to pending members"); + } + } + } catch (Throwable e) { + if (log.isDebugEnabled()) { + String message = "Cannot create a container, will retry in "+(retryInterval/1000)+"s"; + log.debug(message, e); + } + } + + try { + Thread.sleep(retryInterval); + } catch (InterruptedException e1) { + } + } + } + } finally { + TopologyManager.releaseReadLock(); + } + } + + @Override + public void destroy() { + getMinCheckKnowledgeSession().dispose(); + getScaleCheckKnowledgeSession().dispose(); + setDestroyed(true); + if(log.isDebugEnabled()) { + log.debug("DockerClusterMonitor Drools session has been disposed. "+this.toString()); + } + } + + @Override + protected void readConfigurations () { + // same as VM cluster monitor interval + XMLConfiguration conf = ConfUtil.getInstance(null).getConfiguration(); + int monitorInterval = conf.getInt(AutoScalerConstants.AUTOSCALER_MONITOR_INTERVAL, 90000); + setMonitorInterval(monitorInterval); + if (log.isDebugEnabled()) { + log.debug("Kubernetes Cluster Monitor task interval: " + getMonitorInterval()); + } + } + + @Override + public String toString() { + return "DockerClusterMonitor " + + "[ kubernetesHostClusterId=" + getKubernetesClusterCtxt().getKubernetesClusterID() + + ", clusterId=" + getClusterId() + + ", serviceId=" + getServiceId() + "]"; + } + + public String getLbReferenceType() { + return lbReferenceType; + } + + public void setLbReferenceType(String lbReferenceType) { + this.lbReferenceType = lbReferenceType; + } +} \ No newline at end of file diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/KubernetesClusterMonitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/KubernetesClusterMonitor.java deleted file mode 100644 index ac5be7b446..0000000000 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/KubernetesClusterMonitor.java +++ /dev/null @@ -1,186 +0,0 @@ -package org.apache.stratos.autoscaler.monitor; - -import java.util.Properties; - -import org.apache.commons.configuration.XMLConfiguration; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.stratos.autoscaler.KubernetesClusterContext; -import org.apache.stratos.autoscaler.client.cloud.controller.CloudControllerClient; -import org.apache.stratos.autoscaler.policy.model.AutoscalePolicy; -import org.apache.stratos.autoscaler.util.AutoScalerConstants; -import org.apache.stratos.autoscaler.util.ConfUtil; -import org.apache.stratos.common.constants.StratosConstants; -import org.apache.stratos.messaging.domain.topology.ClusterStatus; -import org.apache.stratos.messaging.message.receiver.topology.TopologyManager; - -public class KubernetesClusterMonitor implements Runnable{ - - private static final Log log = LogFactory.getLog(KubernetesClusterMonitor.class); - - protected KubernetesClusterContext kubernetesClusterCtxt; - protected String clusterId; - protected String serviceId; - protected AutoscalePolicy autoscalePolicy; - protected int monitorInterval; - protected boolean isDestroyed; - private ClusterStatus status; - private String lbReferenceType; - private boolean hasPrimary; - private int numberOfReplicasInServiceCluster = 0; - // is container created successfully? - boolean success = false; - int retryInterval = 60000; - - public KubernetesClusterMonitor(KubernetesClusterContext kubernetesClusterCtxt, String serviceClusterID, String serviceId, - AutoscalePolicy autoscalePolicy) { - this.clusterId = serviceClusterID; - this.serviceId = serviceId; - this.kubernetesClusterCtxt = kubernetesClusterCtxt; - this.autoscalePolicy = autoscalePolicy; - readConfigurations(); - } - - private void readConfigurations () { - // same as VM cluster monitor interval - XMLConfiguration conf = ConfUtil.getInstance(null).getConfiguration(); - monitorInterval = conf.getInt(AutoScalerConstants.AUTOSCALER_MONITOR_INTERVAL, 90000); - if (log.isDebugEnabled()) { - log.debug("Kubernetes Cluster Monitor task interval: " + getMonitorInterval()); - } - } - - @Override - public void run() { - try { - // TODO make this configurable, - // this is the delay the min check of normal cluster monitor to wait - // until LB monitor is added - Thread.sleep(60000); - } catch (InterruptedException ignore) { - } - - while (!isDestroyed()) { - if (log.isDebugEnabled()) { - log.debug("Kubernetes cluster monitor is running.. " + this.toString()); - } - try { - if (!ClusterStatus.In_Maintenance.equals(status)) { - monitor(); - } else { - if (log.isDebugEnabled()) { - log.debug("Kubernetes cluster monitor is suspended as the cluster is in " - + ClusterStatus.In_Maintenance + " mode......"); - } - } - } catch (Exception e) { - log.error("Kubernetes cluster monitor: Monitor failed." + this.toString(), - e); - } - try { - Thread.sleep(monitorInterval); - } catch (InterruptedException ignore) { - } - } - } - - private void monitor() { - - String kubernetesClusterId = this.kubernetesClusterCtxt.getKubernetesClusterID(); - - try { - TopologyManager.acquireReadLock(); - Properties props = TopologyManager.getTopology().getService(serviceId).getCluster(clusterId).getProperties(); - int minReplicas = Integer.parseInt(props.getProperty(StratosConstants.KUBERNETES_MIN_REPLICAS)); - - if (this.numberOfReplicasInServiceCluster < minReplicas) { - - while (!success) { - try { - CloudControllerClient.getInstance().createContainer(kubernetesClusterId, clusterId); - success = true; - numberOfReplicasInServiceCluster = minReplicas; - } catch (Throwable e) { - String message = "Cannot create a container, will retry in "+(retryInterval/1000)+"s"; - log.debug(message, e); - } - - try { - Thread.sleep(retryInterval); - } catch (InterruptedException e1) { - } - } - } - } finally { - TopologyManager.releaseReadLock(); - } - } - - @Override - public String toString() { - return "KubernetesClusterMonitor " - + "[ kubernetesHostClusterId=" + this.kubernetesClusterCtxt.getKubernetesClusterID() - + ", clusterId=" + clusterId - + ", serviceId=" + serviceId + "]"; - } - - public String getServiceId() { - return serviceId; - } - - public void setServiceId(String serviceId) { - this.serviceId = serviceId; - } - - public String getClusterId() { - return clusterId; - } - - public void setClusterId(String clusterId) { - this.clusterId = clusterId; - } - - public AutoscalePolicy getAutoscalePolicy() { - return autoscalePolicy; - } - - public void setAutoscalePolicy(AutoscalePolicy autoscalePolicy) { - this.autoscalePolicy = autoscalePolicy; - } - - public int getMonitorInterval() { - return monitorInterval; - } - - public ClusterStatus getStatus() { - return status; - } - - public void setStatus(ClusterStatus status) { - this.status = status; - } - - public boolean isDestroyed() { - return isDestroyed; - } - - public void setDestroyed(boolean isDestroyed) { - this.isDestroyed = isDestroyed; - } - - public String getLbReferenceType() { - return lbReferenceType; - } - - public void setLbReferenceType(String lbReferenceType) { - this.lbReferenceType = lbReferenceType; - } - - public boolean isHasPrimary() { - return hasPrimary; - } - - public void setHasPrimary(boolean hasPrimary) { - this.hasPrimary = hasPrimary; - } -} \ No newline at end of file diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/AbstractMonitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/VMClusterMonitor.java similarity index 59% rename from components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/AbstractMonitor.java rename to components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/VMClusterMonitor.java index c1441bbdc2..ffd6713667 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/AbstractMonitor.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/VMClusterMonitor.java @@ -20,21 +20,17 @@ import java.util.Map; -import org.apache.commons.configuration.XMLConfiguration; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.stratos.autoscaler.NetworkPartitionContext; import org.apache.stratos.autoscaler.deployment.policy.DeploymentPolicy; import org.apache.stratos.autoscaler.policy.model.AutoscalePolicy; import org.apache.stratos.autoscaler.rule.AutoscalerRuleEvaluator; -import org.apache.stratos.autoscaler.util.AutoScalerConstants; -import org.apache.stratos.autoscaler.util.ConfUtil; +import org.apache.stratos.common.enums.ClusterType; import org.apache.stratos.messaging.domain.topology.Cluster; import org.apache.stratos.messaging.domain.topology.Member; import org.apache.stratos.messaging.domain.topology.Service; import org.apache.stratos.messaging.message.receiver.topology.TopologyManager; -import org.drools.runtime.StatefulKnowledgeSession; -import org.drools.runtime.rule.FactHandle; /** * Is responsible for monitoring a service cluster. This runs periodically @@ -42,50 +38,24 @@ * rules engine. * */ - abstract public class AbstractMonitor implements Runnable{ + abstract public class VMClusterMonitor extends AbstractClusterMonitor{ - private static final Log log = LogFactory.getLog(AbstractMonitor.class); + private static final Log log = LogFactory.getLog(VMClusterMonitor.class); // Map protected Map networkPartitionCtxts; protected DeploymentPolicy deploymentPolicy; protected AutoscalePolicy autoscalePolicy; - - protected FactHandle minCheckFactHandle; - protected FactHandle scaleCheckFactHandle; - - protected StatefulKnowledgeSession minCheckKnowledgeSession; - protected StatefulKnowledgeSession scaleCheckKnowledgeSession; - protected boolean isDestroyed; - - protected String clusterId; - protected String serviceId; - - protected AutoscalerRuleEvaluator autoscalerRuleEvaluator; - - // time intereval between two runs of the Monitor. Default is 90000ms. - protected int monitorInterval; - - public AbstractMonitor() { - readConfigurations(); - } - - private void readConfigurations () { - - XMLConfiguration conf = ConfUtil.getInstance(null).getConfiguration(); - monitorInterval = conf.getInt(AutoScalerConstants.AUTOSCALER_MONITOR_INTERVAL, 90000); - if (log.isDebugEnabled()) { - log.debug("Cluster Monitor task interval: " + getMonitorInterval()); - } + protected VMClusterMonitor(String clusterId, String serviceId, ClusterType clusterType, + AutoscalerRuleEvaluator autoscalerRuleEvaluator, + DeploymentPolicy deploymentPolicy, AutoscalePolicy autoscalePolicy, + Map networkPartitionCtxts) { + super(clusterId, serviceId, clusterType, autoscalerRuleEvaluator); + this.deploymentPolicy = deploymentPolicy; + this.autoscalePolicy = autoscalePolicy; + this.networkPartitionCtxts = networkPartitionCtxts; } - @Override - public void run() { - // TODO Auto-generated method stub - - } - - public NetworkPartitionContext getNetworkPartitionCtxt(Member member) { log.info("***** getNetworkPartitionCtxt " + member.getNetworkPartitionId()); String networkPartitionId = member.getNetworkPartitionId(); @@ -108,31 +78,6 @@ public String getPartitionOfMember(String memberId){ return null; } - public void destroy() { - minCheckKnowledgeSession.dispose(); - scaleCheckKnowledgeSession.dispose(); - setDestroyed(true); - if(log.isDebugEnabled()) { - log.debug("Cluster Monitor Drools session has been disposed. "+this.toString()); - } - } - - public boolean isDestroyed() { - return isDestroyed; - } - - public void setDestroyed(boolean isDestroyed) { - this.isDestroyed = isDestroyed; - } - - public String getServiceId() { - return serviceId; - } - - public void setServiceId(String serviceId) { - this.serviceId = serviceId; - } - public DeploymentPolicy getDeploymentPolicy() { return deploymentPolicy; } @@ -148,14 +93,6 @@ public AutoscalePolicy getAutoscalePolicy() { public void setAutoscalePolicy(AutoscalePolicy autoscalePolicy) { this.autoscalePolicy = autoscalePolicy; } - - public String getClusterId() { - return clusterId; - } - - public void setClusterId(String clusterId) { - this.clusterId = clusterId; - } public Map getNetworkPartitionCtxts() { return networkPartitionCtxts; @@ -180,24 +117,4 @@ public void addNetworkPartitionCtxt(NetworkPartitionContext ctxt) { public NetworkPartitionContext getPartitionCtxt(String id) { return this.networkPartitionCtxts.get(id); } - - public StatefulKnowledgeSession getMinCheckKnowledgeSession() { - return minCheckKnowledgeSession; - } - - public void setMinCheckKnowledgeSession(StatefulKnowledgeSession minCheckKnowledgeSession) { - this.minCheckKnowledgeSession = minCheckKnowledgeSession; - } - - public FactHandle getMinCheckFactHandle() { - return minCheckFactHandle; - } - - public void setMinCheckFactHandle(FactHandle minCheckFactHandle) { - this.minCheckFactHandle = minCheckFactHandle; - } - - public int getMonitorInterval() { - return monitorInterval; - } } \ No newline at end of file diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/LbClusterMonitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/VMLbClusterMonitor.java similarity index 65% rename from components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/LbClusterMonitor.java rename to components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/VMLbClusterMonitor.java index 8d6c40bda7..666fa756dc 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/LbClusterMonitor.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/VMLbClusterMonitor.java @@ -20,6 +20,7 @@ import java.util.concurrent.ConcurrentHashMap; +import org.apache.commons.configuration.XMLConfiguration; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.stratos.autoscaler.NetworkPartitionContext; @@ -27,6 +28,9 @@ import org.apache.stratos.autoscaler.deployment.policy.DeploymentPolicy; import org.apache.stratos.autoscaler.policy.model.AutoscalePolicy; import org.apache.stratos.autoscaler.rule.AutoscalerRuleEvaluator; +import org.apache.stratos.autoscaler.util.AutoScalerConstants; +import org.apache.stratos.autoscaler.util.ConfUtil; +import org.apache.stratos.common.enums.ClusterType; import org.apache.stratos.messaging.domain.topology.ClusterStatus; /** @@ -35,23 +39,16 @@ * rules engine. * */ -public class LbClusterMonitor extends AbstractMonitor{ +public class VMLbClusterMonitor extends VMClusterMonitor{ - private static final Log log = LogFactory.getLog(LbClusterMonitor.class); - private ClusterStatus status; + private static final Log log = LogFactory.getLog(VMLbClusterMonitor.class); - public LbClusterMonitor(String clusterId, String serviceId, DeploymentPolicy deploymentPolicy, + public VMLbClusterMonitor(String clusterId, String serviceId, DeploymentPolicy deploymentPolicy, AutoscalePolicy autoscalePolicy) { - this.clusterId = clusterId; - this.serviceId = serviceId; - - this.autoscalerRuleEvaluator = new AutoscalerRuleEvaluator(); - this.scaleCheckKnowledgeSession = autoscalerRuleEvaluator.getScaleCheckStatefulSession(); - this.minCheckKnowledgeSession = autoscalerRuleEvaluator.getMinCheckStatefulSession(); - - this.deploymentPolicy = deploymentPolicy; - this.deploymentPolicy = deploymentPolicy; - networkPartitionCtxts = new ConcurrentHashMap(); + super(clusterId, serviceId, ClusterType.VMLbCluster, new AutoscalerRuleEvaluator(), + deploymentPolicy, autoscalePolicy, + new ConcurrentHashMap()); + readConfigurations(); } @Override @@ -62,7 +59,7 @@ public void run() { log.debug("Cluster monitor is running.. "+this.toString()); } try { - if( !ClusterStatus.In_Maintenance.equals(status)) { + if( !ClusterStatus.In_Maintenance.equals(getStatus())) { monitor(); } else { if (log.isDebugEnabled()) { @@ -74,13 +71,14 @@ public void run() { log.error("Cluster monitor: Monitor failed. "+this.toString(), e); } try { - Thread.sleep(monitorInterval); + Thread.sleep(getMonitorInterval()); } catch (InterruptedException ignore) { } } } - private void monitor() { + @Override + protected void monitor() { // TODO make this concurrent for (NetworkPartitionContext networkPartitionContext : networkPartitionCtxts.values()) { @@ -89,8 +87,8 @@ private void monitor() { .values()) { if (partitionContext != null) { - minCheckKnowledgeSession.setGlobal("clusterId", clusterId); - minCheckKnowledgeSession.setGlobal("isPrimary", false); + getMinCheckKnowledgeSession().setGlobal("clusterId", getClusterId()); + getMinCheckKnowledgeSession().setGlobal("isPrimary", false); if (log.isDebugEnabled()) { log.debug(String.format("Running minimum check for partition %s ", @@ -98,7 +96,7 @@ private void monitor() { } minCheckFactHandle = - AutoscalerRuleEvaluator.evaluateMinCheck(minCheckKnowledgeSession, + AutoscalerRuleEvaluator.evaluateMinCheck(getMinCheckKnowledgeSession(), minCheckFactHandle, partitionContext); // start only in the first partition context @@ -108,19 +106,30 @@ private void monitor() { } } - } - - @Override - public String toString() { - return "LbClusterMonitor [clusterId=" + clusterId + ", serviceId=" + serviceId + "]"; + } + + @Override + public void destroy() { + getMinCheckKnowledgeSession().dispose(); + getMinCheckKnowledgeSession().dispose(); + setDestroyed(true); + if(log.isDebugEnabled()) { + log.debug("LbClusterMonitor Drools session has been disposed. "+this.toString()); + } } - - - public ClusterStatus getStatus() { - return status; + + @Override + protected void readConfigurations () { + XMLConfiguration conf = ConfUtil.getInstance(null).getConfiguration(); + int monitorInterval = conf.getInt(AutoScalerConstants.AUTOSCALER_MONITOR_INTERVAL, 90000); + setMonitorInterval(monitorInterval); + if (log.isDebugEnabled()) { + log.debug("LbClusterMonitor task interval: " + getMonitorInterval()); + } } - public void setStatus(ClusterStatus status) { - this.status = status; + @Override + public String toString() { + return "LbClusterMonitor [clusterId=" + getClusterId() + ", serviceId=" + getServiceId() + "]"; } } diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/ClusterMonitor.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/VMServiceClusterMonitor.java similarity index 72% rename from components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/ClusterMonitor.java rename to components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/VMServiceClusterMonitor.java index 5bb478e94f..ed988592c1 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/ClusterMonitor.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/monitor/VMServiceClusterMonitor.java @@ -18,6 +18,11 @@ */ package org.apache.stratos.autoscaler.monitor; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.commons.configuration.XMLConfiguration; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.stratos.autoscaler.NetworkPartitionContext; @@ -25,44 +30,34 @@ import org.apache.stratos.autoscaler.deployment.policy.DeploymentPolicy; import org.apache.stratos.autoscaler.policy.model.AutoscalePolicy; import org.apache.stratos.autoscaler.rule.AutoscalerRuleEvaluator; +import org.apache.stratos.autoscaler.util.AutoScalerConstants; +import org.apache.stratos.autoscaler.util.ConfUtil; import org.apache.stratos.cloud.controller.stub.pojo.MemberContext; import org.apache.stratos.cloud.controller.stub.pojo.Properties; import org.apache.stratos.cloud.controller.stub.pojo.Property; +import org.apache.stratos.common.enums.ClusterType; import org.apache.stratos.messaging.domain.topology.ClusterStatus; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; - /** * Is responsible for monitoring a service cluster. This runs periodically * and perform minimum instance check and scaling check using the underlying * rules engine. * */ -public class ClusterMonitor extends AbstractMonitor { +public class VMServiceClusterMonitor extends VMClusterMonitor { - private static final Log log = LogFactory.getLog(ClusterMonitor.class); + private static final Log log = LogFactory.getLog(VMServiceClusterMonitor.class); private String lbReferenceType; private boolean hasPrimary; - private ClusterStatus status; - public ClusterMonitor(String clusterId, String serviceId, DeploymentPolicy deploymentPolicy, + public VMServiceClusterMonitor(String clusterId, String serviceId, DeploymentPolicy deploymentPolicy, AutoscalePolicy autoscalePolicy) { - this.clusterId = clusterId; - this.serviceId = serviceId; - - this.autoscalerRuleEvaluator = new AutoscalerRuleEvaluator(); - this.scaleCheckKnowledgeSession = autoscalerRuleEvaluator.getScaleCheckStatefulSession(); - this.minCheckKnowledgeSession = autoscalerRuleEvaluator.getMinCheckStatefulSession(); - - this.deploymentPolicy = deploymentPolicy; - this.autoscalePolicy = autoscalePolicy; - networkPartitionCtxts = new ConcurrentHashMap(); + super(clusterId, serviceId, ClusterType.VMServiceCluster, new AutoscalerRuleEvaluator(), + deploymentPolicy, autoscalePolicy, + new ConcurrentHashMap()); + readConfigurations(); } - - @Override public void run() { @@ -78,7 +73,7 @@ public void run() { log.debug("Cluster monitor is running.. " + this.toString()); } try { - if(!ClusterStatus.In_Maintenance.equals(status)) { + if(!ClusterStatus.In_Maintenance.equals(getStatus())) { monitor(); } else { if (log.isDebugEnabled()) { @@ -90,32 +85,14 @@ public void run() { log.error("Cluster monitor: Monitor failed." + this.toString(), e); } try { - Thread.sleep(monitorInterval); + Thread.sleep(getMonitorInterval()); } catch (InterruptedException ignore) { } } } - private boolean isPrimaryMember(MemberContext memberContext){ - Properties props = memberContext.getProperties(); - if (log.isDebugEnabled()) { - log.debug(" Properties [" + props + "] "); - } - if (props != null && props.getProperties() != null) { - for (Property prop : props.getProperties()) { - if (prop.getName().equals("PRIMARY")) { - if (Boolean.parseBoolean(prop.getValue())) { - log.debug("Adding member id [" + memberContext.getMemberId() + "] " + - "member instance id [" + memberContext.getInstanceId() + "] as a primary member"); - return true; - } - } - } - } - return false; - } - - private void monitor() { + @Override + protected void monitor() { //TODO make this concurrent for (NetworkPartitionContext networkPartitionContext : networkPartitionCtxts.values()) { @@ -139,16 +116,16 @@ private void monitor() { } } primaryMemberListInNetworkPartition.addAll(primaryMemberListInPartition); - minCheckKnowledgeSession.setGlobal("clusterId", clusterId); - minCheckKnowledgeSession.setGlobal("lbRef", lbReferenceType); - minCheckKnowledgeSession.setGlobal("isPrimary", hasPrimary); - minCheckKnowledgeSession.setGlobal("primaryMemberCount", primaryMemberListInPartition.size()); + getMinCheckKnowledgeSession().setGlobal("clusterId", getClusterId()); + getMinCheckKnowledgeSession().setGlobal("lbRef", lbReferenceType); + getMinCheckKnowledgeSession().setGlobal("isPrimary", hasPrimary); + getMinCheckKnowledgeSession().setGlobal("primaryMemberCount", primaryMemberListInPartition.size()); if (log.isDebugEnabled()) { log.debug(String.format("Running minimum check for partition %s ", partitionContext.getPartitionId())); } - minCheckFactHandle = AutoscalerRuleEvaluator.evaluateMinCheck(minCheckKnowledgeSession + minCheckFactHandle = AutoscalerRuleEvaluator.evaluateMinCheck(getMinCheckKnowledgeSession() , minCheckFactHandle, partitionContext); } @@ -161,22 +138,22 @@ private void monitor() { + " flag of loadAverageReset" + loadAverageReset); } if (rifReset || memoryConsumptionReset || loadAverageReset) { - scaleCheckKnowledgeSession.setGlobal("clusterId", clusterId); + getScaleCheckKnowledgeSession().setGlobal("clusterId", getClusterId()); //scaleCheckKnowledgeSession.setGlobal("deploymentPolicy", deploymentPolicy); - scaleCheckKnowledgeSession.setGlobal("autoscalePolicy", autoscalePolicy); - scaleCheckKnowledgeSession.setGlobal("rifReset", rifReset); - scaleCheckKnowledgeSession.setGlobal("mcReset", memoryConsumptionReset); - scaleCheckKnowledgeSession.setGlobal("laReset", loadAverageReset); - scaleCheckKnowledgeSession.setGlobal("lbRef", lbReferenceType); - scaleCheckKnowledgeSession.setGlobal("isPrimary", false); - scaleCheckKnowledgeSession.setGlobal("primaryMembers", primaryMemberListInNetworkPartition); + getScaleCheckKnowledgeSession().setGlobal("autoscalePolicy", autoscalePolicy); + getScaleCheckKnowledgeSession().setGlobal("rifReset", rifReset); + getScaleCheckKnowledgeSession().setGlobal("mcReset", memoryConsumptionReset); + getScaleCheckKnowledgeSession().setGlobal("laReset", loadAverageReset); + getScaleCheckKnowledgeSession().setGlobal("lbRef", lbReferenceType); + getScaleCheckKnowledgeSession().setGlobal("isPrimary", false); + getScaleCheckKnowledgeSession().setGlobal("primaryMembers", primaryMemberListInNetworkPartition); if (log.isDebugEnabled()) { log.debug(String.format("Running scale check for network partition %s ", networkPartitionContext.getId())); log.debug(" Primary members : " + primaryMemberListInNetworkPartition); } - scaleCheckFactHandle = AutoscalerRuleEvaluator.evaluateScaleCheck(scaleCheckKnowledgeSession + scaleCheckFactHandle = AutoscalerRuleEvaluator.evaluateScaleCheck(getScaleCheckKnowledgeSession() , scaleCheckFactHandle, networkPartitionContext); networkPartitionContext.setRifReset(false); @@ -188,10 +165,49 @@ private void monitor() { } } } + + private boolean isPrimaryMember(MemberContext memberContext){ + Properties props = memberContext.getProperties(); + if (log.isDebugEnabled()) { + log.debug(" Properties [" + props + "] "); + } + if (props != null && props.getProperties() != null) { + for (Property prop : props.getProperties()) { + if (prop.getName().equals("PRIMARY")) { + if (Boolean.parseBoolean(prop.getValue())) { + log.debug("Adding member id [" + memberContext.getMemberId() + "] " + + "member instance id [" + memberContext.getInstanceId() + "] as a primary member"); + return true; + } + } + } + } + return false; + } + + @Override + protected void readConfigurations () { + XMLConfiguration conf = ConfUtil.getInstance(null).getConfiguration(); + int monitorInterval = conf.getInt(AutoScalerConstants.AUTOSCALER_MONITOR_INTERVAL, 90000); + setMonitorInterval(monitorInterval); + if (log.isDebugEnabled()) { + log.debug("Cluster Monitor task interval: " + getMonitorInterval()); + } + } + + @Override + public void destroy() { + getMinCheckKnowledgeSession().dispose(); + getScaleCheckKnowledgeSession().dispose(); + setDestroyed(true); + if(log.isDebugEnabled()) { + log.debug("Cluster Monitor Drools session has been disposed. "+this.toString()); + } + } @Override public String toString() { - return "ClusterMonitor [clusterId=" + clusterId + ", serviceId=" + serviceId + + return "ClusterMonitor [clusterId=" + getClusterId() + ", serviceId=" + getServiceId() + ", deploymentPolicy=" + deploymentPolicy + ", autoscalePolicy=" + autoscalePolicy + ", lbReferenceType=" + lbReferenceType + ", hasPrimary=" + hasPrimary + " ]"; @@ -212,12 +228,4 @@ public boolean isHasPrimary() { public void setHasPrimary(boolean hasPrimary) { this.hasPrimary = hasPrimary; } - - public ClusterStatus getStatus() { - return status; - } - - public void setStatus(ClusterStatus status) { - this.status = status; - } } diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/util/AutoscalerUtil.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/util/AutoscalerUtil.java index 1bc9ce77b3..4f58e8da27 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/util/AutoscalerUtil.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/util/AutoscalerUtil.java @@ -31,9 +31,9 @@ import org.apache.stratos.autoscaler.deployment.policy.DeploymentPolicy; import org.apache.stratos.autoscaler.exception.PartitionValidationException; import org.apache.stratos.autoscaler.exception.PolicyValidationException; -import org.apache.stratos.autoscaler.monitor.ClusterMonitor; -import org.apache.stratos.autoscaler.monitor.KubernetesClusterMonitor; -import org.apache.stratos.autoscaler.monitor.LbClusterMonitor; +import org.apache.stratos.autoscaler.monitor.VMServiceClusterMonitor; +import org.apache.stratos.autoscaler.monitor.DockerServiceClusterMonitor; +import org.apache.stratos.autoscaler.monitor.VMLbClusterMonitor; import org.apache.stratos.autoscaler.partition.PartitionGroup; import org.apache.stratos.autoscaler.partition.PartitionManager; import org.apache.stratos.autoscaler.policy.PolicyManager; @@ -73,292 +73,292 @@ private AutoscalerUtil() { * @throws PolicyValidationException * @throws PartitionValidationException */ - public static ClusterMonitor getClusterMonitor(Cluster cluster) throws PolicyValidationException, PartitionValidationException { - // FIXME fix the following code to correctly update - // AutoscalerContext context = AutoscalerContext.getInstance(); - if (null == cluster) { - return null; - } - - String autoscalePolicyName = cluster.getAutoscalePolicyName(); - String deploymentPolicyName = cluster.getDeploymentPolicyName(); - - if (log.isDebugEnabled()) { - log.debug("Deployment policy name: " + deploymentPolicyName); - log.debug("Autoscaler policy name: " + autoscalePolicyName); - } - - AutoscalePolicy policy = - PolicyManager.getInstance() - .getAutoscalePolicy(autoscalePolicyName); - DeploymentPolicy deploymentPolicy = - PolicyManager.getInstance() - .getDeploymentPolicy(deploymentPolicyName); - - if (deploymentPolicy == null) { - String msg = "Deployment Policy is null. Policy name: " + deploymentPolicyName; - log.error(msg); - throw new PolicyValidationException(msg); - } - - Partition[] allPartitions = deploymentPolicy.getAllPartitions(); - if (allPartitions == null) { - String msg = - "Deployment Policy's Partitions are null. Policy name: " + - deploymentPolicyName; - log.error(msg); - throw new PolicyValidationException(msg); - } - - CloudControllerClient.getInstance().validateDeploymentPolicy(cluster.getServiceName(), deploymentPolicy); - - ClusterMonitor clusterMonitor = - new ClusterMonitor(cluster.getClusterId(), - cluster.getServiceName(), - deploymentPolicy, policy); - clusterMonitor.setStatus(ClusterStatus.Created); - - for (PartitionGroup partitionGroup: deploymentPolicy.getPartitionGroups()){ - - NetworkPartitionContext networkPartitionContext = new NetworkPartitionContext(partitionGroup.getId(), - partitionGroup.getPartitionAlgo(), partitionGroup.getPartitions()); - - for(Partition partition: partitionGroup.getPartitions()){ - PartitionContext partitionContext = new PartitionContext(partition); - partitionContext.setServiceName(cluster.getServiceName()); - partitionContext.setProperties(cluster.getProperties()); - partitionContext.setNetworkPartitionId(partitionGroup.getId()); - - for (Member member: cluster.getMembers()){ - String memberId = member.getMemberId(); - if(member.getPartitionId().equalsIgnoreCase(partition.getId())){ - MemberContext memberContext = new MemberContext(); - memberContext.setClusterId(member.getClusterId()); - memberContext.setMemberId(memberId); - memberContext.setPartition(partition); - memberContext.setProperties(convertMemberPropsToMemberContextProps(member.getProperties())); - - if(MemberStatus.Activated.equals(member.getStatus())){ - partitionContext.addActiveMember(memberContext); -// networkPartitionContext.increaseMemberCountOfPartition(partition.getNetworkPartitionId(), 1); -// partitionContext.incrementCurrentActiveMemberCount(1); - - } else if(MemberStatus.Created.equals(member.getStatus()) || MemberStatus.Starting.equals(member.getStatus())){ - partitionContext.addPendingMember(memberContext); - -// networkPartitionContext.increaseMemberCountOfPartition(partition.getNetworkPartitionId(), 1); - } else if(MemberStatus.Suspended.equals(member.getStatus())){ -// partitionContext.addFaultyMember(memberId); - } - partitionContext.addMemberStatsContext(new MemberStatsContext(memberId)); - if(log.isInfoEnabled()){ - log.info(String.format("Member stat context has been added: [member] %s", memberId)); - } - } - - } - networkPartitionContext.addPartitionContext(partitionContext); - if(log.isInfoEnabled()){ - log.info(String.format("Partition context has been added: [partition] %s", - partitionContext.getPartitionId())); - } - } - - clusterMonitor.addNetworkPartitionCtxt(networkPartitionContext); - if(log.isInfoEnabled()){ - log.info(String.format("Network partition context has been added: [network partition] %s", - networkPartitionContext.getId())); - } - } - - - // find lb reference type - java.util.Properties props = cluster.getProperties(); - - if(props.containsKey(Constants.LOAD_BALANCER_REF)) { - String value = props.getProperty(Constants.LOAD_BALANCER_REF); - clusterMonitor.setLbReferenceType(value); - if(log.isDebugEnabled()) { - log.debug("Set the lb reference type: "+value); - } - } - - // set hasPrimary property - // hasPrimary is true if there are primary members available in that cluster - clusterMonitor.setHasPrimary(Boolean.parseBoolean(cluster.getProperties().getProperty(Constants.IS_PRIMARY))); - - log.info("Cluster monitor created: "+clusterMonitor.toString()); - return clusterMonitor; - } - - private static Properties convertMemberPropsToMemberContextProps( - java.util.Properties properties) { - Properties props = new Properties(); - for (Map.Entry e : properties.entrySet() ) { - Property prop = new Property(); - prop.setName((String)e.getKey()); - prop.setValue((String)e.getValue()); - props.addProperties(prop); - } - return props; - } - - - public static LbClusterMonitor getLBClusterMonitor(Cluster cluster) throws PolicyValidationException, PartitionValidationException { - // FIXME fix the following code to correctly update - // AutoscalerContext context = AutoscalerContext.getInstance(); - if (null == cluster) { - return null; - } - - String autoscalePolicyName = cluster.getAutoscalePolicyName(); - String deploymentPolicyName = cluster.getDeploymentPolicyName(); - - if (log.isDebugEnabled()) { - log.debug("Deployment policy name: " + deploymentPolicyName); - log.debug("Autoscaler policy name: " + autoscalePolicyName); - } - - AutoscalePolicy policy = - PolicyManager.getInstance() - .getAutoscalePolicy(autoscalePolicyName); - DeploymentPolicy deploymentPolicy = - PolicyManager.getInstance() - .getDeploymentPolicy(deploymentPolicyName); - - if (deploymentPolicy == null) { - String msg = "Deployment Policy is null. Policy name: " + deploymentPolicyName; - log.error(msg); - throw new PolicyValidationException(msg); - } - - String clusterId = cluster.getClusterId(); - LbClusterMonitor clusterMonitor = - new LbClusterMonitor(clusterId, - cluster.getServiceName(), - deploymentPolicy, policy); - clusterMonitor.setStatus(ClusterStatus.Created); - // partition group = network partition context - for (PartitionGroup partitionGroup : deploymentPolicy.getPartitionGroups()) { - - NetworkPartitionLbHolder networkPartitionLbHolder = - PartitionManager.getInstance() - .getNetworkPartitionLbHolder(partitionGroup.getId()); +// public static ClusterMonitor getClusterMonitor(Cluster cluster) throws PolicyValidationException, PartitionValidationException { +// // FIXME fix the following code to correctly update +// // AutoscalerContext context = AutoscalerContext.getInstance(); +// if (null == cluster) { +// return null; +// } +// +// String autoscalePolicyName = cluster.getAutoscalePolicyName(); +// String deploymentPolicyName = cluster.getDeploymentPolicyName(); +// +// if (log.isDebugEnabled()) { +// log.debug("Deployment policy name: " + deploymentPolicyName); +// log.debug("Autoscaler policy name: " + autoscalePolicyName); +// } +// +// AutoscalePolicy policy = +// PolicyManager.getInstance() +// .getAutoscalePolicy(autoscalePolicyName); +// DeploymentPolicy deploymentPolicy = +// PolicyManager.getInstance() +// .getDeploymentPolicy(deploymentPolicyName); +// +// if (deploymentPolicy == null) { +// String msg = "Deployment Policy is null. Policy name: " + deploymentPolicyName; +// log.error(msg); +// throw new PolicyValidationException(msg); +// } +// +// Partition[] allPartitions = deploymentPolicy.getAllPartitions(); +// if (allPartitions == null) { +// String msg = +// "Deployment Policy's Partitions are null. Policy name: " + +// deploymentPolicyName; +// log.error(msg); +// throw new PolicyValidationException(msg); +// } +// +// CloudControllerClient.getInstance().validateDeploymentPolicy(cluster.getServiceName(), deploymentPolicy); +// +// ClusterMonitor clusterMonitor = +// new ClusterMonitor(cluster.getClusterId(), +// cluster.getServiceName(), +// deploymentPolicy, policy); +// clusterMonitor.setStatus(ClusterStatus.Created); +// +// for (PartitionGroup partitionGroup: deploymentPolicy.getPartitionGroups()){ +// +// NetworkPartitionContext networkPartitionContext = new NetworkPartitionContext(partitionGroup.getId(), +// partitionGroup.getPartitionAlgo(), partitionGroup.getPartitions()); +// +// for(Partition partition: partitionGroup.getPartitions()){ +// PartitionContext partitionContext = new PartitionContext(partition); +// partitionContext.setServiceName(cluster.getServiceName()); +// partitionContext.setProperties(cluster.getProperties()); +// partitionContext.setNetworkPartitionId(partitionGroup.getId()); +// +// for (Member member: cluster.getMembers()){ +// String memberId = member.getMemberId(); +// if(member.getPartitionId().equalsIgnoreCase(partition.getId())){ +// MemberContext memberContext = new MemberContext(); +// memberContext.setClusterId(member.getClusterId()); +// memberContext.setMemberId(memberId); +// memberContext.setPartition(partition); +// memberContext.setProperties(convertMemberPropsToMemberContextProps(member.getProperties())); +// +// if(MemberStatus.Activated.equals(member.getStatus())){ +// partitionContext.addActiveMember(memberContext); +//// networkPartitionContext.increaseMemberCountOfPartition(partition.getNetworkPartitionId(), 1); +//// partitionContext.incrementCurrentActiveMemberCount(1); +// +// } else if(MemberStatus.Created.equals(member.getStatus()) || MemberStatus.Starting.equals(member.getStatus())){ +// partitionContext.addPendingMember(memberContext); +// +//// networkPartitionContext.increaseMemberCountOfPartition(partition.getNetworkPartitionId(), 1); +// } else if(MemberStatus.Suspended.equals(member.getStatus())){ +//// partitionContext.addFaultyMember(memberId); +// } +// partitionContext.addMemberStatsContext(new MemberStatsContext(memberId)); +// if(log.isInfoEnabled()){ +// log.info(String.format("Member stat context has been added: [member] %s", memberId)); +// } +// } +// +// } +// networkPartitionContext.addPartitionContext(partitionContext); +// if(log.isInfoEnabled()){ +// log.info(String.format("Partition context has been added: [partition] %s", +// partitionContext.getPartitionId())); +// } +// } +// +// clusterMonitor.addNetworkPartitionCtxt(networkPartitionContext); +// if(log.isInfoEnabled()){ +// log.info(String.format("Network partition context has been added: [network partition] %s", +// networkPartitionContext.getId())); +// } +// } +// +// +// // find lb reference type +// java.util.Properties props = cluster.getProperties(); +// +// if(props.containsKey(Constants.LOAD_BALANCER_REF)) { +// String value = props.getProperty(Constants.LOAD_BALANCER_REF); +// clusterMonitor.setLbReferenceType(value); +// if(log.isDebugEnabled()) { +// log.debug("Set the lb reference type: "+value); +// } +// } +// +// // set hasPrimary property +// // hasPrimary is true if there are primary members available in that cluster +// clusterMonitor.setHasPrimary(Boolean.parseBoolean(cluster.getProperties().getProperty(Constants.IS_PRIMARY))); +// +// log.info("Cluster monitor created: "+clusterMonitor.toString()); +// return clusterMonitor; +// } +// +// private static Properties convertMemberPropsToMemberContextProps( +// java.util.Properties properties) { +// Properties props = new Properties(); +// for (Map.Entry e : properties.entrySet() ) { +// Property prop = new Property(); +// prop.setName((String)e.getKey()); +// prop.setValue((String)e.getValue()); +// props.addProperties(prop); +// } +// return props; +// } +// +// +// public static LbClusterMonitor getLBClusterMonitor(Cluster cluster) throws PolicyValidationException, PartitionValidationException { +// // FIXME fix the following code to correctly update +// // AutoscalerContext context = AutoscalerContext.getInstance(); +// if (null == cluster) { +// return null; +// } +// +// String autoscalePolicyName = cluster.getAutoscalePolicyName(); +// String deploymentPolicyName = cluster.getDeploymentPolicyName(); +// +// if (log.isDebugEnabled()) { +// log.debug("Deployment policy name: " + deploymentPolicyName); +// log.debug("Autoscaler policy name: " + autoscalePolicyName); +// } +// +// AutoscalePolicy policy = +// PolicyManager.getInstance() +// .getAutoscalePolicy(autoscalePolicyName); +// DeploymentPolicy deploymentPolicy = +// PolicyManager.getInstance() +// .getDeploymentPolicy(deploymentPolicyName); +// +// if (deploymentPolicy == null) { +// String msg = "Deployment Policy is null. Policy name: " + deploymentPolicyName; +// log.error(msg); +// throw new PolicyValidationException(msg); +// } +// +// String clusterId = cluster.getClusterId(); +// LbClusterMonitor clusterMonitor = +// new LbClusterMonitor(clusterId, +// cluster.getServiceName(), +// deploymentPolicy, policy); +// clusterMonitor.setStatus(ClusterStatus.Created); +// // partition group = network partition context +// for (PartitionGroup partitionGroup : deploymentPolicy.getPartitionGroups()) { +// +// NetworkPartitionLbHolder networkPartitionLbHolder = // PartitionManager.getInstance() // .getNetworkPartitionLbHolder(partitionGroup.getId()); - // FIXME pick a random partition - Partition partition = - partitionGroup.getPartitions()[new Random().nextInt(partitionGroup.getPartitions().length)]; - PartitionContext partitionContext = new PartitionContext(partition); - partitionContext.setServiceName(cluster.getServiceName()); - partitionContext.setProperties(cluster.getProperties()); - partitionContext.setNetworkPartitionId(partitionGroup.getId()); - partitionContext.setMinimumMemberCount(1);//Here it hard codes the minimum value as one for LB cartridge partitions - - NetworkPartitionContext networkPartitionContext = new NetworkPartitionContext(partitionGroup.getId(), - partitionGroup.getPartitionAlgo(), partitionGroup.getPartitions()) ; - for (Member member : cluster.getMembers()) { - String memberId = member.getMemberId(); - if (member.getNetworkPartitionId().equalsIgnoreCase(networkPartitionContext.getId())) { - MemberContext memberContext = new MemberContext(); - memberContext.setClusterId(member.getClusterId()); - memberContext.setMemberId(memberId); - memberContext.setPartition(partition); - - if (MemberStatus.Activated.equals(member.getStatus())) { - partitionContext.addActiveMember(memberContext); -// networkPartitionContext.increaseMemberCountOfPartition(partition.getNetworkPartitionId(), 1); -// partitionContext.incrementCurrentActiveMemberCount(1); - } else if (MemberStatus.Created.equals(member.getStatus()) || - MemberStatus.Starting.equals(member.getStatus())) { - partitionContext.addPendingMember(memberContext); -// networkPartitionContext.increaseMemberCountOfPartition(partition.getNetworkPartitionId(), 1); - } else if (MemberStatus.Suspended.equals(member.getStatus())) { -// partitionContext.addFaultyMember(memberId); - } - - partitionContext.addMemberStatsContext(new MemberStatsContext(memberId)); - if(log.isInfoEnabled()){ - log.info(String.format("Member stat context has been added: [member] %s", memberId)); - } - } - - } - networkPartitionContext.addPartitionContext(partitionContext); - - // populate lb cluster id in network partition context. - java.util.Properties props = cluster.getProperties(); - - // get service type of load balanced cluster - String loadBalancedServiceType = props.getProperty(Constants.LOAD_BALANCED_SERVICE_TYPE); - - if(props.containsKey(Constants.LOAD_BALANCER_REF)) { - String value = props.getProperty(Constants.LOAD_BALANCER_REF); - - if (value.equals(org.apache.stratos.messaging.util.Constants.DEFAULT_LOAD_BALANCER)) { - networkPartitionLbHolder.setDefaultLbClusterId(clusterId); - - } else if (value.equals(org.apache.stratos.messaging.util.Constants.SERVICE_AWARE_LOAD_BALANCER)) { - String serviceName = cluster.getServiceName(); - // TODO: check if this is correct - networkPartitionLbHolder.addServiceLB(serviceName, clusterId); - - if (loadBalancedServiceType != null && !loadBalancedServiceType.isEmpty()) { - networkPartitionLbHolder.addServiceLB(loadBalancedServiceType, clusterId); - if (log.isDebugEnabled()) { - log.debug("Added cluster id " + clusterId + " as the LB cluster id for service type " + loadBalancedServiceType); - } - } - } - } - - clusterMonitor.addNetworkPartitionCtxt(networkPartitionContext); - } - - log.info("LB Cluster monitor created: "+clusterMonitor.toString()); - return clusterMonitor; - } - - public static KubernetesClusterMonitor getKubernetesClusterMonitor(Cluster cluster) { - - if (null == cluster) { - return null; - } - - String autoscalePolicyName = cluster.getAutoscalePolicyName(); - if (log.isDebugEnabled()) { - log.debug("Autoscaler policy name: " + autoscalePolicyName); - } - - AutoscalePolicy policy = PolicyManager.getInstance().getAutoscalePolicy(autoscalePolicyName); - java.util.Properties props = cluster.getProperties(); - String kubernetesHostClusterID = props.getProperty(StratosConstants.KUBERNETES_CLUSTER_ID); - KubernetesClusterContext kubernetesClusterCtxt = new KubernetesClusterContext(kubernetesHostClusterID); - - KubernetesClusterMonitor kubernetesClusterMonitor = new KubernetesClusterMonitor( - kubernetesClusterCtxt, - cluster.getClusterId(), - cluster.getServiceName(), - policy); - - kubernetesClusterMonitor.setStatus(ClusterStatus.Created); - - // find lb reference type - if(props.containsKey(Constants.LOAD_BALANCER_REF)) { - String value = props.getProperty(Constants.LOAD_BALANCER_REF); - kubernetesClusterMonitor.setLbReferenceType(value); - if(log.isDebugEnabled()) { - log.debug("Set the lb reference type: "+value); - } - } - - // set hasPrimary property - // hasPrimary is true if there are primary members available in that cluster - kubernetesClusterMonitor.setHasPrimary(Boolean.parseBoolean(props.getProperty(Constants.IS_PRIMARY))); - - log.info("Kubernetes cluster monitor created: "+ kubernetesClusterMonitor.toString()); - return kubernetesClusterMonitor; - } +//// PartitionManager.getInstance() +//// .getNetworkPartitionLbHolder(partitionGroup.getId()); +// // FIXME pick a random partition +// Partition partition = +// partitionGroup.getPartitions()[new Random().nextInt(partitionGroup.getPartitions().length)]; +// PartitionContext partitionContext = new PartitionContext(partition); +// partitionContext.setServiceName(cluster.getServiceName()); +// partitionContext.setProperties(cluster.getProperties()); +// partitionContext.setNetworkPartitionId(partitionGroup.getId()); +// partitionContext.setMinimumMemberCount(1);//Here it hard codes the minimum value as one for LB cartridge partitions +// +// NetworkPartitionContext networkPartitionContext = new NetworkPartitionContext(partitionGroup.getId(), +// partitionGroup.getPartitionAlgo(), partitionGroup.getPartitions()) ; +// for (Member member : cluster.getMembers()) { +// String memberId = member.getMemberId(); +// if (member.getNetworkPartitionId().equalsIgnoreCase(networkPartitionContext.getId())) { +// MemberContext memberContext = new MemberContext(); +// memberContext.setClusterId(member.getClusterId()); +// memberContext.setMemberId(memberId); +// memberContext.setPartition(partition); +// +// if (MemberStatus.Activated.equals(member.getStatus())) { +// partitionContext.addActiveMember(memberContext); +//// networkPartitionContext.increaseMemberCountOfPartition(partition.getNetworkPartitionId(), 1); +//// partitionContext.incrementCurrentActiveMemberCount(1); +// } else if (MemberStatus.Created.equals(member.getStatus()) || +// MemberStatus.Starting.equals(member.getStatus())) { +// partitionContext.addPendingMember(memberContext); +//// networkPartitionContext.increaseMemberCountOfPartition(partition.getNetworkPartitionId(), 1); +// } else if (MemberStatus.Suspended.equals(member.getStatus())) { +//// partitionContext.addFaultyMember(memberId); +// } +// +// partitionContext.addMemberStatsContext(new MemberStatsContext(memberId)); +// if(log.isInfoEnabled()){ +// log.info(String.format("Member stat context has been added: [member] %s", memberId)); +// } +// } +// +// } +// networkPartitionContext.addPartitionContext(partitionContext); +// +// // populate lb cluster id in network partition context. +// java.util.Properties props = cluster.getProperties(); +// +// // get service type of load balanced cluster +// String loadBalancedServiceType = props.getProperty(Constants.LOAD_BALANCED_SERVICE_TYPE); +// +// if(props.containsKey(Constants.LOAD_BALANCER_REF)) { +// String value = props.getProperty(Constants.LOAD_BALANCER_REF); +// +// if (value.equals(org.apache.stratos.messaging.util.Constants.DEFAULT_LOAD_BALANCER)) { +// networkPartitionLbHolder.setDefaultLbClusterId(clusterId); +// +// } else if (value.equals(org.apache.stratos.messaging.util.Constants.SERVICE_AWARE_LOAD_BALANCER)) { +// String serviceName = cluster.getServiceName(); +// // TODO: check if this is correct +// networkPartitionLbHolder.addServiceLB(serviceName, clusterId); +// +// if (loadBalancedServiceType != null && !loadBalancedServiceType.isEmpty()) { +// networkPartitionLbHolder.addServiceLB(loadBalancedServiceType, clusterId); +// if (log.isDebugEnabled()) { +// log.debug("Added cluster id " + clusterId + " as the LB cluster id for service type " + loadBalancedServiceType); +// } +// } +// } +// } +// +// clusterMonitor.addNetworkPartitionCtxt(networkPartitionContext); +// } +// +// log.info("LB Cluster monitor created: "+clusterMonitor.toString()); +// return clusterMonitor; +// } +// +// public static DockerClusterMonitor getDockerClusterMonitor(Cluster cluster) { +// +// if (null == cluster) { +// return null; +// } +// +// String autoscalePolicyName = cluster.getAutoscalePolicyName(); +// if (log.isDebugEnabled()) { +// log.debug("Autoscaler policy name: " + autoscalePolicyName); +// } +// +// AutoscalePolicy policy = PolicyManager.getInstance().getAutoscalePolicy(autoscalePolicyName); +// java.util.Properties props = cluster.getProperties(); +// String kubernetesHostClusterID = props.getProperty(StratosConstants.KUBERNETES_CLUSTER_ID); +// KubernetesClusterContext kubernetesClusterCtxt = new KubernetesClusterContext(kubernetesHostClusterID); +// +// DockerClusterMonitor dockerClusterMonitor = new DockerClusterMonitor( +// kubernetesClusterCtxt, +// cluster.getClusterId(), +// cluster.getServiceName(), +// policy); +// +// dockerClusterMonitor.setStatus(ClusterStatus.Created); +// +// // find lb reference type +// if(props.containsKey(Constants.LOAD_BALANCER_REF)) { +// String value = props.getProperty(Constants.LOAD_BALANCER_REF); +// dockerClusterMonitor.setLbReferenceType(value); +// if(log.isDebugEnabled()) { +// log.debug("Set the lb reference type: "+value); +// } +// } +// +//// // set hasPrimary property +//// // hasPrimary is true if there are primary members available in that cluster +//// dockerClusterMonitor.setHasPrimary(Boolean.parseBoolean(props.getProperty(Constants.IS_PRIMARY))); +// +// log.info("Docker cluster monitor created: "+ dockerClusterMonitor.toString()); +// return dockerClusterMonitor; +// } public static Properties getProperties(final OMElement elt) { diff --git a/components/org.apache.stratos.common/src/main/java/org/apache/stratos/common/enums/ClusterType.java b/components/org.apache.stratos.common/src/main/java/org/apache/stratos/common/enums/ClusterType.java new file mode 100644 index 0000000000..8842fb6995 --- /dev/null +++ b/components/org.apache.stratos.common/src/main/java/org/apache/stratos/common/enums/ClusterType.java @@ -0,0 +1,5 @@ +package org.apache.stratos.common.enums; + +public enum ClusterType { + VMServiceCluster, VMLbCluster, DockerServiceCluster, DockerLbCluster; +} From 8f49d7fba8db8abc78ff5460f1df27e94c56f3a4 Mon Sep 17 00:00:00 2001 From: R-Rajkumar Date: Tue, 23 Sep 2014 14:32:34 +0530 Subject: [PATCH 2/2] member fault events removed for docker clusters --- .../AutoscalerHealthStatEventReceiver.java | 36 +----------- .../AutoscalerTopologyEventReceiver.java | 55 ++----------------- 2 files changed, 5 insertions(+), 86 deletions(-) diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/message/receiver/health/AutoscalerHealthStatEventReceiver.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/message/receiver/health/AutoscalerHealthStatEventReceiver.java index 2efcef0a49..ccb0f4ed19 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/message/receiver/health/AutoscalerHealthStatEventReceiver.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/message/receiver/health/AutoscalerHealthStatEventReceiver.java @@ -913,41 +913,7 @@ private void handleMemberFaultEvent(String clusterId, String memberId) { memberId, partitionId, clusterId)); } } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { - try{ - TopologyManager.acquireReadLock(); - Member member = findMember(memberId); - - if(null == member){ - return; - } - if(!member.isActive()){ - if(log.isDebugEnabled()){ - log.debug(String.format("Member activated event has not received for the member %s. Therefore ignoring" + - " the member fault health stat", memberId)); - } - return; - } - }finally{ - TopologyManager.releaseReadLock(); - } - - KubernetesClusterContext kubernetesClusterContext = ((ContainerClusterMonitor) monitor).getKubernetesClusterCtxt(); - if (!kubernetesClusterContext.activeMemberExist(memberId)) { - if(log.isDebugEnabled()){ - log.debug(String.format("Could not find the active member in partition context, [member] %s ", memberId)); - } - return; - } - //terminate the faulty member - CloudControllerClient.getInstance().terminateContainer(memberId); - //remove from active member list - kubernetesClusterContext.removeActiveMemberById(memberId); - - if (log.isInfoEnabled()) { - log.info(String.format("Faulty member is terminated and removed from the active members list: [member] %s [kub cluster] %s [cluster] %s ", - memberId, kubernetesClusterContext.getKubernetesClusterID(), clusterId)); - } - + // no need to do anything } } catch (TerminationException e) { diff --git a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/message/receiver/topology/AutoscalerTopologyEventReceiver.java b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/message/receiver/topology/AutoscalerTopologyEventReceiver.java index 6525eba913..cb16768fab 100644 --- a/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/message/receiver/topology/AutoscalerTopologyEventReceiver.java +++ b/components/org.apache.stratos.autoscaler/src/main/java/org/apache/stratos/autoscaler/message/receiver/topology/AutoscalerTopologyEventReceiver.java @@ -174,16 +174,7 @@ protected void onEvent(Event event) { memberId, partitionId, clusterId)); } } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { - KubernetesClusterContext kubernetesClusterContext = ((ContainerClusterMonitor) monitor).getKubernetesClusterCtxt(); - //terminate the shutdown ready container - CloudControllerClient.getInstance().terminateContainer(memberId); - //remove from active member list - kubernetesClusterContext.removeActiveMemberById(memberId); - - if (log.isInfoEnabled()) { - log.info(String.format("Member is terminated and removed from the active members list: [member] %s [kub cluster] %s [cluster] %s ", - memberId, kubernetesClusterContext.getKubernetesClusterID(), clusterId)); - } + // no need to do anything } } catch (TerminationException e) { @@ -351,29 +342,7 @@ protected void onEvent(Event event) { log.info(String.format("Member stat context has been removed successfully: [member] %s", memberId)); } } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { - - KubernetesClusterContext kubernetesClusterContext = ((ContainerClusterMonitor) monitor).getKubernetesClusterCtxt(); - kubernetesClusterContext.removeMemberStatsContext(memberId); - - if (kubernetesClusterContext.removeTerminationPendingMember(memberId)) { - if (log.isDebugEnabled()) { - log.debug(String.format("Member is removed from termination pending members list: [member] %s", memberId)); - } - } else if (kubernetesClusterContext.removePendingMember(memberId)) { - if (log.isDebugEnabled()) { - log.debug(String.format("Member is removed from pending members list: [member] %s", memberId)); - } - } else if (kubernetesClusterContext.removeActiveMemberById(memberId)) { - log.warn(String.format("Member is in the wrong list and it is removed from active members list", memberId)); - } else if (kubernetesClusterContext.removeObsoleteMember(memberId)){ - log.warn(String.format("Member's obsolated timeout has been expired and it is removed from obsolated members list", memberId)); - } else { - log.warn(String.format("Member is not available in any of the list active, pending and termination pending", memberId)); - } - - if (log.isInfoEnabled()) { - log.info(String.format("Member stat context has been removed successfully: [member] %s", memberId)); - } + // no need to do anything } } catch (Exception e) { @@ -480,17 +449,7 @@ protected void onEvent(Event event) { memberId, partitionId, clusterId)); } } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { - KubernetesClusterContext kubernetesClusterContext; - kubernetesClusterContext = ((ContainerClusterMonitor) monitor).getKubernetesClusterCtxt(); - // terminate the shutdown ready member - CloudControllerClient.getInstance().terminateContainer(memberId); - // remove from active member list - kubernetesClusterContext.removeActiveMemberById(memberId); - - if (log.isInfoEnabled()) { - log.info(String.format("Member is terminated and removed from the active members list: [member] %s [kub cluster] %s [cluster] %s ", - memberId, kubernetesClusterContext.getKubernetesClusterID(), clusterId)); - } + // no need to do anything } } catch (TerminationException e) { @@ -537,13 +496,7 @@ protected void onEvent(Event event) { } partitionContext.moveActiveMemberToTerminationPendingMembers(memberId); } else if(monitor.getClusterType() == ClusterType.DockerServiceCluster) { - KubernetesClusterContext kubernetesClusterContext; - kubernetesClusterContext = ((ContainerClusterMonitor) monitor).getKubernetesClusterCtxt(); - kubernetesClusterContext.addMemberStatsContext(new MemberStatsContext(memberId)); - if (log.isDebugEnabled()) { - log.debug(String.format("Member has been moved as pending termination: [member] %s", memberId)); - } - kubernetesClusterContext.moveActiveMemberToTerminationPendingMembers(memberId); + // no need to do anything } } catch (Exception e) {