From dd8b231abb9ca7af1b261d1f6eb4532916d0d019 Mon Sep 17 00:00:00 2001 From: rezra3 Date: Tue, 5 Jun 2018 09:39:03 +0300 Subject: [PATCH 1/2] codeine-239 peer should not change address during up time --- .../connectors/StatusMysqlConnector.java | 52 ++-- .../codeine/jsons/peer_status/PeerStatus.java | 6 +- .../jsons/peer_status/PeerStatusJsonV2.java | 276 +++++++++--------- .../codeine/version/RsyncSourceGetter.java | 31 +- .../ProjectConfigurationInPeerUpdater.java | 6 +- 5 files changed, 182 insertions(+), 189 deletions(-) diff --git a/src/common/codeine/db/mysql/connectors/StatusMysqlConnector.java b/src/common/codeine/db/mysql/connectors/StatusMysqlConnector.java index 511654be..26024f15 100755 --- a/src/common/codeine/db/mysql/connectors/StatusMysqlConnector.java +++ b/src/common/codeine/db/mysql/connectors/StatusMysqlConnector.java @@ -60,7 +60,7 @@ public void createTables() { public void putReplaceStatus(PeerStatusJsonV2 p) { String json = gson.toJson(p); log.info("will update status to " + dbUtils.server() + "\n" + json); - dbUtils.executeUpdate("DELETE FROM "+TABLE_NAME+" WHERE peer_key = '" + p.peer_old_key() + "'"); + dbUtils.executeUpdate("DELETE FROM "+TABLE_NAME+" WHERE peer_key = '" + p.peer_key() + "'"); dbUtils.executeUpdate("REPLACE INTO "+TABLE_NAME+" (peer_key, data, update_time ) VALUES (?, ?, CURRENT_TIMESTAMP())", p.peer_key(), json); } @@ -93,39 +93,35 @@ private void updateNodesWithPeer(PeerStatusJsonV2 peerStatus) { public void updatePeersStatus(final long timeToRemove, final long timeToDisc) { final List idToRemove = Lists.newArrayList(); final List idToDisc = Lists.newArrayList(); - Function function = new Function() { - @Override - public Void apply(ResultSet rs){ - try { - String key = rs.getString("peer_key"); + Function function = rs -> { + try { + String key = rs.getString("peer_key"); // PeerStatusString status = PeerStatusString.valueOf(rs.getString("status")); - String value = rs.getString("data"); - String status = rs.getString("status"); - PeerStatusJsonV2 peerStatus = gson.fromJson(value, PeerStatusJsonV2.class); - PeerType peerType = peerStatus.peer_type(); - long timeToRemovePeer = peerType == PeerType.Reporter ? timeToRemove + TimeUnit.DAYS.toMinutes(7) : timeToRemove; - long timeToDiscPeer = peerType == PeerType.Reporter ? timeToDisc + TimeUnit.DAYS.toMinutes(7) : timeToDisc; - long timeDiff = rs.getLong("TIME_DIFF"); - log.debug("time diff is " + timeDiff); - if (timeDiff > timeToRemovePeer){ - log.info("time diff is " + timeDiff); - log.info("deleting " + peerStatus); + String value = rs.getString("data"); + String status = rs.getString("status"); + PeerStatusJsonV2 peerStatus = gson.fromJson(value, PeerStatusJsonV2.class); + PeerType peerType = peerStatus.peer_type(); + long timeToRemovePeer = peerType == PeerType.Reporter ? timeToRemove + TimeUnit.DAYS.toMinutes(7) : timeToRemove; + long timeToDiscPeer = peerType == PeerType.Reporter ? timeToDisc + TimeUnit.DAYS.toMinutes(7) : timeToDisc; + long timeDiff = rs.getLong("TIME_DIFF"); + log.debug("time diff is " + timeDiff); + if (timeDiff > timeToRemovePeer){ + log.info("time diff is " + timeDiff); + log.info("deleting " + peerStatus); // rs.deleteRow(); - idToRemove.add(key); - } - else if (timeDiff > timeToDiscPeer && !status.equals(PeerStatusString.Disc.toString())){ - log.info("time diff is " + timeDiff); - log.info("update to disc " + peerStatus); - idToDisc.add(key); + idToRemove.add(key); + } + else if (timeDiff > timeToDiscPeer && !status.equals(PeerStatusString.Disc.toString())){ + log.info("time diff is " + timeDiff); + log.info("update to disc " + peerStatus); + idToDisc.add(key); // rs.updateString("status", "Disc"); // rs.updateRow(); - } - return null; - } catch (SQLException e) { - throw ExceptionUtils.asUnchecked(e); } + return null; + } catch (SQLException e) { + throw ExceptionUtils.asUnchecked(e); } - }; dbUtils.executeUpdateableQuery("select *,TIMESTAMPDIFF(MINUTE,update_time,CURRENT_TIMESTAMP()) as TIME_DIFF from " + TABLE_NAME, function); if (webConfJsonStore.get().readonly_web_server()) { diff --git a/src/common/codeine/jsons/peer_status/PeerStatus.java b/src/common/codeine/jsons/peer_status/PeerStatus.java index fb480077..5cedd96a 100755 --- a/src/common/codeine/jsons/peer_status/PeerStatus.java +++ b/src/common/codeine/jsons/peer_status/PeerStatus.java @@ -26,6 +26,8 @@ public class PeerStatus { private static final Logger log = Logger.getLogger(PeerStatus.class); private Map project_name_to_status = Maps.newConcurrentMap(); + private String canonical_host_name = InetUtils.getLocalHost().getCanonicalHostName(); + private String host_address = InetUtils.getLocalHost().getHostAddress(); @Inject private CodeineRuntimeInfo codeineRuntimeInfo; @@ -96,9 +98,9 @@ public Map project_name_to_status() { } public PeerStatusJsonV2 createJson() { - return new PeerStatusJsonV2(InetUtils.getLocalHost().getHostName(), codeineRuntimeInfo.port(), + return new PeerStatusJsonV2(codeineRuntimeInfo.port(), codeineRuntimeInfo.version(), codeineRuntimeInfo.startTime(), Constants.getInstallDir(), - PathHelper.getTarFile(), project_name_to_status(), InetUtils.getLocalHost().getHostAddress(), System.getProperty("DNS_DOMAIN_NAME"), InetUtils.getLocalHost().getCanonicalHostName()); + PathHelper.getTarFile(), project_name_to_status(), host_address, System.getProperty("DNS_DOMAIN_NAME"), canonical_host_name); } public String updateVersion(ProjectJson project, String node, String alias, String version) { diff --git a/src/common/codeine/jsons/peer_status/PeerStatusJsonV2.java b/src/common/codeine/jsons/peer_status/PeerStatusJsonV2.java index 7f7384ca..734e333c 100755 --- a/src/common/codeine/jsons/peer_status/PeerStatusJsonV2.java +++ b/src/common/codeine/jsons/peer_status/PeerStatusJsonV2.java @@ -16,139 +16,145 @@ @SuppressWarnings("unused") public class PeerStatusJsonV2 { - private String peer_key; - //TODO remove after cf-engine in build > 1.1.309 - private String peer_old_key; - private String peer_host_port; - private String peer_ip; - private String user_dns_domain; - private Map project_name_to_status = Maps.newHashMap();//Lists.newArrayList(); - private String host; - private String canonical_host; //TODO introduced in codeine 1202, can be used after cfengine is in that build - private int port; - private String version; - private String tar; - private long start_time; - private long update_time;//updated in directory server when first seen - private long update_time_from_peer; - private String install_dir; - private PeerType peer_type; - private transient PeerStatusString status; - - public PeerStatusJsonV2(String host, int port, String version, long start_time, String install_dir, String tar, Map project_name_to_status, String peer_ip, String user_dns_domain, String canonical_host) { - super(); - this.host = host; - this.canonical_host = canonical_host; - this.port = port; - this.peer_ip = peer_ip; - this.version = version; - this.start_time = start_time; - this.install_dir = install_dir; - this.tar = tar; - this.project_name_to_status = Maps.newHashMap(project_name_to_status); - this.peer_old_key = host + ":" + install_dir; - this.peer_key = host + ":" + HttpUtils.specialEncode(install_dir); - this.peer_host_port = host + ":" + port; - this.user_dns_domain = user_dns_domain; - this.peer_type = PeerType.Daemon; - this.project_name_to_status.put(Constants.CODEINE_NODES_PROJECT_NAME, createInternalProject()); - this.update_time = System.currentTimeMillis(); - this.update_time_from_peer = System.currentTimeMillis(); - } - private ProjectStatus createInternalProject() { - NodeWithMonitorsInfo node_info = new NodeWithMonitorsInfo(this, this.peer_key, this.host, Constants.CODEINE_NODES_PROJECT_NAME, Maps.newHashMap()); - node_info.version(this.version); - node_info.tags(Lists.newArrayList(project_name_to_status.keySet())); - ProjectStatus ps = new ProjectStatus(Constants.CODEINE_NODES_PROJECT_NAME, node_info); - return ps; - } - public PeerStatusJsonV2(String peer_key, ProjectStatus projectStatus) { - super(); - this.project_name_to_status = Maps.newHashMap(); - this.project_name_to_status.put(projectStatus.project_name(), projectStatus); - this.update_time = System.currentTimeMillis(); - this.update_time_from_peer = System.currentTimeMillis(); - this.peer_key = peer_key; - this.peer_type = PeerType.Reporter; - } - - public void addProjectStatus(String name, ProjectStatus status) { - HashMap tempList = Maps.newHashMap(project_name_to_status); - tempList.put(name, status); - project_name_to_status = tempList; - } - - public Map project_name_to_status() { - return Collections.unmodifiableMap(project_name_to_status); - } - - public String peer_key() { - return peer_key; - } - - public String host_port() { - return host + ":" + port; - } - public String canonical_host_port() { - return canonical_host + ":" + port; - } - public String ip_port() { - return peer_ip + ":" + port; - } - - public String address_port() { - if (!StringUtils.isEmpty(user_dns_domain)) { - return host + "." + user_dns_domain + ":" + port; - } else if (!StringUtils.isEmpty(canonical_host)) { - return canonical_host_port(); - } else { - return host_port(); - } - } - - public long update_time() { - return update_time; - } - public long update_time_from_peer() { - return update_time_from_peer; - } - - public String key() { - return peer_key(); - } - - public String version() { - return version; - } - - public String host() { - return host; - } - - public String tar() { - return tar; - } - public void status(PeerStatusString status) { - this.status = status; - } - public PeerStatusString status() { - return status; - } - public void updateNodesWithPeer() { - for (ProjectStatus projectStatus : project_name_to_status.values()) { - projectStatus.updateNodesWithPeer(this); - } - } - public PeerType peer_type() { - return peer_type; - } - public String peer_old_key() { - return peer_old_key; - } - @Override - public String toString() { - return "PeerStatusJsonV2 [host_port()=" + host_port() + ", update_time()=" + new Date(update_time()) - + ", update_time_from_peer()=" + new Date(update_time_from_peer()) + ", peer_type()=" + peer_type() + "]"; - } - + + private String peer_key; + private String peer_host_port; + private String peer_ip; + private String user_dns_domain; + private Map project_name_to_status; + private String canonical_host; + private int port; + private String version; + private String tar; + private long start_time; + private long update_time;//updated in directory server when first seen + private long update_time_from_peer; + private String install_dir; + private PeerType peer_type; + private transient PeerStatusString status; + + public PeerStatusJsonV2(int port, String version, long start_time, + String install_dir, String tar, Map project_name_to_status, + String peer_ip, String user_dns_domain, String canonical_host) { + super(); + this.canonical_host = canonical_host; + this.port = port; + this.peer_ip = peer_ip; + this.version = version; + this.start_time = start_time; + this.install_dir = install_dir; + this.tar = tar; + this.project_name_to_status = Maps.newHashMap(project_name_to_status); + this.peer_key = canonical_host + ":" + HttpUtils.specialEncode(install_dir); + this.peer_host_port = canonical_host + ":" + port; + this.user_dns_domain = user_dns_domain; + this.peer_type = PeerType.Daemon; + this.project_name_to_status + .put(Constants.CODEINE_NODES_PROJECT_NAME, createInternalProject()); + this.update_time = System.currentTimeMillis(); + this.update_time_from_peer = System.currentTimeMillis(); + } + + private ProjectStatus createInternalProject() { + NodeWithMonitorsInfo node_info = new NodeWithMonitorsInfo(this, this.peer_key, this.canonical_host, + Constants.CODEINE_NODES_PROJECT_NAME, Maps.newHashMap()); + node_info.version(this.version); + node_info.tags(Lists.newArrayList(project_name_to_status.keySet())); + ProjectStatus ps = new ProjectStatus(Constants.CODEINE_NODES_PROJECT_NAME, node_info); + return ps; + } + + public PeerStatusJsonV2(String peer_key, ProjectStatus projectStatus) { + super(); + this.project_name_to_status = Maps.newHashMap(); + this.project_name_to_status.put(projectStatus.project_name(), projectStatus); + this.update_time = System.currentTimeMillis(); + this.update_time_from_peer = System.currentTimeMillis(); + this.peer_key = peer_key; + this.peer_type = PeerType.Reporter; + } + + public void addProjectStatus(String name, ProjectStatus status) { + HashMap tempList = Maps.newHashMap(project_name_to_status); + tempList.put(name, status); + project_name_to_status = tempList; + } + + public Map project_name_to_status() { + return Collections.unmodifiableMap(project_name_to_status); + } + + public String peer_key() { + return peer_key; + } + + public String host_port() { + return canonical_host + ":" + port; + } + + public String canonical_host_port() { + return canonical_host + ":" + port; + } + + public String ip_port() { + return peer_ip + ":" + port; + } + + public String address_port() { + if (!StringUtils.isEmpty(user_dns_domain)) { + return canonical_host + "." + user_dns_domain + ":" + port; + } + return canonical_host_port(); + } + + public long update_time() { + return update_time; + } + + public long update_time_from_peer() { + return update_time_from_peer; + } + + public String key() { + return peer_key(); + } + + public String version() { + return version; + } + + public String tar() { + return tar; + } + + public void status(PeerStatusString status) { + this.status = status; + } + + public PeerStatusString status() { + return status; + } + + public void updateNodesWithPeer() { + for (ProjectStatus projectStatus : project_name_to_status.values()) { + projectStatus.updateNodesWithPeer(this); + } + } + + public PeerType peer_type() { + return peer_type; + } + + public String canonical_host() { + return canonical_host; + } + + @Override + public String toString() { + return "PeerStatusJsonV2 [host_port()=" + host_port() + ", update_time()=" + new Date( + update_time()) + + ", update_time_from_peer()=" + new Date(update_time_from_peer()) + ", peer_type()=" + + peer_type() + "]"; + } + } diff --git a/src/directory/codeine/version/RsyncSourceGetter.java b/src/directory/codeine/version/RsyncSourceGetter.java index 6a4f5638..c364b1d8 100755 --- a/src/directory/codeine/version/RsyncSourceGetter.java +++ b/src/directory/codeine/version/RsyncSourceGetter.java @@ -1,19 +1,15 @@ package codeine.version; -import java.util.List; -import java.util.Random; - -import javax.inject.Inject; - import codeine.api.NodeGetter; import codeine.jsons.info.VersionInfo; import codeine.jsons.peer_status.PeerStatusJsonV2; import codeine.utils.StringUtils; import codeine.utils.network.InetUtils; - -import com.google.common.base.Predicate; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import java.util.List; +import java.util.Random; +import javax.inject.Inject; public class RsyncSourceGetter { @@ -36,19 +32,12 @@ public String getForVersion(String version, final String hostname) { // if ("a".equals("a")){ // return versionsMapping.info(versionName).default_rsync_source; // } - List filteredByVersion = Lists.newArrayList(Iterables.filter(nodesGetter.peers(), new Predicate() { - @Override - public boolean apply(PeerStatusJsonV2 peerStatus){ - return versionName.equals(peerStatus.version()) && !hostname.equals(peerStatus.host()) - && !StringUtils.isEmpty(peerStatus.tar()); - } - })); - List filteredByDomain = Lists.newArrayList(Iterables.filter(filteredByVersion, new Predicate() { - @Override - public boolean apply(PeerStatusJsonV2 peerStatus){ - return hostDomain.equals(InetUtils.domain(peerStatus.host())) && !hostname.equals(peerStatus.host()); - } - })); + List filteredByVersion = Lists.newArrayList(Iterables.filter(nodesGetter.peers(), + peerStatus -> versionName.equals(peerStatus.version()) && !hostname.equals(peerStatus.canonical_host()) + && !StringUtils.isEmpty(peerStatus.tar()))); + List filteredByDomain = Lists.newArrayList(Iterables.filter(filteredByVersion, + peerStatus -> + hostDomain.equals(InetUtils.domain(peerStatus.canonical_host())) && !hostname.equals(peerStatus.canonical_host()))); if (!filteredByDomain.isEmpty()){ String hostUrl = hostUrl(filteredByDomain); if (null != hostUrl){ @@ -71,7 +60,7 @@ private String hostUrl(List list) { return null;// with decreasing probability - don't use that list to prevent bad version somewhere affecting } PeerStatusJsonV2 peerStatusJson = list.get(nextInt); - return peerStatusJson.host() + ":" + peerStatusJson.tar(); + return peerStatusJson.canonical_host() + ":" + peerStatusJson.tar(); } } diff --git a/src/web_server/codeine/ProjectConfigurationInPeerUpdater.java b/src/web_server/codeine/ProjectConfigurationInPeerUpdater.java index 8587354a..3e5fd093 100755 --- a/src/web_server/codeine/ProjectConfigurationInPeerUpdater.java +++ b/src/web_server/codeine/ProjectConfigurationInPeerUpdater.java @@ -113,9 +113,9 @@ private Collection getPeersFromProject(ProjectJson p log.debug("Nodes from project: " + nodes); } for (PeerStatusJsonV2 peer : peersProjectsStatus.peer_to_projects().values()) { - log.debug("checking peer on host " + peer.host()); - if (null != peer.host()) { - if (nodes.contains(StringUtils.toLowerCase(peer.host()))) { + log.debug("checking peer on host " + peer.canonical_host()); + if (null != peer.canonical_host()) { + if (nodes.contains(StringUtils.toLowerCase(peer.canonical_host()))) { peers.add(peer); } } From 78a46ef8df1c301265d07a82d09df11f8efeee27 Mon Sep 17 00:00:00 2001 From: rezra3 Date: Tue, 5 Jun 2018 10:01:34 +0300 Subject: [PATCH 2/2] codeine-239 peer should not change address during up time --- src/common/codeine/api/NodeWithPeerInfo.java | 4 ++-- src/common/codeine/jsons/peer_status/PeerStatusJsonV2.java | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/common/codeine/api/NodeWithPeerInfo.java b/src/common/codeine/api/NodeWithPeerInfo.java index 399284c5..e1364739 100755 --- a/src/common/codeine/api/NodeWithPeerInfo.java +++ b/src/common/codeine/api/NodeWithPeerInfo.java @@ -15,7 +15,7 @@ public NodeWithPeerInfo(String name, String alias, PeerStatusJsonV2 peer) { super(name, alias); this.peer = peer; if (null != peer) { - peer_host_port = peer.host_port(); + peer_host_port = peer.canonical_host_port(); peer_address = peer.address_port(); peer_key = peer.key(); peer_status = peer.status(); @@ -32,7 +32,7 @@ public PeerStatusJsonV2 peer() { public void peer(PeerStatusJsonV2 peer) { this.peer = peer; - peer_host_port = peer.host_port(); + peer_host_port = peer.canonical_host_port(); peer_address = peer.address_port(); peer_key = peer.key(); peer_status = peer.status(); diff --git a/src/common/codeine/jsons/peer_status/PeerStatusJsonV2.java b/src/common/codeine/jsons/peer_status/PeerStatusJsonV2.java index 734e333c..d7be1dac 100755 --- a/src/common/codeine/jsons/peer_status/PeerStatusJsonV2.java +++ b/src/common/codeine/jsons/peer_status/PeerStatusJsonV2.java @@ -88,10 +88,6 @@ public String peer_key() { return peer_key; } - public String host_port() { - return canonical_host + ":" + port; - } - public String canonical_host_port() { return canonical_host + ":" + port; } @@ -151,7 +147,7 @@ public String canonical_host() { @Override public String toString() { - return "PeerStatusJsonV2 [host_port()=" + host_port() + ", update_time()=" + new Date( + return "PeerStatusJsonV2 [host_port()=" + canonical_host() + ", update_time()=" + new Date( update_time()) + ", update_time_from_peer()=" + new Date(update_time_from_peer()) + ", peer_type()=" + peer_type() + "]";