From c91a91504c7846c59b2a87f6142b25d1c0ba8ddb Mon Sep 17 00:00:00 2001 From: Giulio Rossetti Date: Sat, 19 Aug 2017 12:35:28 +0200 Subject: [PATCH 1/6] Version 0.3.0 --- rdyn/__init__.py | 2 +- rdyn/__main__.py | 2 +- rdyn/alg/RDyn_v2.py | 528 +++++++++++++++++++++++++++++++++++++++++ rdyn/alg/__init__.py | 3 +- rdyn/test/rdyn_test.py | 19 +- requirements.txt | 3 +- setup.py | 4 +- 7 files changed, 545 insertions(+), 16 deletions(-) create mode 100644 rdyn/alg/RDyn_v2.py diff --git a/rdyn/__init__.py b/rdyn/__init__.py index dd0c360..5459ead 100644 --- a/rdyn/__init__.py +++ b/rdyn/__init__.py @@ -1 +1 @@ -from rdyn.alg.RDyn import RDyn \ No newline at end of file +from rdyn.alg.RDyn_v2 import RDynV2 as RDyn \ No newline at end of file diff --git a/rdyn/__main__.py b/rdyn/__main__.py index 380938f..c2b967a 100644 --- a/rdyn/__main__.py +++ b/rdyn/__main__.py @@ -1,3 +1,3 @@ -from rdyn.alg.RDyn import main +from rdyn.alg.RDyn_v2 import main main() diff --git a/rdyn/alg/RDyn_v2.py b/rdyn/alg/RDyn_v2.py new file mode 100644 index 0000000..e2adbe7 --- /dev/null +++ b/rdyn/alg/RDyn_v2.py @@ -0,0 +1,528 @@ +import os +import networkx as nx +import numpy as np +import random +import tqdm +import future.utils +import six +import sys + +__author__ = 'Giulio Rossetti' +__license__ = "GPL" +__email__ = "giulio.rossetti@gmail.com" +__version__ = "0.2.0" + + +class RDynV2(object): + + def __init__(self, size=1000, iterations=100, avg_deg=15, sigma=.6, + lambdad=1, alpha=2.5, paction=1, prenewal=.8, + quality_threshold=.2, new_node=.0, del_node=.0, max_evts=1): + + # set the network generator parameters + self.size = size + self.iterations = iterations + self.avg_deg = avg_deg + self.sigma = sigma + self.lambdad = lambdad + self.exponent = alpha + self.paction = paction + self.renewal = prenewal + self.new_node = new_node + self.del_node = del_node + self.max_evts = max_evts + + # event targets + self.communities_involved = [] + + # initialize communities data structures + self.communities = {} + self.node_to_com = [i for i in range(0, size)] + self.total_coms = 0 + self.performed_community_action = "START\n" + self.quality_threshold = quality_threshold + self.exp_node_degs = [] + + # initialize the graph + self.graph = nx.empty_graph(self.size) + + self.base = os.getcwd() + + # initialize output files + self.output_dir = "%s%sresults%s%s_%s_%s_%s_%s_%s_%s" % \ + (self.base, os.sep, os.sep, self.size, self.iterations, self.avg_deg, self.sigma, self.renewal, self.quality_threshold, self.max_evts) + + if not os.path.exists(self.output_dir): + os.makedirs(self.output_dir) + + self.out_interactions = open("%s%sinteractions.txt" % (self.output_dir, os.sep), "w") + self.out_events = open("%s%sevents.txt" % (self.output_dir, os.sep), "w") + self.stable = 0 + + self.it = 0 + self.count = 0 + + def execute(self, simplified=True): + """ + + :return: + """ + # generate degree sequence + self.__compute_degree_sequence() + + # generate community size dist + exp_com_s = self.__compute_community_size_distribution() + + # assign node to community + self.__node_to_community_initial_assignement(exp_com_s) + + # main loop (iteration) + for self.it in tqdm.tqdm(range(0, self.iterations), ncols=100): + + # community check and event generation + comp = nx.number_connected_components(self.graph) + if comp <= len(self.communities): + if self.__test_communities(): + self.__generate_event(simplified) + + # node removal + ar = random.random() + if ar < self.del_node: + self.__remove_node() + + # node addition + ar = random.random() + if ar < self.new_node: + self.__add_node() + + # get nodes within communities that needs to adjust + nodes = self.__get_nodes() + + # inner loop (nodes) + for n in nodes: + + # discard deleted nodes + if self.node_to_com[n] == -1: + continue + + # check for decayed edges + removal = self.__get_vanished_edges(n) + + # removal phase + for n1 in removal: + r = random.random() + + # edge renewal phase + # check for intra/inter renewal thresholds + if r <= self.renewal and self.node_to_com[n1] == self.node_to_com[n]\ + or r > self.renewal and self.node_to_com[n1] != self.node_to_com[n]: + + # Exponential decay + timeout = (self.it + 1) + int(random.expovariate(self.lambdad)) + self.graph.edge[n][n1]["d"] = timeout + + else: + # edge to be removed + self.out_interactions.write("%s\t%s\t-\t%s\t%s\n" % (self.it, self.count, n, n1)) + self.graph.remove_edge(n, n1) + + # expected degree reached + if self.graph.degree(n) >= self.exp_node_degs[n]: + continue + + # decide if the node is active during this iteration + action = random.random() + + # the node has not yet reached it expected degree and it acts in this round + if self.graph.degree([n])[n] < self.exp_node_degs[n] and (action <= self.paction or self.it == 0): + + com_nodes = list(self.communities[self.node_to_com[n]]) + + # probability for intra/inter community edges + r = random.random() + + # check if at least sigma% of the node link are within the community + s = self.graph.subgraph(com_nodes) + d = s.degree([n])[n] # Intra community edges + + if r <= self.sigma and d < len(com_nodes) - 1: + self.__new_intra_community_edge(s, n) + + # inter-community edges + elif r > self.sigma: + # if self.exp_node_degs[n]-d < (1-self.sigma) * s.number_of_nodes(): + self.__new_inter_community_edge(n) + + self.__output_communities() + self.out_events.write("%s\n\t%s\n" % (self.iterations, self.performed_community_action)) + self.out_interactions.flush() + self.out_interactions.close() + self.out_events.flush() + self.out_events.close() + return self.stable + + + def __new_intra_community_edge(self, s, n): + + n_neigh = set(s.neighbors(n)) + + random.shuffle(list(n_neigh)) + target = None + + # selecting target node + candidates = {j: (self.exp_node_degs[j] - self.graph.degree(j)) for j in s.nodes() + if (self.exp_node_degs[j] - self.graph.degree(j)) > 0 and j != n} + + if len(candidates) > 0: + target = random.sample(list(candidates), 1)[0] + + # Interaction Exponential decay + timeout = (self.it + 1) + int(random.expovariate(self.lambdad)) + + # Edge insertion + if target is not None and not self.graph.has_edge(n, target) and target != n: + self.graph.add_edge(n, target, {"d": timeout}) + self.count += 1 + self.out_interactions.write("%s\t%s\t+\t%s\t%s\n" % (self.it, self.count, n, target)) + + def __new_inter_community_edge(self, n): + # randomly identifying a target community + try: + cid = random.sample(set(self.communities.keys()) - {self.node_to_com[n]}, 1)[0] + except: + return + + s = self.graph.subgraph(self.communities[cid]) + + # check for available nodes within the identified community + candidates = {j: (self.exp_node_degs[j] - self.graph.degree(j)) for j in s.nodes() + if (self.exp_node_degs[j] - self.graph.degree(j)) > 0 and j != n} + + # PA selection on available community nodes + if len(candidates) > 0: + candidatesp = list(np.array(list(candidates.values()), dtype='float') / sum(list(candidates.values()))) + target = np.random.choice(list(candidates.keys()), 1, candidatesp)[0] + + if self.graph.has_node(target) and not self.graph.has_edge(n, target): + # Interaction exponential decay + timeout = (self.it + 1) + int(random.expovariate(self.lambdad)) + self.graph.add_edge(n, target, {"d": timeout}) + self.count += 1 + self.out_interactions.write("%s\t%s\t+\t%s\t%s\n" % (self.it, self.count, n, target)) + + def __compute_degree_sequence(self): + minv = float(self.avg_deg) / (2 ** (1 / (self.exponent - 1))) + s = [2] + while not nx.is_valid_degree_sequence(s): + s = list(map(int, nx.utils.powerlaw_sequence(self.graph.number_of_nodes(), self.exponent))) + x = [int(p + minv) for p in s] + self.exp_node_degs = sorted(x) + + def __compute_community_size_distribution(self): + min_node_degree = min(self.exp_node_degs) + min_size_com = int(min_node_degree * self.sigma) + s = list(map(int, nx.utils.powerlaw_sequence(int(self.graph.number_of_nodes()/self.avg_deg), self.exponent))) + sizes = sorted([p + min_size_com for p in s]) + while sum(sizes) > self.graph.number_of_nodes(): + for i, cs in enumerate(sizes): + sizes[i] = cs - 1 + + for c, _ in enumerate(sizes): + self.communities[c] = [] + return sizes + + def __node_to_community_initial_assignement(self, community_sizes): + degs = [(i, v) for i, v in enumerate(self.exp_node_degs)] + unassigned = [] + + for n in degs: + nid, nd = n + + assigned = False + for c, c_size in enumerate(community_sizes): + c_taken = len(self.communities[c]) + + # check if the node can be added to the community + if float(nd) * self.sigma <= c_size and c_taken < c_size: + self.communities[c].append(nid) + assigned = True + break + + if not assigned: + unassigned.append(n) + + if len(unassigned) > 0: + for i in unassigned: + for cid in self.communities: + self.communities[cid].append(i[0]) + community_sizes[cid] += 1 + self.exp_node_degs[i[0]] = community_sizes[cid] - 1 + break + + ntc = [] + for cid, nodes in future.utils.iteritems(self.communities): + for n in nodes: + ntc.append((n, cid)) + + for node_com in ntc: + self.node_to_com[node_com[0]] = node_com[1] + + def __test_communities(self): + mcond = 0 + for k in self.communities_involved: + c = self.communities[k] + if len(c) == 0: + return False + + s = self.graph.subgraph(c) + comps = nx.number_connected_components(s) + + if comps > 1: + cs = nx.connected_components(s) + i = random.sample(six.next(cs), 1)[0] + j = random.sample(six.next(cs), 1)[0] + timeout = (self.it + 1) + int(random.expovariate(self.lambdad)) + self.graph.add_edge(i, j, {"d": timeout}) + self.count += 1 + self.out_interactions.write("%s\t%s\t+\t%s\t%s\n" % (self.it, self.count, i, j)) + return False + + score = self.__conductance_test(k, s) + if score > mcond: + mcond = score + if mcond > self.quality_threshold: + return False + return True + + def __conductance_test(self, comid, community): + s_degs = community.degree() + g_degs = self.graph.degree(community.nodes()) + + # Conductance + edge_across = 2 * sum([g_degs[n] - s_degs[n] for n in community.nodes()]) + c_nodes_total_edges = community.number_of_edges() + (2 * edge_across) + + if edge_across > 0: + ratio = float(edge_across) / float(c_nodes_total_edges) + if ratio > self.quality_threshold: + self.communities_involved.append(comid) + self.communities_involved = list(set(self.communities_involved)) + + for i in community.nodes(): + nn = self.graph.neighbors(i) + for j in nn: + if j not in community.nodes(): + self.count += 1 + self.out_interactions.write("%s\t%s\t-\t%s\t%s\n" % (self.it, self.count, i, j)) + self.graph.remove_edge(i, j) + continue + return ratio + return 0 + + def __generate_event(self, simplified=True): + communities_involved = [] + self.stable += 1 + + options = ["M", "S"] + + evt_number = random.sample(range(1, self.max_evts+1), 1)[0] + evs = np.random.choice(options, evt_number, p=[.5, .5], replace=True) + chosen = [] + + if len(self.communities) == 1: + evs = "S" + + self.__output_communities() + if "START" in self.performed_community_action: + self.out_events.write("%s:\t%s" % (self.it, self.performed_community_action)) + else: + self.out_events.write("%s:\n%s" % (self.it, self.performed_community_action)) + + self.performed_community_action = "" + + for p in evs: + + if p == "M": + # Generate a single merge + if len(self.communities) == 1: + continue + candidates = list(set(self.communities.keys()) - set(chosen)) + + # promote merging of small communities + cl = [len(v) for c, v in future.utils.iteritems(self.communities) if c in candidates] + comd = 1-np.array(cl, dtype="float")/sum(cl) + comd /= sum(comd) + + ids = [] + try: + ids = np.random.choice(candidates, 2, p=list(comd), replace=False) + except: + continue + + # ids = random.sample(candidates, 2) + chosen.extend(ids) + communities_involved.extend([ids[0]]) + + for node in self.communities[ids[1]]: + self.node_to_com[node] = ids[0] + + self.performed_community_action = "%s MERGE\t%s\n" % (self.performed_community_action, ids) + + self.communities[ids[0]].extend(self.communities[ids[1]]) + del self.communities[ids[1]] + + else: + # Generate a single splits + if len(self.communities) == 1: + continue + + candidates = list(set(self.communities.keys()) - set(chosen)) + + cl = [len(v) for c, v in future.utils.iteritems(self.communities) if c in candidates] + comd = np.array(cl, dtype="float")/sum(cl) + + try: + ids = np.random.choice(candidates, 1, p=list(comd), replace=False) + except: + continue + + c_nodes = len(self.communities[ids[0]]) + + if c_nodes > 6: + try: + size = random.sample(range(3, c_nodes-3), 1)[0] + first = random.sample(self.communities[ids[0]], size) + except: + continue + + cid = max(list(self.communities.keys())) + 1 + chosen.extend([ids[0], cid]) + communities_involved.extend([ids[0], cid]) + + self.performed_community_action = "%s SPLIT\t%s\t%s\n" % \ + (self.performed_community_action, ids[0], [ids[0], cid]) + # adjusting max degree + for node in first: + self.node_to_com[node] = cid + if self.exp_node_degs[node] > (len(first)-1) * self.sigma: + self.exp_node_degs[node] = int((len(first)-1) + (len(first)-1) * (1-self.sigma)) + + self.communities[cid] = first + self.communities[ids[0]] = [ci for ci in self.communities[ids[0]] if ci not in first] + + # adjusting max degree + for node in self.communities[ids[0]]: + if self.exp_node_degs[node] > (len(self.communities[ids[0]])-1) * self.sigma: + self.exp_node_degs[node] = int((len(self.communities[ids[0]])-1) + + (len(self.communities[ids[0]])-1) * (1-self.sigma)) + + self.out_events.flush() + if not simplified: + self.communities_involved = list(self.communities.keys()) + else: + self.communities_involved = communities_involved + + + def __add_node(self): + nid = self.size + self.graph.add_node(nid) + cid = random.sample(list(self.communities.keys()), 1)[0] + self.communities[cid].append(nid) + self.node_to_com.append(cid) + deg = random.sample(range(2, int((len(self.communities[cid])-1) + + (len(self.communities[cid])-1)*(1-self.sigma))), 1)[0] + if deg == 0: + deg = 1 + self.exp_node_degs.append(deg) + self.size += 1 + + def __remove_node(self): + + com_sel = [c for c, v in future.utils.iteritems(self.communities) if len(v) > 3] + if len(com_sel) > 0: + cid = random.sample(com_sel, 1)[0] + s = self.graph.subgraph(self.communities[cid]) + min_value = min(s.degree().values()) + candidates = [k for k in s.degree() if s.degree()[k] == min_value] + nid = random.sample(candidates, 1)[0] + for e in self.graph.edges([nid]): + self.count += 1 + self.out_interactions.write("%s\t%s\t-\t%s\t%s\n" % (self.it, self.count, e[0], e[1])) + self.graph.remove_edge(e[0], e[1]) + + self.exp_node_degs[nid] = 0 + self.node_to_com[nid] = -1 + nodes = set(self.communities[cid]) + self.communities[cid] = list(nodes - {nid}) + self.graph.remove_node(nid) + + def __get_nodes(self): + if len(self.communities_involved) == 0: + return self.graph.nodes() + else: + nodes = {} + for cid in self.communities_involved: + for nid in self.communities[cid]: + nodes[nid] = None + return list(nodes.keys()) + + def __get_vanished_edges(self, n): + removal = [] + node_neighbors = nx.all_neighbors(self.graph, n) + if len(self.communities) >= nx.number_connected_components(self.graph): + for n1 in node_neighbors: + delay = self.graph.get_edge_data(n, n1)['d'] + if delay == self.it: + removal.append(n1) + return removal + + def __output_communities(self): + + self.total_coms = len(self.communities) + out = open("%s%scommunities-%s.txt" % (self.output_dir, os.sep, self.it), "w") + for c, v in future.utils.iteritems(self.communities): + out.write("%s\t%s\n" % (c, v)) + out.flush() + out.close() + + outg = open("%s%sgraph-%s.txt" % (self.output_dir, os.sep, self.it), "w") + for e in self.graph.edges(): + outg.write("%s\t%s\n" % (e[0], e[1])) + outg.flush() + outg.close() + + +def main(): + import argparse + + sys.stdout.write("-------------------------------------\n") + sys.stdout.write(" {RDyn} \n") + sys.stdout.write(" Graph Generator \n") + sys.stdout.write(" Handling Community Dynamics \n") + sys.stdout.write("-------------------------------------\n") + sys.stdout.write("Author: " + __author__ + "\n") + sys.stdout.write("Email: " + __email__ + "\n") + sys.stdout.write("------------------------------------\n") + + parser = argparse.ArgumentParser() + parser.add_argument('nodes', type=int, help='Number of nodes', default=1000) + parser.add_argument('iterations', type=int, help='Number of iterations', default=1000) + parser.add_argument('simplified', type=bool, help='Simplified execution', default=True) + parser.add_argument('-d', '--avg_degree', type=int, help='Average node degree', default=15) + parser.add_argument('-s', '--sigma', type=float, help='Sigma', default=0.7) + parser.add_argument('-l', '--lbd', type=float, help='Lambda community size distribution', default=1) + parser.add_argument('-a', '--alpha', type=int, help='Alpha degree distribution', default=2.5) + parser.add_argument('-p', '--prob_action', type=float, help='Probability of node action', default=1) + parser.add_argument('-r', '--prob_renewal', type=float, help='Probability of edge renewal', default=0.8) + parser.add_argument('-q', '--quality_threshold', type=float, help='Conductance quality threshold', default=0.3) + parser.add_argument('-n', '--new_nodes', type=float, help='Probability of node appearance', default=0) + parser.add_argument('-j', '--delete_nodes', type=float, help='Probability of node vanishing', default=0) + parser.add_argument('-e', '--max_events', type=int, help='Max number of community events for stable iteration', default=1) + + args = parser.parse_args() + rdyn = RDynV2(size=args.nodes, iterations=args.iterations, avg_deg=args.avg_degree, + sigma=args.sigma, lambdad=args.lbd, alpha=args.alpha, paction=args.prob_action, + prenewal=args.prob_renewal, quality_threshold=args.quality_threshold, + new_node=args.new_nodes, del_node=args.delete_nodes, max_evts=args.max_events) + rdyn.execute(simplified=args.simplified) + diff --git a/rdyn/alg/__init__.py b/rdyn/alg/__init__.py index 416a718..2d35146 100644 --- a/rdyn/alg/__init__.py +++ b/rdyn/alg/__init__.py @@ -1 +1,2 @@ -from .RDyn import RDyn \ No newline at end of file +from .RDyn import RDyn as RDynV1 +from .RDyn_v2 import RDynV2 as RDyn \ No newline at end of file diff --git a/rdyn/test/rdyn_test.py b/rdyn/test/rdyn_test.py index 1f295d8..62830a8 100644 --- a/rdyn/test/rdyn_test.py +++ b/rdyn/test/rdyn_test.py @@ -1,23 +1,22 @@ import unittest import shutil -from rdyn.alg.RDyn import RDyn +from rdyn.alg.RDyn_v2 import RDynV2 class RDynTestCase(unittest.TestCase): def test_rdyn_simplified(self): - rdb = RDyn(size=1000, iterations=100) + print("1") + rdb = RDynV2(size=500, iterations=100) rdb.execute(simplified=True) - - rdb = RDyn(size=100, iterations=10) - try: - rdb.execute(simplified=True) - except: - pass - - rdb = RDyn(size=1000, iterations=100, new_node=0.1, del_node=0.1, max_evts=2, paction=0.8) + print("2") + rdb = RDynV2(size=500, iterations=100, max_evts=2) + rdb.execute(simplified=True) + print("3") + rdb = RDynV2(size=500, iterations=100, new_node=0.1, del_node=0.1, max_evts=2, paction=0.8) rdb.execute(simplified=False) + print("Done") shutil.rmtree("results") diff --git a/requirements.txt b/requirements.txt index 7668ec8..af7941a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ future networkx==1.11 scipy==0.17.0 -tqdm \ No newline at end of file +tqdm +six \ No newline at end of file diff --git a/setup.py b/setup.py index f9f79b2..a0352b0 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ # long_description = f.read() setup(name='rdyn', - version='0.2.1', + version='0.3.0', license='BSD-2-Clause', description='Graph benchmark handling community dynamics', url='https://github.com/GiulioRossetti/RDyn', @@ -36,7 +36,7 @@ # that you indicate whether you support Python 2, Python 3 or both. 'Programming Language :: Python', 'Programming Language :: Python :: 2.7', - # 'Programming Language :: Python :: 3' + 'Programming Language :: Python :: 3' ], keywords=['complex-networks', 'network generator', 'dynamic networks', 'community dynamics'], install_requires=['networkx', 'future', ''], From 0406b9a8c7ff4badba072ff064f8982baad66900 Mon Sep 17 00:00:00 2001 From: Giulio Rossetti Date: Sat, 19 Aug 2017 12:40:22 +0200 Subject: [PATCH 2/6] Coverage fix --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 9131f58..6cd8535 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,6 +3,7 @@ omit = setup.py rdyn/__main__.py rdyn/test/* + rdyn/alg/RDyn.py [report] exclude_lines = From 0e940406f11e0c6480513ee05540b94b865271cc Mon Sep 17 00:00:00 2001 From: Giulio Rossetti Date: Sat, 19 Aug 2017 12:41:01 +0200 Subject: [PATCH 3/6] Travis update --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 11cafe3..8459201 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,9 @@ language: python python: - "2.7" + - "3.4" + - "3.5" + - "3.6" before_install: - pip install pytest pytest-cov From 4f2469f7e952ee3c8b09c7d8c6c6b277c12a8411 Mon Sep 17 00:00:00 2001 From: Giulio Rossetti Date: Sat, 19 Aug 2017 12:46:47 +0200 Subject: [PATCH 4/6] requirements update --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index af7941a..b7f6d7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ future networkx==1.11 -scipy==0.17.0 tqdm six \ No newline at end of file From 9d47ef4a574657b94c2999cc0cb1c03f7a548ab3 Mon Sep 17 00:00:00 2001 From: Giulio Rossetti Date: Sat, 19 Aug 2017 12:48:55 +0200 Subject: [PATCH 5/6] import bug fix --- rdyn/alg/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdyn/alg/__init__.py b/rdyn/alg/__init__.py index 2d35146..560ba7a 100644 --- a/rdyn/alg/__init__.py +++ b/rdyn/alg/__init__.py @@ -1,2 +1,2 @@ -from .RDyn import RDyn as RDynV1 +# from .RDyn import RDyn as RDynV1 from .RDyn_v2 import RDynV2 as RDyn \ No newline at end of file From b72221beb719716148d2f9d706aba3fc340c263e Mon Sep 17 00:00:00 2001 From: Giulio Rossetti Date: Sat, 19 Aug 2017 13:02:44 +0200 Subject: [PATCH 6/6] remove old version --- README.md | 4 +- rdyn/alg/RDyn.py | 576 ------------------------------------------- rdyn/alg/__init__.py | 1 - 3 files changed, 2 insertions(+), 579 deletions(-) delete mode 100644 rdyn/alg/RDyn.py diff --git a/README.md b/README.md index 6e3e542..694e539 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,6 @@ In order to generate a dynamic graph of 1000 nodes for 1000 iterations applying ```bash python rdyn 1000 1000 True ``` -The minimum number of nodes allowed is 1000. ## As python library @@ -149,5 +148,6 @@ RDyn is written in python and requires the following package to run: - python>=2.7.11 - networkx==1.11 - numpy==1.11.1 -- scipy==0.17.0 - tqdm +- six +- future diff --git a/rdyn/alg/RDyn.py b/rdyn/alg/RDyn.py deleted file mode 100644 index 76c1f6c..0000000 --- a/rdyn/alg/RDyn.py +++ /dev/null @@ -1,576 +0,0 @@ -import os -import networkx as nx -import math -import numpy as np -import random -import scipy.stats as stats -import tqdm -import sys -import past.builtins -import future.utils - -__author__ = 'Giulio Rossetti' -__license__ = "GPL" -__email__ = "giulio.rossetti@gmail.com" -__version__ = "0.2.0" - - -class RDyn(object): - - def __init__(self, size=1000, iterations=1000, avg_deg=15, sigma=.7, - lambdad=1, alpha=3, paction=1, prenewal=.8, - quality_threshold=.3, new_node=.0, del_node=.0, max_evts=1): - - # set the network generator parameters - self.size = size - self.iterations = iterations - self.avg_deg = avg_deg - self.sigma = sigma - self.lambdad = lambdad - self.exponent = alpha - self.paction = paction - self.renewal = prenewal - self.new_node = new_node - self.del_node = del_node - self.max_evts = max_evts - - # event targets - self.communities_involved = [] - - # initialize communities data structures - self.communities = {} - self.node_to_com = [] - self.total_coms = 0 - self.performed_community_action = "START\n" - self.quality_threshold = quality_threshold - self.exp_node_degs = [] - - # initialize the graph - self.graph = nx.empty_graph(self.size) - - self.base = os.getcwd() - - # initialize output files - self.output_dir = "%s%sresults%s%s_%s_%s_%s_%s_%s_%s" % \ - (self.base, os.sep, os.sep, self.size, self.iterations, self.avg_deg, self.sigma, self.renewal, self.quality_threshold, self.max_evts) - - if not os.path.exists(self.output_dir): - os.makedirs(self.output_dir) - - self.out_interactions = open("%s%sinteractions.txt" % (self.output_dir, os.sep), "w") - self.out_events = open("%s%sevents.txt" % (self.output_dir, os.sep), "w") - self.stable = 0 - - self.it = 0 - self.count = 0 - - def __get_assignation(self, community_sizes): - - degs = [(i, self.exp_node_degs[i]) for i in past.builtins.xrange(0, len(self.exp_node_degs))] - - for c in past.builtins.xrange(0, len(community_sizes)): - self.communities[c] = [] - - unassigned = [] - - for n in degs: - nid, nd = n - assigned = False - for c in past.builtins.xrange(0, len(community_sizes)): - c_size = community_sizes[c] - c_taken = len(self.communities[c]) - if c_size/float(nd) >= self.sigma and c_taken < c_size: - self.communities[c].append(nid) - assigned = True - break - if not assigned: - unassigned.append(n) - - slots_available = [(k, (community_sizes[k] - len(self.communities[k]))) for k in past.builtins.xrange(0, len(community_sizes)) - if (community_sizes[k] - len(self.communities[k])) > 0] - - if len(unassigned) > 0: - for i in unassigned: - for c in past.builtins.xrange(0, len(slots_available)): - cid, av = slots_available[c] - if av > 0: - self.communities[cid].append(i[0]) - self.exp_node_degs[i[0]] = community_sizes[cid] - 1 - slots_available[c] = (cid, av-1) - break - - ntc = {} - for cid, nodes in future.utils.iteritems(self.communities): - for n in nodes: - ntc[n] = cid - - nodes = ntc.keys() - nodes.sort() - - for n in nodes: - self.node_to_com.append(ntc[n]) - - @staticmethod - def __truncated_power_law(alpha, maxv, minv=1): - """ - - :param maxv: - :param minv: - :param alpha: - :return: - :rtype: object - """ - x = np.arange(1, maxv + 1, dtype='float') - pmf = 1 / x ** alpha - pmf /= pmf.sum() - ds = stats.rv_discrete(values=(range(minv, maxv + 1), pmf)) - - return ds - - def __add_node(self): - nid = self.size - self.graph.add_node(nid) - cid = random.sample(self.communities.keys(), 1)[0] - self.communities[cid].append(nid) - self.node_to_com.append(cid) - deg = random.sample(range(2, int((len(self.communities[cid])-1) + - (len(self.communities[cid])-1)*(1-self.sigma))), 1)[0] - if deg == 0: - deg = 1 - self.exp_node_degs.append(deg) - self.size += 1 - - def __remove_node(self): - - com_sel = [c for c, v in future.utils.iteritems(self.communities) if len(v) > 3] - if len(com_sel) > 0: - cid = random.sample(com_sel, 1)[0] - s = self.graph.subgraph(self.communities[cid]) - min_value = min(s.degree().itervalues()) - candidates = [k for k in s.degree() if s.degree()[k] == min_value] - nid = random.sample(candidates, 1)[0] - for e in self.graph.edges([nid]): - self.count += 1 - self.out_interactions.write("%s\t%s\t-\t%s\t%s\n" % (self.it, self.count, e[0], e[1])) - self.graph.remove_edge(e[0], e[1]) - - self.exp_node_degs[nid] = 0 - self.node_to_com[nid] = -1 - nodes = set(self.communities[cid]) - self.communities[cid] = list(nodes - {nid}) - self.graph.remove_node(nid) - - def __get_degree_sequence(self): - """ - - :return: - - :rtype: object - """ - minx = float(self.avg_deg) / (2 ** (1 / (self.exponent - 1))) - - while True: - exp_deg_dist = self.__truncated_power_law(self.exponent, self.size, int(math.ceil(minx))) - degs = list(exp_deg_dist.rvs(size=self.size)) - - if nx.is_valid_degree_sequence(degs): - return degs, int(minx) - - def __test_communities(self): - mcond = 0 - for k in self.communities_involved: - c = self.communities[k] - if len(c) == 0: - return False - - s = self.graph.subgraph(c) - comps = nx.number_connected_components(s) - - if comps > 1: - cs = nx.connected_components(s) - i = random.sample(cs.next(), 1)[0] - j = random.sample(cs.next(), 1)[0] - timeout = (self.it + 1) + int(random.expovariate(self.lambdad)) - self.graph.add_edge(i, j, {"d": timeout}) - self.count += 1 - self.out_interactions.write("%s\t%s\t+\t%s\t%s\n" % (self.it, self.count, i, j)) - return False - - score = self.__conductance_test(k, s) - if score > mcond: - mcond = score - if mcond > self.quality_threshold: - return False - - return True - - def __conductance_test(self, comid, community): - s_degs = community.degree() - g_degs = self.graph.degree(community.nodes()) - - # Conductance - edge_across = 2 * sum([g_degs[n] - s_degs[n] for n in community.nodes()]) - c_nodes_total_edges = community.number_of_edges() + (2 * edge_across) - - if edge_across > 0: - ratio = float(edge_across) / float(c_nodes_total_edges) - if ratio > self.quality_threshold: - self.communities_involved.append(comid) - self.communities_involved = list(set(self.communities_involved)) - - for i in community.nodes(): - nn = self.graph.neighbors(i) - for j in nn: - if j not in community.nodes(): - self.count += 1 - self.out_interactions.write("%s\t%s\t-\t%s\t%s\n" % (self.it, self.count, i, j)) - self.graph.remove_edge(i, j) - continue - return ratio - return 0 - - def __generate_event(self, simplified=True): - communities_involved = [] - self.stable += 1 - - options = ["M", "S"] - - evt_number = random.sample(range(1, self.max_evts+1), 1)[0] - evs = np.random.choice(options, evt_number, p=[.5, .5], replace=True) - chosen = [] - - if len(self.communities) == 1: - evs = "S" - - self.__output_communities() - if "START" in self.performed_community_action: - self.out_events.write("%s:\t%s" % (self.it, self.performed_community_action)) - else: - self.out_events.write("%s:\n%s" % (self.it, self.performed_community_action)) - - self.performed_community_action = "" - - for p in evs: - - if p == "M": - # Generate a single merge - if len(self.communities) == 1: - continue - candidates = list(set(self.communities.keys()) - set(chosen)) - - # promote merging of small communities - cl = [len(v) for c, v in future.utils.iteritems(self.communities) if c in candidates] - comd = 1-np.array(cl, dtype="float")/sum(cl) - comd /= sum(comd) - - ids = [] - try: - ids = np.random.choice(candidates, 2, p=list(comd), replace=False) - except: - continue - - # ids = random.sample(candidates, 2) - chosen.extend(ids) - communities_involved.extend([ids[0]]) - - for node in self.communities[ids[1]]: - self.node_to_com[node] = ids[0] - - self.performed_community_action = "%s MERGE\t%s\n" % (self.performed_community_action, ids) - - self.communities[ids[0]].extend(self.communities[ids[1]]) - del self.communities[ids[1]] - - else: - # Generate a single splits - if len(self.communities) == 1: - continue - - candidates = list(set(self.communities.keys()) - set(chosen)) - - cl = [len(v) for c, v in future.utils.iteritems(self.communities) if c in candidates] - comd = np.array(cl, dtype="float")/sum(cl) - - try: - ids = np.random.choice(candidates, 1, p=list(comd), replace=False) - except: - continue - - c_nodes = len(self.communities[ids[0]]) - if c_nodes > 6: - try: - size = random.sample(range(3, c_nodes-3), 1)[0] - first = random.sample(self.communities[ids[0]], size) - except: - continue - cid = max(self.communities.keys()) + 1 - chosen.extend([ids[0], cid]) - communities_involved.extend([ids[0], cid]) - - self.performed_community_action = "%s SPLIT\t%s\t%s\n" % \ - (self.performed_community_action, ids[0], [ids[0], cid]) - # adjusting max degree - for node in first: - self.node_to_com[node] = cid - if self.exp_node_degs[node] > (len(first)-1) * self.sigma: - self.exp_node_degs[node] = int((len(first)-1) + (len(first)-1) * (1-self.sigma)) - - self.communities[cid] = first - self.communities[ids[0]] = [ci for ci in self.communities[ids[0]] if ci not in first] - - # adjusting max degree - for node in self.communities[ids[0]]: - if self.exp_node_degs[node] > (len(self.communities[ids[0]])-1) * self.sigma: - self.exp_node_degs[node] = int((len(self.communities[ids[0]])-1) + - (len(self.communities[ids[0]])-1) * (1-self.sigma)) - - self.out_events.flush() - if not simplified: - communities_involved = self.communities.keys() - - return self.node_to_com, self.communities, communities_involved - - def __output_communities(self): - - self.total_coms = len(self.communities) - out = open("%s%scommunities-%s.txt" % (self.output_dir, os.sep, self.it), "w") - for c, v in future.utils.iteritems(self.communities): - out.write("%s\t%s\n" % (c, v)) - out.flush() - out.close() - - outg = open("%s%sgraph-%s.txt" % (self.output_dir, os.sep, self.it), "w") - for e in self.graph.edges(): - outg.write("%s\t%s\n" % (e[0], e[1])) - outg.flush() - outg.close() - - def __get_community_size_distribution(self, mins=3): - cv, nc = 0, 2 - cms = [] - - nc += 2 - com_s = self.__truncated_power_law(2, self.size/self.avg_deg, mins) - exp_com_s = com_s.rvs(size=self.size) - - # complete coverage - while cv <= 1: - - cms = random.sample(exp_com_s, nc) - cv = float(sum(cms)) / self.size - nc += 1 - - while True: - if sum(cms) <= self.size: - break - - for cm in past.builtins.xrange(-1, -len(cms), -1): - if sum(cms) <= self.size: - break - elif sum(cms) > self.size and cms[cm] > mins: - cms[cm] -= 1 - - return sorted(cms, reverse=True) - - def execute(self, simplified=True): - """ - - :return: - """ - if self.size < 1000: - print("Minimum network size: 1000 nodes") - exit(0) - - # generate pawerlaw degree sequence - self.exp_node_degs, mind = self.__get_degree_sequence() - - # generate community size dist - exp_com_s = self.__get_community_size_distribution(mins=mind+1) - - # assign node to community - self.__get_assignation(exp_com_s) - - self.total_coms = len(self.communities) - - # main loop (iteration) - for self.it in tqdm.tqdm(range(0, self.iterations), ncols=100): - - # community check and event generation - comp = nx.number_connected_components(self.graph) - if self.it > 2*self.avg_deg and comp <= len(self.communities): - if self.__test_communities(): - self.node_to_com, self.communities, self.communities_involved = self.__generate_event(simplified) - - # node removal - ar = random.random() - if ar < self.del_node: - self.__remove_node() - - # node addition - ar = random.random() - if ar < self.new_node: - self.__add_node() - - self.out_interactions.flush() - - # get nodes within selected communities - if len(self.communities_involved) == 0: - nodes = self.graph.nodes() - else: - nodes = [] - for ci in self.communities_involved: - nodes.extend(self.communities[ci]) - nodes = set(nodes) - random.shuffle(list(nodes)) - - # inner loop (nodes) - for n in nodes: - - # discard deleted nodes - if self.node_to_com[n] == -1: - continue - - # check for decayed edges - nn = nx.all_neighbors(self.graph, n) - - removal = [] - for n1 in nn: - delay = self.graph.get_edge_data(n, n1)['d'] - if delay == self.it: - removal.append(n1) - - # removal phase - for n1 in removal: - r = random.random() - - # edge renewal phase - # check for intra/inter renewal thresholds - if r <= self.renewal and self.node_to_com[n1] == self.node_to_com[n]\ - or r > self.renewal and self.node_to_com[n1] != self.node_to_com[n]: - - # Exponential decay - timeout = (self.it + 1) + int(random.expovariate(self.lambdad)) - self.graph.edge[n][n1]["d"] = timeout - - else: - # edge to be removed - self.out_interactions.write("%s\t%s\t-\t%s\t%s\n" % (self.it, self.count, n, n1)) - self.graph.remove_edge(n, n1) - - if self.graph.degree(n) >= self.exp_node_degs[n]: - continue - - # decide if the node is active during this iteration - action = random.random() - - # the node has not yet reached it expected degree and it acts in this round - if self.graph.degree(n) < self.exp_node_degs[n] and (action <= self.paction or self.it == 0): - - com_nodes = set(self.communities[self.node_to_com[n]]) - - # probability for intra/inter community edges - r = random.random() - - # check if at least sigma% of the node link are within the community - s = self.graph.subgraph(self.communities[self.node_to_com[n]]) - d = s.degree(n) - - # Intra-community edge - if d < len(com_nodes) - 1 and r <= self.sigma: - n_neigh = set(s.neighbors(n)) - - random.shuffle(list(n_neigh)) - target = None - - # selecting target node - candidates = {j: (self.exp_node_degs[j] - self.graph.degree(j)) for j in s.nodes() - if (self.exp_node_degs[j] - self.graph.degree(j)) > 0 and j != n} - - if len(candidates) > 0: - try: - target = random.sample(candidates, 1)[0] - except: - continue - - # Interaction Exponential decay - timeout = (self.it + 1) + int(random.expovariate(self.lambdad)) - - # Edge insertion - if target is not None and not self.graph.has_edge(n, target) and target != n: - self.graph.add_edge(n, target, {"d": timeout}) - self.count += 1 - self.out_interactions.write("%s\t%s\t+\t%s\t%s\n" % (self.it, self.count, n, target)) - else: - continue - - # inter-community edges - elif r > self.sigma and \ - self.exp_node_degs[n]-d < (1-self.sigma) * len(s.nodes()): - - # randomly identifying a target community - try: - cid = random.sample(set(self.communities.keys()) - {self.node_to_com[n]}, 1)[0] - except: - continue - - s = self.graph.subgraph(self.communities[cid]) - - # check for available nodes within the identified community - candidates = {j: (self.exp_node_degs[j] - self.graph.degree(j)) for j in s.nodes() - if (self.exp_node_degs[j] - self.graph.degree(j)) > 0 and j != n} - - # PA selection on available community nodes - if len(candidates) > 0: - candidatesp = list(np.array(candidates.values(), dtype='float') / sum(candidates.values())) - target = np.random.choice(candidates.keys(), 1, candidatesp)[0] - - if self.graph.has_node(target) and not self.graph.has_edge(n, target): - - # Interaction exponential decay - timeout = (self.it + 1) + int(random.expovariate(self.lambdad)) - self.graph.add_edge(n, target, {"d": timeout}) - self.count += 1 - self.out_interactions.write("%s\t%s\t+\t%s\t%s\n" % (self.it, self.count, n, target)) - - self.__output_communities() - self.out_events.write("%s\n\t%s\n" % (self.iterations, self.performed_community_action)) - self.out_interactions.flush() - self.out_interactions.close() - self.out_events.flush() - self.out_events.close() - return self.stable - - -def main(): - import argparse - - sys.stdout.write("-------------------------------------\n") - sys.stdout.write(" {RDyn} \n") - sys.stdout.write(" Graph Generator \n") - sys.stdout.write(" Handling Community Dynamics \n") - sys.stdout.write("-------------------------------------\n") - sys.stdout.write("Author: " + __author__ + "\n") - sys.stdout.write("Email: " + __email__ + "\n") - sys.stdout.write("------------------------------------\n") - - parser = argparse.ArgumentParser() - parser.add_argument('nodes', type=int, help='Number of nodes', default=1000) - parser.add_argument('iterations', type=int, help='Number of iterations', default=1000) - parser.add_argument('simplified', type=bool, help='Simplified execution', default=True) - parser.add_argument('-d', '--avg_degree', type=int, help='Average node degree', default=15) - parser.add_argument('-s', '--sigma', type=float, help='Sigma', default=0.7) - parser.add_argument('-l', '--lbd', type=float, help='Lambda community size distribution', default=1) - parser.add_argument('-a', '--alpha', type=int, help='Alpha degree distribution', default=2.5) - parser.add_argument('-p', '--prob_action', type=float, help='Probability of node action', default=1) - parser.add_argument('-r', '--prob_renewal', type=float, help='Probability of edge renewal', default=0.8) - parser.add_argument('-q', '--quality_threshold', type=float, help='Conductance quality threshold', default=0.3) - parser.add_argument('-n', '--new_nodes', type=float, help='Probability of node appearance', default=0) - parser.add_argument('-j', '--delete_nodes', type=float, help='Probability of node vanishing', default=0) - parser.add_argument('-e', '--max_events', type=int, help='Max number of community events for stable iteration', default=1) - - args = parser.parse_args() - rdyn = RDyn(size=args.nodes, iterations=args.iterations, avg_deg=args.avg_degree, - sigma=args.sigma, lambdad=args.lbd, alpha=args.alpha, paction=args.prob_action, - prenewal=args.prob_renewal, quality_threshold=args.quality_threshold, - new_node=args.new_nodes, del_node=args.delete_nodes, max_evts=args.max_events) - rdyn.execute(simplified=args.simplified) diff --git a/rdyn/alg/__init__.py b/rdyn/alg/__init__.py index 560ba7a..435a1c1 100644 --- a/rdyn/alg/__init__.py +++ b/rdyn/alg/__init__.py @@ -1,2 +1 @@ -# from .RDyn import RDyn as RDynV1 from .RDyn_v2 import RDynV2 as RDyn \ No newline at end of file