merge

.gitignore

@@ -1,7 +1,10 @@
+reproduce/
+reproduce
+data/
 Makefile
 .DS_Store
 .vscode/
-data/
+*.pkl
 results/
 
 #python

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Dai, Hanjun and Khalil, Elias B and Zhang, Yuyu and Dilkina, Bistra and Song, Le
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,5 +1,3 @@
-## **** Under Construction *****
-
 # graph_comb_opt 
 Implementation of "Learning Combinatorial Optimization Algorithms over Graphs" (https://arxiv.org/abs/1704.01665)
 
@@ -64,6 +62,21 @@ The above script will load the 1000 test graphs you generated before, and output
 
     Here the second column shows a solution found by S2V-DQN, in the same order of how each node is picked. 
 
+# 2. Experiments on real-world data
+
+For TSP we test on part of the tsplib instances;
+For MVC and SCP, we use memetracker dataset; 
+For MAXCUT, we test on optsicom dataset; 
+
+All the data can be found through the dropbox link below. Code folders that start with 'realworld' are for this set of experiments. 
+
+# Reproducing the results that reported in the paper
+
+Here is the link to the dataset that was used in the paper:
+
+https://www.dropbox.com/sh/r39596h8e26nhsp/AADRm5mb82xn7h3BB4KXgETsa?dl=0
+
+
 # Reference
 
 Please cite our work if you find our code/paper is useful to your work. 
@@ -74,26 +87,3 @@ Please cite our work if you find our code/paper is useful to your work.
       journal={arXiv preprint arXiv:1704.01665},
       year={2017}
     }
-
-## Check list
-1. S2V-DQN on synthetic data
-* **Minimum Vertex Cover (done)**
-* Set Cover (coming soon)
-* **Maxcut (done)**
-* 2D TSP (coming soon)
-
-2. Synthetic data generator
-* **Minimum Vertex Cover (done)**
-* Set Cover (coming soon)
-* **Maxcut (done)**
-* 2D TSP (coming soon)
-
-3. S2V-DQN on real-world data
-* MemeTracker MVC (coming soon)
-* MemeTracker Set Cover (coming soon)
-* Optsicom Maxcut (coming soon)
-* TSPLIB 2D TSP (coming soon)
-
-4. data generator for real-world data
-* MemeTracker MVC (coming soon)
-* MemeTracker Set Cover (coming soon)

code/data_generator/scp/gen_graph_only.py

@@ -0,0 +1,66 @@
+import os
+import sys
+import cPickle as cp
+import random
+import numpy as np
+import networkx as nx
+# from tqdm import tqdm
+
+def gen_setcover_inst(opt):
+    min_n = int(opt['min_n'])
+    max_n = int(opt['max_n'])
+    frac_primal = float(opt['frac_primal'])
+    p = float(opt['edge_prob'])
+
+    cur_n = np.random.randint(max_n - min_n + 1) + min_n
+    num_primal = int(cur_n * frac_primal)
+    num_dual = cur_n - num_primal
+
+    a = range(num_primal)
+    b = range(num_primal, num_dual + num_primal)
+
+    g = nx.Graph()
+    g.add_nodes_from(a, bipartite=0)
+    g.add_nodes_from(b, bipartite=1)
+
+    colHasOneBool = [0]*num_primal
+
+    for i in range(num_dual):
+        # guarantee that each element is in at least 2 sets, based on http://link.springer.com/chapter/10.1007%2FBFb0120886#page-1
+        k1 = np.random.randint(num_primal)
+        g.add_edge(k1, i + num_primal)
+        k2 = np.random.randint(num_primal)
+        g.add_edge(k2, i + num_primal)
+        for j in range(num_primal):
+            if j == k1 or j == k2:
+                continue
+            r = np.random.rand()
+            if r < p:
+                g.add_edge(j, i + num_primal)
+                colHasOneBool[j] = 1
+
+    # guarantee that each set has at least 1 element, based on http://link.springer.com/chapter/10.1007%2FBFb0120886#page-1
+    for j in range(num_primal):
+        if colHasOneBool[j] == 0:
+            randrow = np.random.randint(num_dual)
+            g.add_edge(j, randrow + num_primal)
+
+    return g
+
+if __name__ == '__main__':
+    opt = {}
+    for i in range(1, len(sys.argv), 2):
+        opt[sys.argv[i][1:]] = sys.argv[i + 1]
+
+    num_graph = int(opt['num_graph'])
+
+    if 'seed' not in opt:
+        seed = 1
+    else:
+        seed = int(opt['seed'])
+    np.random.seed(seed=seed)
+
+    with open('%s/nrange-%s-%s-n_graph-%s-p-%s-frac_primal-%s-seed-%s.pkl' % (opt['save_dir'], opt['min_n'], opt['max_n'], opt['num_graph'], opt['edge_prob'], opt['frac_primal'], seed), 'wb') as fout:
+        for i in range(num_graph):
+            g = gen_setcover_inst(opt)
+            cp.dump(g, fout, cp.HIGHEST_PROTOCOL)

code/data_generator/scp/run_generate.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+min_n=15
+max_n=20
+ep=0.05
+f=0.2
+output_root=../../../data/scp
+
+if [ ! -e $output_root ];
+then
+    mkdir -p $output_root
+fi
+
+python gen_graph_only.py \
+    -save_dir $output_root \
+    -max_n $max_n \
+    -min_n $min_n \
+    -num_graph 1000 \
+    -edge_prob 0.05 \
+    -frac_primal 0.2

code/memetracker/meme.py

@@ -0,0 +1,198 @@
+import numpy as np
+import networkx as nx
+
+import os
+import random
+import sys
+
+def build_full_graph(pathtofile, graphtype):
+	node_dict = {}
+
+	if graphtype == 'undirected':
+		g = nx.Graph()
+	elif graphtype == 'directed':
+		g = nx.DiGraph()
+	else:
+		print('Unrecognized graph type .. aborting!')
+		return -1
+
+	times = []
+	with open(pathtofile) as f:
+		content = f.readlines()
+	content = [x.strip() for x in content]
+	content = content[1:]
+
+	for line in content:
+		entries = line.split()
+		src_str = entries[1]
+		dst_str = entries[2]
+
+		if src_str not in node_dict:
+			node_dict[src_str] = len(node_dict)
+			g.add_node(node_dict[src_str])
+		if dst_str not in node_dict:
+			node_dict[dst_str] = len(node_dict)
+			g.add_node(node_dict[dst_str])
+
+		src_idx = node_dict[src_str]
+		dst_idx = node_dict[dst_str]
+
+		w = 0
+		c = 0
+		if g.has_edge(src_idx,dst_idx):
+			w = g[src_idx][dst_idx]['weight']
+			c = g[src_idx][dst_idx]['count']
+
+		g.add_edge(src_idx,dst_idx,weight=w + 1.0/float(entries[-1]),count=c + 1)
+
+		times.append(float(entries[-1]))
+
+	for edge in g.edges_iter(data=True):
+		src_idx = edge[0]
+		dst_idx = edge[1]
+		w = edge[2]['weight']
+		c = edge[2]['count']
+		g[src_idx][dst_idx]['weight'] = w/c
+
+	return g, node_dict
+
+def get_mvc_graph(ig,prob_quotient=10):
+	g = ig.copy()
+	# flip coin for each edge, remove it if coin has value > edge probability ('weight')
+	for edge in g.edges_iter(data=True):
+		src_idx = edge[0]
+		dst_idx = edge[1]
+		w = edge[2]['weight']
+		coin = random.random()
+		if coin > w/prob_quotient:
+			g.remove_edge(src_idx,dst_idx)
+
+	# get set of nodes in largest component
+	cc = sorted(nx.connected_components(g), key = len, reverse=True)
+	lcc = cc[0]
+
+	# remove all nodes not in largest component
+	numrealnodes = 0
+	node_map = {}
+	for node in g.nodes():
+		if node not in lcc:
+			g.remove_node(node)
+			continue
+		node_map[node] = numrealnodes
+		numrealnodes += 1
+
+	# re-create the largest component with nodes indexed from 0 sequentially
+	g2 = nx.Graph()
+	for edge in g.edges_iter(data=True):
+		src_idx = node_map[edge[0]]
+		dst_idx = node_map[edge[1]]
+		w = edge[2]['weight']
+		g2.add_edge(src_idx,dst_idx,weight=w)
+
+	return g2
+
+def get_scp_graph(ig,prob_quotient=10):
+	g = ig.copy()
+	# flip coin for each edge, remove it if coin has value > edge probability ('weight')
+	for edge in g.edges_iter(data=True):
+		src_idx = edge[0]
+		dst_idx = edge[1]
+		w = edge[2]['weight']
+		coin = random.random()
+		if coin > w/prob_quotient:
+			g.remove_edge(src_idx,dst_idx)
+
+	# remove nodes with in-degree and out-degree both 0
+	numrealnodes = 0
+	node_map = {}
+	for node in g.nodes():
+		if g.degree(node) == 0:
+			g.remove_node(node)
+			continue
+		node_map[node] = numrealnodes
+		numrealnodes += 1		
+
+	# re-index nodes from 0 sequentially
+	g2 = nx.DiGraph()
+	for edge in g.edges_iter(data=True):
+		src_idx = node_map[edge[0]]
+		dst_idx = node_map[edge[1]]
+		g2.add_edge(src_idx,dst_idx)
+
+	# get each node's reachable set of descendants; keep track of number of sets and elements
+	num_sets = 0
+	num_elements = 0#nx.number_of_nodes(g2)
+	set_map = {}
+	element_map = {}
+	node_desc = {}
+	for node in g2.nodes():
+		if g2.out_degree(node) > 0:
+			descendants = nx.descendants(g2,node)
+			node_desc[node] = descendants
+			set_map[node] = num_sets
+			num_sets += 1
+		if g2.in_degree(node) > 0:
+			element_map[node] = num_elements
+			num_elements += 1
+
+	# build bipartite graph
+	a = range(num_sets)
+	b = range(num_sets, num_sets + num_elements)
+	bg = nx.Graph()
+	bg.add_nodes_from(a, bipartite=0)
+	bg.add_nodes_from(b, bipartite=1)
+	for rset in node_desc:
+		src_idx = set_map[rset]
+		for desc in node_desc[rset]:
+			dst_idx = element_map[desc] + num_sets
+			bg.add_edge(src_idx,dst_idx)
+
+	# if element is in only one set, add it to another random set
+	for el in range(num_sets, num_sets + num_elements):
+		if bg.degree(el) == 1:
+			randset = np.random.randint(num_sets)
+			while bg.has_edge(el,randset):
+				randset = np.random.randint(num_sets)
+			bg.add_edge(el,randset)
+
+	# for s in range(num_sets):
+	# 	print(bg.degree(s))
+	# for el in range(num_sets, num_sets + num_elements):
+	# 	if bg.degree(el) <= 1:
+	# 		print('HERE')
+	# 		print(bg.degree(el))
+
+	return bg
+
+def visualize(g,pdfname='graph.pdf'):
+	import matplotlib.pyplot as plt
+	pos=nx.spring_layout(g,iterations=100) # positions for all nodes
+	nx.draw_networkx_nodes(g,pos,node_size=1)
+	nx.draw_networkx_edges(g,pos)
+	plt.axis('off')
+	plt.savefig(pdfname,bbox_inches="tight")
+
+def visualize_bipartite(g,pdfname='graph_scp.pdf'):
+	import matplotlib.pyplot as plt
+	X, Y = nx.bipartite.sets(g)
+	pos = dict()
+	pos.update( (n, (1, i)) for i, n in enumerate(X) ) # put nodes from X at x=1
+	pos.update( (n, (2, i)) for i, n in enumerate(Y) ) # put nodes from Y at x=2
+	nx.draw_networkx_nodes(g,pos,node_size=1)
+	nx.draw_networkx_edges(g,pos)
+	plt.axis('off')
+	plt.savefig(pdfname,bbox_inches="tight")
+
+if __name__ == '__main__':
+	# build full graphs of both types
+	print("Building undirected graph ...")
+	g_undirected, node_dict = build_full_graph('InfoNet5000Q1000NEXP.txt','undirected')
+	print("Building directed graph ...")
+	g_directed, node_dict = build_full_graph('InfoNet5000Q1000NEXP.txt','directed')
+
+	print(nx.number_of_nodes(g_undirected))
+	print(nx.number_of_edges(g_undirected))
+
+	print(nx.number_of_nodes(g_directed))
+	print(nx.number_of_edges(g_directed))
+