graph_viz.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import sqlite3 as lite\n",
    "\n",
    "import igraph\n",
    "import networkx as nx\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Grab data\n",
    "We really don't have that much data, so just load it all into memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with lite.connect('database.db') as con:\n",
    "    cur = con.cursor()\n",
    "    \n",
    "    cur.execute('select band_id,band from Bands where band_id in (select band_id from Similarities)')\n",
    "    bands_list = cur.fetchall()\n",
    "    \n",
    "    cur.execute('select band_id,similar_to_id,score from Similarities where similar_to_id in (select band_id from Similarities)')\n",
    "    sim_list = cur.fetchall()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_nodes = len(bands_list)\n",
    "num_edges = len(sim_list)\n",
    "print(num_nodes,num_edges)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "band_id_to_band = {band_id:band for band_id,band in bands_list}\n",
    "band_to_band_id = {band:band_id for band_id,band in bands_list}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(bands_list[0:10])\n",
    "print(sim_list[0:10])\n",
    "node_list = list(set(band_id for band_id,_,_ in sim_list))\n",
    "edge_list = [(band_id,similar_to_id,score) for band_id,similar_to_id,score in sim_list]\n",
    "print(edge_list[0:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Kamada-Kawai with `nx`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "G = nx.DiGraph()\n",
    "G.add_weighted_edges_from(edge_list[0:50])\n",
    "G = nx.relabel_nodes(G, band_id_to_band)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos = nx.kamada_kawai_layout(G)\n",
    "#pos = nx.nx_agraph.graphviz_layout(G)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 1, figsize=(16, 9));\n",
    "nx.draw_networkx(G, pos=pos)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Kamada-Kawai with `igraph`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "band_id_to_igraph_node = {band_id:i for i,band_id in enumerate(band_id_to_band.keys())}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "G = igraph.Graph()\n",
    "G.add_vertices(map(lambda band_id: band_id_to_igraph_node[band_id], node_list))\n",
    "G.add_edges(map(lambda t: (band_id_to_igraph_node[t[0]],band_id_to_igraph_node[t[1]]), edge_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(node_list[0:2])\n",
    "print(edge_list[0:2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## D3.js?\n",
    "Well, I didn't really give them a fair chance, but if it's not interactive...\n",
    "Let's try getting D3 inside the notebook.\n",
    "https://ipython-books.github.io/64-visualizing-a-networkx-graph-in-the-notebook-with-d3js/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "G = nx.DiGraph()\n",
    "G.add_weighted_edges_from(edge_list[:25000])\n",
    "G = nx.relabel_nodes(G, band_id_to_band)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/questions/8576737/how-to-only-keep-nodes-in-networkx-graph-with-2-outgoing-edges-or-0-outgoing-ed\n",
    "outdeg = G.out_degree()\n",
    "keep = [n for (n,deg) in outdeg if outdeg[n] > 1]\n",
    "G = G.subgraph(keep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fig, ax = plt.subplots(1, 1, figsize=(8, 6));\n",
    "#nx.draw_networkx(G, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(list(G.nodes())[0:10])\n",
    "nodes = [{'name': str(i)} for i in G.nodes()]\n",
    "links = [{'source': u[0], 'target': u[1]}\n",
    "         for u in G.edges()]\n",
    "with open('graph.json', 'w') as f:\n",
    "    json.dump({'nodes': nodes, 'links': links},\n",
    "              f, indent=4,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%html\n",
    "<div id=\"d3-example\"></div>\n",
    "<style>\n",
    ".node {stroke: #fff; stroke-width: 1.5px;}\n",
    ".link {stroke: #999; stroke-opacity: .6;}\n",
    "</style>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%javascript\n",
    "// We load the d3.js library from the Web.\n",
    "require.config({paths:\n",
    "    {d3: \"http://d3js.org/d3.v3.min\"}});\n",
    "require([\"d3\"], function(d3) {\n",
    "  // The code in this block is executed when the\n",
    "  // d3.js library has been loaded.\n",
    "\n",
    "  // First, we specify the size of the canvas\n",
    "  // containing the visualization (size of the\n",
    "  // <div> element).\n",
    "  var width = 300, height = 300;\n",
    "\n",
    "  // We create a color scale.\n",
    "  var color = d3.scale.category10();\n",
    "\n",
    "  // We create a force-directed dynamic graph layout.\n",
    "  var force = d3.layout.force()\n",
    "    .charge(-120)\n",
    "    .linkDistance(30)\n",
    "    .size([width, height]);\n",
    "\n",
    "  // In the <div> element, we create a <svg> graphic\n",
    "  // that will contain our interactive visualization.\n",
    "  var svg = d3.select(\"#d3-example\").select(\"svg\")\n",
    "  if (svg.empty()) {\n",
    "    svg = d3.select(\"#d3-example\").append(\"svg\")\n",
    "          .attr(\"width\", width)\n",
    "          .attr(\"height\", height);\n",
    "  }\n",
    "\n",
    "  // We load the JSON file.\n",
    "  d3.json(\"graph.json\", function(error, graph) {\n",
    "    // In this block, the file has been loaded\n",
    "    // and the 'graph' object contains our graph.\n",
    "\n",
    "    // We load the nodes and links in the\n",
    "    // force-directed graph.\n",
    "    force.nodes(graph.nodes)\n",
    "      .links(graph.links)\n",
    "      .start();\n",
    "\n",
    "    // We create a <line> SVG element for each link\n",
    "    // in the graph.\n",
    "    var link = svg.selectAll(\".link\")\n",
    "      .data(graph.links)\n",
    "      .enter().append(\"line\")\n",
    "      .attr(\"class\", \"link\");\n",
    "\n",
    "    // We create a <circle> SVG element for each node\n",
    "    // in the graph, and we specify a few attributes.\n",
    "    var node = svg.selectAll(\".node\")\n",
    "      .data(graph.nodes)\n",
    "      .enter().append(\"circle\")\n",
    "      .attr(\"class\", \"node\")\n",
    "      .attr(\"r\", 5)  // radius\n",
    "      .call(force.drag);\n",
    "\n",
    "    // The name of each node is the node number.\n",
    "    node.append(\"title\")\n",
    "        .text(function(d) { return d.name; });\n",
    "\n",
    "    // We bind the positions of the SVG elements\n",
    "    // to the positions of the dynamic force-directed\n",
    "    // graph, at each time step.\n",
    "    force.on(\"tick\", function() {\n",
    "      link.attr(\"x1\", function(d){return d.source.x})\n",
    "          .attr(\"y1\", function(d){return d.source.y})\n",
    "          .attr(\"x2\", function(d){return d.target.x})\n",
    "          .attr(\"y2\", function(d){return d.target.y});\n",
    "\n",
    "      node.attr(\"cx\", function(d){return d.x})\n",
    "          .attr(\"cy\", function(d){return d.y});\n",
    "    });\n",
    "  });\n",
    "});"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%html\n",
    "<meta charset=\"utf-8\">\n",
    "<style>\n",
    "\n",
    ".links line {\n",
    "  stroke: #999;\n",
    "  stroke-opacity: 0.6;\n",
    "}\n",
    "\n",
    ".nodes circle {\n",
    "  stroke: #fff;\n",
    "  stroke-width: 1.5px;\n",
    "}\n",
    "\n",
    "text {\n",
    "  font-family: sans-serif;\n",
    "  font-size: 10px;\n",
    "}\n",
    "\n",
    "</style>\n",
    "<svg width=\"960\" height=\"600\" id=\"d3ex2\"></svg>\n",
    "<script src=\"https://d3js.org/d3.v4.min.js\"></script>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%html\n",
    "<meta charset=\"utf-8\">\n",
    "<style>\n",
    "\n",
    ".links line {\n",
    "  stroke: #999;\n",
    "  stroke-opacity: 0.6;\n",
    "}\n",
    "\n",
    ".nodes circle {\n",
    "  stroke: #fff;\n",
    "  stroke-width: 1.5px;\n",
    "}\n",
    "\n",
    "text {\n",
    "  font-family: sans-serif;\n",
    "  font-size: 10px;\n",
    "}\n",
    "\n",
    "</style>\n",
    "<svg width=\"960\" height=\"600\" id=\"d3ex2\"></svg>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Okay, the graph is too big for direct visualization\n",
    "How about picking a band and showing the subgraph that's just k hops away from that band?\n",
    "Sounds like this is called the ego subgraph."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "band='Panopticon'\n",
    "\n",
    "G = nx.DiGraph()\n",
    "G.add_weighted_edges_from(edge_list)\n",
    "G = nx.relabel_nodes(G, band_id_to_band)\n",
    "\n",
    "# https://stackoverflow.com/questions/17301887/how-to-compute-nearby-nodes-with-networkx\n",
    "ego = nx.ego_graph(G, 'Panopticon', radius=2, center=True)\n",
    "\n",
    "len(ego.nodes())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fig, ax = plt.subplots(1, 1, figsize=(8, 6));\n",
    "#nx.draw_networkx(ego, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nodes = [{'name': str(i)} for i in ego.nodes()]\n",
    "links = [{'source': u[0], 'target': u[1]}\n",
    "         for u in ego.edges()]\n",
    "with open('ego.json', 'w') as f:\n",
    "    json.dump({'nodes': nodes, 'links': links},\n",
    "              f, indent=4,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%html\n",
    "<meta charset=\"utf-8\">\n",
    "<style>\n",
    "\n",
    ".links line {\n",
    "  stroke: #999;\n",
    "  stroke-opacity: 0.6;\n",
    "}\n",
    "\n",
    ".nodes circle {\n",
    "  stroke: #fff;\n",
    "  stroke-width: 1.5px;\n",
    "}\n",
    "\n",
    "text {\n",
    "  font-family: sans-serif;\n",
    "  font-size: 10px;\n",
    "}\n",
    "\n",
    "</style>\n",
    "<svg width=\"960\" height=\"600\" id=\"d3ego\"></svg>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%javascript\n",
    "require.config({paths:\n",
    "    {d3: \"http://d3js.org/d3.v4.min\"}});\n",
    "require([\"d3\"], function(d3) {\n",
    "\n",
    "var svg = d3.select(\"#d3ego\"),\n",
    "    width = +svg.attr(\"width\"),\n",
    "    height = +svg.attr(\"height\");\n",
    "\n",
    "//var color = d3.scaleOrdinal(d3.schemeCategory20);\n",
    "\n",
    "var simulation = d3.forceSimulation()\n",
    "    .force(\"link\", d3.forceLink().id(function(d) { return d.name; }))\n",
    "    .force(\"charge\", d3.forceManyBody())\n",
    "    .force(\"center\", d3.forceCenter(width / 2, height / 2));\n",
    "\n",
    "d3.json(\"ego.json\", function(error, graph) {\n",
    "  if (error) throw error;\n",
    "\n",
    "  var link = svg.append(\"g\")\n",
    "      .attr(\"class\", \"links\")\n",
    "    .selectAll(\"line\")\n",
    "    .data(graph.links)\n",
    "    .enter().append(\"line\");\n",
    "      //.attr(\"stroke-width\", function(d) { return Math.sqrt(d.value); });\n",
    "\n",
    "  var node = svg.append(\"g\")\n",
    "      .attr(\"class\", \"nodes\")\n",
    "    .selectAll(\"g\")\n",
    "    .data(graph.nodes)\n",
    "    .enter().append(\"g\")\n",
    "    \n",
    "  var circles = node.append(\"circle\")\n",
    "      .attr(\"r\", 5)\n",
    "      //.attr(\"fill\", function(d) { return color(d.group); })\n",
    "      .call(d3.drag()\n",
    "          .on(\"start\", dragstarted)\n",
    "          .on(\"drag\", dragged)\n",
    "          .on(\"end\", dragended));\n",
    "\n",
    "  var lables = node.append(\"text\")\n",
    "      .text(function(d) {\n",
    "        return d.name;\n",
    "      })\n",
    "      .attr('x', 6)\n",
    "      .attr('y', 3);\n",
    "\n",
    "  node.append(\"title\")\n",
    "      .text(function(d) { return d.name; });\n",
    "\n",
    "  simulation\n",
    "      .nodes(graph.nodes)\n",
    "      .on(\"tick\", ticked);\n",
    "\n",
    "  simulation.force(\"link\")\n",
    "      .links(graph.links);\n",
    "\n",
    "  function ticked() {\n",
    "    link\n",
    "        .attr(\"x1\", function(d) { return d.source.x; })\n",
    "        .attr(\"y1\", function(d) { return d.source.y; })\n",
    "        .attr(\"x2\", function(d) { return d.target.x; })\n",
    "        .attr(\"y2\", function(d) { return d.target.y; });\n",
    "\n",
    "    node\n",
    "        .attr(\"transform\", function(d) {\n",
    "          return \"translate(\" + d.x + \",\" + d.y + \")\";\n",
    "        })\n",
    "  }\n",
    "});\n",
    "\n",
    "function dragstarted(d) {\n",
    "  if (!d3.event.active) simulation.alphaTarget(0.1).restart();\n",
    "  d.fx = d.x;\n",
    "  d.fy = d.y;\n",
    "}\n",
    "\n",
    "function dragged(d) {\n",
    "  d.fx = d3.event.x;\n",
    "  d.fy = d3.event.y;\n",
    "}\n",
    "\n",
    "function dragended(d) {\n",
    "  if (!d3.event.active) simulation.alphaTarget(0);\n",
    "  d.fx = null;\n",
    "  d.fy = null;\n",
    "}\n",
    "});"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}