Merge pull request #17 from kijimaD/dev

Add d3 graph

.github/workflows/publish.yml

@@ -47,6 +47,13 @@ jobs:
       - name: Build node graph
         run: make roam-graph
 
+      - name: Setup Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+      - name: Build node graph
+        run: make node-graph
+
       - name: check
         run: ls -al ./public
 

.gitignore

@@ -2,7 +2,10 @@
 /.org-cache
 /.packages
 /node_modules
+/node_graph/__pycache__
+
 public/**/*.html
+public/js/graph.json
 
 cache.json
 git-file.dat

Makefile

@@ -6,6 +6,9 @@ build:
 	emacs --batch -l ./publish1.el --funcall kd/publish
 roam-graph:
 	emacs --batch -l ./publish1.el --funcall org-roam-graph-save
+node-graph:
+	pip3 install -r requirements.txt
+	python3 node_graph/build_graph.py > public/js/graph.json
 file-graph:
 	ruby git-file.rb > git-file.dat
 	gnuplot git-file.plot

index.org

@@ -3,19 +3,28 @@
 :END:
 #+title: Imsomnia
 
-* 概要
-プログラム関連の活動記録。
-
-- [[file:sitemap.html][Sitemap]]
-- [[id:a0f58a2a-e92d-496e-9c81-dc5401ab314f][Author History]]
-* Node graph
 #+caption: ページ間のリンクを示す
 #+BEGIN_EXPORT html
+<div id="main-graph">
+  <svg>
+  <defs>
+    <filter x="0" y="0" width="1" height="1" id="solid">
+      <feflood flood-color="#f7f7f7" flood-opacity="0.9"></feflood>
+      <fecomposite in="SourceGraphic" operator="xor"></fecomposite>
+    </filter>
+  </defs>
+  <rect id="base_rect" width="100%" height="100%" fill="#ffffff"></rect>
+  </svg>
+</div>
+
 <img src="./graph.svg"
      alt="graph"
-     style="position: relative;
-            width: 1000px;" />
+     style="position: relative; width: 1000px;" />
 #+END_EXPORT
+* 概要
+プログラム関連の記録。
+- [[file:sitemap.html][Sitemap]]
+- [[id:a0f58a2a-e92d-496e-9c81-dc5401ab314f][Author History]]
 * Repository stat
 期間ごとのファイル数と行数。
 

node_graph/build_graph.py

@@ -0,0 +1,66 @@
+# https://hugocisneros.com/blog/my-org-roam-notes-workflow/
+
+import itertools
+import json
+import sys
+
+import networkx as nx
+import networkx.algorithms.link_analysis.pagerank_alg as pag
+import networkx.algorithms.community as com
+from networkx.drawing.nx_pydot import read_dot
+from networkx.readwrite import json_graph
+
+from roam_db_to_json import build_graph
+
+N_COM = 7  # Desired number of communities
+N_MISSING = 20  # Number of predicted missing links
+MAX_NODES = 200  # Number of nodes in the final graph
+
+def compute_centrality(dot_graph: nx.DiGraph) -> None:
+    """Add a `centrality` attribute to each node with its PageRank score.
+    """
+    simp_graph = nx.Graph(dot_graph)
+    central = pag.pagerank(simp_graph)
+    min_cent = min(central.values())
+    central = {i: central[i] - min_cent for i in central}
+    max_cent = max(central.values())
+    central = {i: central[i] / max_cent for i in central}
+    nx.set_node_attributes(dot_graph, central, "centrality")
+    sorted_cent = sorted(dot_graph, key=lambda x: dot_graph.nodes[x]["centrality"])
+    for n in sorted_cent[:-MAX_NODES]:
+        dot_graph.remove_node(n)
+
+
+def compute_communities(dot_graph: nx.DiGraph, n_com: int) -> None:
+    """Add a `communityLabel` attribute to each node according to their
+    computed community.
+    """
+    simp_graph = nx.Graph(dot_graph)
+    communities = com.girvan_newman(simp_graph)
+    labels = [tuple(sorted(c) for c in unities) for unities in
+              itertools.islice(communities, n_com - 1, n_com)][0]
+    label_dict = {l_key: i for i in range(len(labels)) for l_key in labels[i]}
+    nx.set_node_attributes(dot_graph, label_dict, "communityLabel")
+
+
+def add_missing_links(dot_graph: nx.DiGraph, n_missing: int) -> None:
+    """Add some missing links to the graph by using top ranking inexisting
+    links by ressource allocation index.
+    """
+    simp_graph = nx.Graph(dot_graph)
+    preds = nx.ra_index_soundarajan_hopcroft(simp_graph, community="communityLabel")
+    new = sorted(preds, key=lambda x: -x[2])[:n_missing]
+    for link in new:
+        # sys.stderr.write(f"Predicted edge {link[0]} {link[1]}\n")
+        dot_graph.add_edge(link[0], link[1], predicted=link[2])
+
+
+if __name__ == "__main__":
+    # sys.stderr.write("Reading graph...")
+    DOT_GRAPH = build_graph()
+    compute_centrality(DOT_GRAPH)
+    compute_communities(DOT_GRAPH, N_COM)
+    add_missing_links(DOT_GRAPH, N_MISSING)
+    # sys.stderr.write("Done\n")
+    JS_GRAPH = json_graph.node_link_data(DOT_GRAPH)
+    sys.stdout.write(json.dumps(JS_GRAPH))

node_graph/roam_db_to_json.py

@@ -0,0 +1,31 @@
+import networkx as nx
+import pathlib
+import sqlite3
+
+def to_rellink(inp: str) -> str:
+    return pathlib.Path(inp).stem
+
+
+def build_graph() -> any:
+    """Build a graph from the org-roam database."""
+    graph = nx.DiGraph()
+    home = pathlib.Path.home()
+    conn = sqlite3.connect(home / ".emacs.d" / "org-roam.db")
+
+    # Query all nodes first
+    nodes = conn.execute("SELECT file, id, title FROM nodes WHERE level = 0;")
+    # A double JOIN to get all nodes that are connected by a link
+    links = conn.execute("SELECT n1.id, nodes.id FROM ((nodes AS n1) "
+                         "JOIN links ON n1.id = links.source) "
+                         "JOIN (nodes AS n2) ON links.dest = nodes.id "
+                         "WHERE links.type = '\"id\"';")
+    # Populate the graph
+    graph.add_nodes_from((n[1], {
+        "label": n[2].strip("\""),
+        "tooltip": n[2].strip("\""),
+        "lnk": to_rellink(n[0]).lower(),
+        "id": n[1].strip("\"")
+    }) for n in  nodes)
+    graph.add_edges_from(n for n in links if n[0] in graph.nodes and n[1] in graph.nodes)
+    conn.close()
+    return graph

public/js/graph.js

@@ -0,0 +1,189 @@
+d3.json("/js/graph.json").then(function(data) {
+    // Canvas size
+    height = 1100;
+    width = 1600;
+    scale = 1.;
+    // Radius function for nodes. Node radius are function of centrality
+    radius = d => {
+        if (!d.radius) {
+            d.radius = 11 + 24 * Math.pow(d.centrality, 9/5);
+        }
+        return d.radius;
+    };
+    color = "#ffffff";
+
+    // Number of colors is the number of clusters (given by communityLabel)
+    num_colors = Math.max(...data.nodes.map(d => d.communityLabel)) + 1;
+    angleArr = [...Array(num_colors).keys()].map(x => 2 * Math.PI * x / num_colors);
+    // Cluster centers around an circle
+    centersx = angleArr.map(x => Math.cos(Math.PI + x));
+    centersy = angleArr.map(x => Math.sin(Math.PI + x));
+    // Color palette
+    nodeColors = [
+        '#C98914',
+        '#C55F1A',
+        '#4189AD',
+        '#007500',
+        '#968674',
+        '#5E998A',
+        "#363ea9",
+    ];
+    // Color function just maps cluster to color palette
+    nodeColor = d => {
+        return nodeColors[d.communityLabel];
+    };
+    // Make the nodes draggable
+    drag = simulation => {
+        function dragsubject(event) {
+            return simulation.find(event.x, event.y);
+        }
+
+        function dragstarted(event) {
+            if (!event.active) simulation.alphaTarget(0.3).restart();
+            event.subject.fx = event.subject.x;
+            event.subject.fy = event.subject.y;        }
+
+        function dragged(event) {
+            event.subject.fx = event.x;
+            event.subject.fy = event.y;
+        }
+
+        function dragended(event) {
+            if (!event.active) simulation.alphaTarget(0);
+            event.subject.fx = null;
+            event.subject.fy = null;
+        }
+
+        return d3.drag()
+            .subject(dragsubject)
+            .on("start", dragstarted)
+            .on("drag", dragged)
+            .on("end", dragended);
+    };
+    // Make nodes interactive to hovering
+    handleMouseOver = (d, i) => {
+        nde = d3.select(d.currentTarget);
+        nde.attr("fill", "#999")
+            .attr("r", nde.attr("r") * 1.4);
+
+        d3.selectAll("text")
+            .filter('#' + CSS.escape(d.currentTarget.id))
+            .style("display", "block");
+
+        d3.selectAll("line")
+            .attr("stroke-width", 1);
+
+        d3.selectAll("line")
+            .filter((l, _) =>
+                    l.source.index == i.index ||
+                    l.target.index == i.index)
+            .attr("stroke-width", 8);
+    };
+    handleMouseOut = (d, _) => {
+        nde = d3.select(d.currentTarget);
+        nde.attr("fill", nodeColor)
+            .attr("r", nde.attr("r") / 1.4);
+
+        d3.selectAll("text")
+            .filter('#' + CSS.escape(d.currentTarget.id))
+            .style("display", "none");
+
+    };
+
+    // Graph data
+    const links = data.links.map(d => Object.create(d));
+    const nodes = data.nodes.map(d => Object.create(d));
+
+    // Force simulation for the graph
+    simulation = d3.forceSimulation(nodes)
+    .alpha(0.9)
+    .velocityDecay(0.6)
+        .force("link", d3.forceLink(links).id(d => d.id).strength(.1))
+        .force("charge", d3.forceManyBody()
+               .strength(-500))
+        .force('collision',
+               d3.forceCollide().radius(d => radius(d) * 1.2).strength(1.5))
+        .force('x', d3.forceX().x(function(d) {
+            return width / 2 +  (width / 4) * centersx[d.communityLabel];
+        }).strength(0.25))
+        .force('y', d3.forceY().y(function(d) {
+            return height / 2 +  (height / 8) * centersy[d.communityLabel];
+        }).strength(0.25));
+
+    // Create all the graph elements
+    const svg = d3.select("svg")
+          .attr('max-width', '60%')
+          .attr('class', 'node-graph')
+          .attr("viewBox", [0, 0, width, height]);
+
+    const link = svg.append("g")
+          .attr("stroke", "#888")
+          .attr("stroke-opacity", 0.6)
+          .selectAll("line")
+          .data(links)
+          .join("line")
+          .attr("stroke-dasharray", d => (d.predicted? "5,5": "0,0"))
+          .attr("stroke-width", 1);
+
+    const node = svg.append("g")
+          .selectAll("circle")
+          .data(nodes)
+          .join("a")
+          .attr("xlink:href", d => {
+              return "./" + d.lnk;
+          })
+          .attr("id", d => "circle_" + d.lnk)
+          .append("circle")
+          .attr("id", d => d.id.toLowerCase())
+          .attr("r", radius)
+          .attr("fill", nodeColor)
+          .attr("stroke", "#000")
+          .attr("stroke-width", 1.3)
+          .on("mouseover", handleMouseOver)
+          .on("mouseout", handleMouseOut)
+          .call(drag(simulation));
+
+    node.append("title")
+        .text(d => d.label.replace(/"/g, ''));
+
+    // Nodes have a label that is visible on hover
+    // They have two layers a rectangle "background" and the text on top
+    const label = svg.append("g")
+          .selectAll("text")
+          .data(nodes)
+          .join("g");
+    const label_background = label.append("text")
+          .style("font-size", "45px")
+          .text(function (d) { return "  "+ d.label.replace(/"/g, '') + "  "; })
+          .attr("dy", -30)
+          .attr("id", d => d.id.toLowerCase())
+          .attr("class", "node_label")
+          .style("display", "none")
+          .style("pointer-events", "none")
+          .style("alignment-baseline", "middle")
+          // .attr("filter", "url(#solid)");
+    const label_text = label.append("text")
+          .style("fill", "#222")
+          .style("font-size", "15px")
+          .text(function (d) { return "  "+ d.label.replace(/"/g, '') + "  "; })
+          .attr("dy", -25)
+          .attr("id", d => d.id.toLowerCase())
+          .attr("class", "node_label")
+
+    // Run the simulation
+    simulation.on("tick", () => {
+        link.attr("x1", d => d.source.x)
+            .attr("y1", d => d.source.y)
+            .attr("x2", d => d.target.x)
+            .attr("y2", d => d.target.y);
+
+        node.attr("cx", d => d.x)
+            .attr("cy", d => d.y);
+
+        label_text.attr("x", d => d.x)
+            .attr("y", d => d.y);
+
+        label_background.attr("x", d => d.x)
+            .attr("y", d => d.y);
+    });
+});

publish1.el

@@ -96,7 +96,9 @@
       (concat
        "<link rel='stylesheet' href='https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css' />"
        "<link rel='stylesheet' href='css/site.css' />"
-       "<link rel='stylesheet' href='css/code.css' />"))
+       "<link rel='stylesheet' href='css/code.css' />"
+       "<script defer src='https://cdnjs.cloudflare.com/ajax/libs/d3/7.2.1/d3.min.js' integrity='sha512-wkduu4oQG74ySorPiSRStC0Zl8rQfjr/Ty6dMvYTmjZw6RS5bferdx8TR7ynxeh79ySEp/benIFFisKofMjPbg==' crossorigin='anonymous' referrerpolicy='no-referrer'></script>"
+       "<script defer src='/js/graph.js'></script>"))
 
 ;; Compile
 (setq org-publish-project-alist

requirements.txt

@@ -0,0 +1,2 @@
+scipy
+networkx