Merge pull request #14 from OKN-CollabNext/sqlite-indexing

Use sqlite instead of json for graph data

observable/docs/data/graph.json.py → observable/docs/data/graph.sqlite.py

@@ -1,4 +1,8 @@
-import json
+import os
+import sqlite3
+import tempfile
+
+import pandas as pd
 
 from collabnext.openalex.authors import get_affiliated_authors
 from collabnext.openalex.edges import (
@@ -56,4 +60,20 @@
 nodes = [*institution_nodes, *author_nodes, *work_nodes, *topic_nodes]
 edges = [*affiliated_author_edges, *work_author_edges, *work_topic_edges]
 
-print(json.dumps({"nodes": nodes, "edges": edges}))
+# Create nodes dataframe
+df_nodes = pd.DataFrame(nodes, columns=["id", "label", "type"])
+
+# Create edges dataframe
+df_edges = pd.DataFrame(
+    edges, columns=["id", "start", "end", "label", "start_type", "end_type"]
+)
+
+# Save the dataframe to a SQLite database
+with tempfile.NamedTemporaryFile(suffix=".sqlite", delete=False) as temp_file:
+    temp_filename = temp_file.name
+    with sqlite3.connect(temp_filename) as conn:
+        df_nodes.to_sql("nodes", conn, index=False)
+        df_edges.to_sql("edges", conn, index=False)
+
+# Print db file to stdout
+os.system(f"cat {temp_filename}")

observable/docs/index.md

@@ -15,11 +15,30 @@ const query = view(Inputs.text());
 ### Your Peer Network
 
 ```js
-const graph = FileAttachment("data/graph.json").json();
+import { SQLiteDatabaseClient } from "npm:@observablehq/sqlite";
+const db = FileAttachment("data/graph.sqlite").sqlite();
 ```
 
 ```js
-const { nodes, edges } = graph;
+const nodes = db.query(
+  `
+  SELECT
+    *
+  FROM
+    nodes
+  `
+);
+```
+
+```js
+const edges = db.query(
+  `
+  SELECT
+    *
+  FROM
+    edges
+  `
+);
 ```
 
 <script src="https://unpkg.com/@memgraph/orb/dist/browser/orb.min.js"></script>

pyproject.toml

@@ -13,6 +13,7 @@ packages = [
 python = "^3.11"
 pyalex = "^0.14"
 python-dotenv = "^1.0.1"
+pandas = "^2.2.2"
 
 
 [tool.poetry.group.dev.dependencies]