Add simple reference visualization notebook for CC.

notebooks/cc/references.ipynb

@@ -0,0 +1,199 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "collapsed": true,
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   },
+   "source": [
+    "# References"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import networkx as nx\n",
+    "import matplotlib.pyplot as plt\n",
+    "from sec_certs.dataset.common_criteria import CCDataset\n",
+    "%matplotlib inline\n",
+    "plt.rcParams['figure.figsize'] = (10, 6)\n",
+    "\n",
+    "dset = CCDataset.from_web_latest()\n",
+    "certs_with_ids = {cert.heuristics.cert_id: cert for cert in dset if cert.heuristics.cert_id}\n",
+    "\n",
+    "print(f\"Certificates in dataset: {len(dset)}\")\n",
+    "print(f\"Certificates with extracted IDs: {len(certs_with_ids)}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   },
+   "source": [
+    "## Certificate report references"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "refs_cr = nx.DiGraph()\n",
+    "for cert_id, cert in certs_with_ids.items():\n",
+    "    refs_cr.add_node(cert_id, cert=cert)\n",
+    "for cert_id, cert in certs_with_ids.items():\n",
+    "    if cr_refs := cert.heuristics.report_references.directly_referencing:\n",
+    "        for ref_id in cr_refs:\n",
+    "            refs_cr.add_edge(cert_id, ref_id, type=(\"cr\",))\n",
+    "print(f\"References in certificate reports: {len(refs_cr.edges)}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   },
+   "source": [
+    "## Security target references"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "refs_st = nx.DiGraph()\n",
+    "for cert_id, cert in certs_with_ids.items():\n",
+    "    refs_st.add_node(cert_id, cert=cert)\n",
+    "for cert_id, cert in certs_with_ids.items():\n",
+    "    if st_refs := cert.heuristics.st_references.directly_referencing:\n",
+    "        for ref_id in st_refs:\n",
+    "            refs_st.add_edge(cert_id, ref_id, type=(\"st\",))\n",
+    "print(f\"References in security targets: {len(refs_st.edges)}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Combined references"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "refs = nx.DiGraph()\n",
+    "refs.add_nodes_from(refs_cr)\n",
+    "refs.add_nodes_from(refs_st)\n",
+    "refs.add_edges_from(refs_cr.edges.data())\n",
+    "for edge in refs_st.edges:\n",
+    "    if edge in refs.edges:\n",
+    "        refs.edges[edge][\"type\"] = (\"cr\", \"st\")\n",
+    "    else:\n",
+    "        refs.add_edge(edge, *refs_st.edges[edge])\n",
+    "print(f\"Combined references (not double counted): {len(refs.edges)}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Certificate overview\n",
+    "Enter the certificate you are interested in below and see its reference graph component."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cert_id = \"ANSSI-CC-2019/02\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "cert = certs_with_ids.get(cert_id)\n",
+    "if cert is None:\n",
+    "    print(f\"Certificate with id {cert_id} is not present in the dataset.\")\n",
+    "\n",
+    "for component in nx.weakly_connected_components(refs):\n",
+    "    if cert_id in component:\n",
+    "        break\n",
+    "\n",
+    "view = nx.subgraph_view(refs, lambda node: node in component)\n",
+    "print(f\"Certificate with id {cert_id}:\")\n",
+    "print(f\" - is in a component with {len(view.nodes)} certificates and {len(view.edges)} references.\")\n",
+    "print(f\" - references {list(view[cert_id].keys())}\")\n",
+    "print(f\" - is referenced by {list(view.predecessors(cert_id))}\")\n",
+    "print(f\" - its page is at https://seccerts.org/cc/{cert.dgst}/\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "nx.draw(view, pos=nx.planar_layout(view), with_labels=True)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}

sec_certs/helpers.py

@@ -21,7 +21,7 @@
 import requests
 from PyPDF2 import PdfFileReader
 from PyPDF2.generic import BooleanObject, FloatObject, IndirectObject, NumberObject
-from tqdm import tqdm as tqdm_original
+from tqdm.auto import tqdm as tqdm_original
 
 import sec_certs.constants as constants
 from sec_certs.cert_rules import REGEXEC_SEP