Patch to resolve issue 360 remove !!set and other odd markup in yaml output of graph-summary output

.gitignore

@@ -1,6 +1,7 @@
 env/
 py*/
 __pycache__
+.pytest_cache
 *.pyc
 *.egg-info
 tests/logs/*.log

kgx/graph_operations/meta_knowledge_graph.py

@@ -583,7 +583,7 @@ def get_number_of_categories(self) -> int:
         # return len([c for c in self.node_stats.keys() if c != 'unknown'])
         return len(self.node_stats.keys())
 
-    def get_node_stats(self) -> Dict[str, Category]:
+    def get_node_stats(self) -> Dict[str, Dict]:
         """
         Returns
         -------
@@ -593,7 +593,21 @@ def get_node_stats(self) -> Dict[str, Category]:
         # We no longer track 'unknown' node category counts - non TRAPI 1.1. compliant output
         # if 'unknown' in self.node_stats and not self.node_stats['unknown'].get_count():
         #     self.node_stats.pop('unknown')
-        return self.node_stats
+
+        # Here we assume that the node_stats are complete and will now
+        # be exported in a graph summary for the module, thus we aim to
+        # Convert the 'MetaKnowledgeGraph.Category' object into vanilla
+        # Python dictionary and lists, to facilitate output
+        category_stats = dict()
+        for category_curie in self.node_stats.keys():
+            category_obj = self.node_stats[category_curie]
+            category_stats[category_curie] = dict()
+            # Convert id_prefixes Set into a sorted List
+            category_stats[category_curie]["id_prefixes"] = sorted(category_obj.category_stats["id_prefixes"])
+            category_stats[category_curie]["count"] = category_obj.category_stats["count"]
+            category_stats[category_curie]["count_by_source"] = category_obj.category_stats["count_by_source"]
+
+        return category_stats
 
     def get_edge_stats(self) -> List[Dict[str, Any]]:
         """
@@ -914,7 +928,7 @@ def save(self, file, name: str = None, file_format: str = "json") -> None:
         """
         stats = self.get_graph_summary(name)
         if not file_format or file_format == "json":
-            dump(stats, file, indent=4, default=mkg_default)
+            dump(stats, file, indent=4)
         else:
             yaml.dump(stats, file)
 

kgx/graph_operations/summarize_graph.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, Optional, Any, Callable, Set
+from typing import Dict, List, Optional, Any, Callable
 from sys import stderr
 
 import re
@@ -39,6 +39,7 @@ def gs_default(o):
     """
     JSONEncoder 'default' function override to
     properly serialize 'Set' objects (into 'List')
+    :param o
     """
     if isinstance(o, GraphSummary.Category):
         return o.json_object()
@@ -231,7 +232,7 @@ def __init__(self, category_curie: str, summary):
                 self.summary.node_stats[NODE_CATEGORIES].add(self.category_curie)
             self.summary.node_stats[NODE_ID_PREFIXES_BY_CATEGORY][
                 self.category_curie
-            ] = set()
+            ] = list()
             self.summary.node_stats[COUNT_BY_CATEGORY][self.category_curie] = {
                 "count": 0
             }
@@ -276,14 +277,14 @@ def _get_category_curie_by_index(cls, cid: int) -> str:
             """
             return cls._category_curie_map[cid]
 
-        def get_id_prefixes(self) -> Set:
+        def get_id_prefixes(self) -> List:
             """
             Returns
             -------
-            Set[str]
-                Set of identifier prefix (strings) used by nodes of this Category.
+            List[str]
+                List of identifier prefix (strings) used by nodes of this Category.
             """
-            return set(self.category_stats["count_by_id_prefix"].keys())
+            return list(self.category_stats["count_by_id_prefix"].keys())
 
         def get_count_by_id_prefixes(self):
             """
@@ -591,9 +592,8 @@ def get_node_stats(self) -> Dict[str, Any]:
             for node_category in self.node_categories.values():
                 self._compile_category_stats(node_category)
 
-            self.node_stats[NODE_CATEGORIES] = sorted(
-                list(self.node_stats[NODE_CATEGORIES])
-            )
+            self.node_stats[NODE_CATEGORIES] = sorted(self.node_stats[NODE_CATEGORIES])
+            self.node_stats[NODE_ID_PREFIXES] = sorted(self.node_stats[NODE_ID_PREFIXES])
 
             if self.node_facet_properties:
                 for facet_property in self.node_facet_properties:

requirements.txt

@@ -11,7 +11,7 @@ rdflib~=5.0.0
 Click~=7.0
 neo4jrestclient>=0.0
 pyyaml>=0.0
-linkml>=1.0.0
+linkml>=1.1.13
 prologterms>=0.0.5
 shexjsg>=0.6.5
 terminaltables>=3.1.0

tests/unit/test_cli_utils.py

@@ -35,7 +35,7 @@ def test_get_report_format_types():
     assert "json" in format_types
 
 
-def test_graph_summary1():
+def test_kgx_graph_summary():
     """
     Test graph summary, where the output report type is kgx-map.
     """
@@ -66,16 +66,16 @@ def test_graph_summary1():
     assert "biolink:interacts_with" in summary_stats["edge_stats"]["predicates"]
 
 
-def test_graph_summary2a():
+def test_meta_knowledge_graph_as_json():
     """
-    Test graph summary, where the output report type
-    is meta-knowledge-graph, default JSON report format type.
+    Test graph summary, where the output report type is a meta-knowledge-graph,
+    with results output as the default JSON report format type.
     """
     inputs = [
         os.path.join(RESOURCE_DIR, "graph_nodes.tsv"),
         os.path.join(RESOURCE_DIR, "graph_edges.tsv"),
     ]
-    output = os.path.join(TARGET_DIR, "graph_stats2.json")
+    output = os.path.join(TARGET_DIR, "meta-knowledge-graph.json")
     summary_stats = graph_summary(
         inputs,
         "tsv",
@@ -95,16 +95,16 @@ def test_graph_summary2a():
     assert summary_stats["name"] == "Default Meta-Knowledge-Graph"
 
 
-def test_graph_summary2b():
+def test_meta_knowledge_graph_as_yaml():
     """
-    Test graph summary, where the output report type
-    is meta-knowledge-graph, set as YAML report format type.
+    Test graph summary, where the output report type is a meta-knowledge-graph,
+    with results output as the YAML report output format type.
     """
     inputs = [
         os.path.join(RESOURCE_DIR, "graph_nodes.tsv"),
         os.path.join(RESOURCE_DIR, "graph_edges.tsv"),
     ]
-    output = os.path.join(TARGET_DIR, "graph_stats2.yaml")
+    output = os.path.join(TARGET_DIR, "meta-knowledge-graph.yaml")
     summary_stats = graph_summary(
         inputs,
         "tsv",
@@ -113,7 +113,7 @@ def test_graph_summary2b():
         report_type="meta-knowledge-graph",
         node_facet_properties=["provided_by"],
         edge_facet_properties=["aggregator_knowledge_source"],
-        report_format="yaml",
+        report_format="yaml"
     )
 
     assert os.path.exists(output)
@@ -122,16 +122,16 @@ def test_graph_summary2b():
     assert "edges" in summary_stats
 
 
-def test_graph_summary2c():
+def test_meta_knowledge_graph_as_json_streamed():
     """
-    Test graph summary, where the output report type
-    is meta-knowledge-graph, set as YAML report format type.
+    Test graph summary processed in stream mode, where the output report type
+    is meta-knowledge-graph, output as the default JSON report format type.
     """
     inputs = [
         os.path.join(RESOURCE_DIR, "graph_nodes.tsv"),
         os.path.join(RESOURCE_DIR, "graph_edges.tsv"),
     ]
-    output = os.path.join(TARGET_DIR, "graph_stats2c.json")
+    output = os.path.join(TARGET_DIR, "meta-knowledge-graph-streamed.json")
     summary_stats = graph_summary(
         inputs=inputs,
         input_format="tsv",