Merge pull request #262 from multinet-app/add_downloaders

Add first downloaders

multinet/__init__.py

@@ -7,7 +7,7 @@
 from typing import Optional, MutableMapping, Any, Tuple, Union
 
 from . import api
-from . import uploaders
+from . import uploaders, downloaders
 from .errors import ServerError
 
 
@@ -27,6 +27,9 @@ def create_app(config: Optional[MutableMapping] = None) -> Flask:
     app.register_blueprint(uploaders.nested_json.bp, url_prefix="/api/nested_json")
     app.register_blueprint(uploaders.d3_json.bp, url_prefix="/api/d3_json")
 
+    app.register_blueprint(downloaders.csv.bp, url_prefix="/api")
+    app.register_blueprint(downloaders.d3_json.bp, url_prefix="/api")
+
     # Register error handler.
     @app.errorhandler(ServerError)
     def handle_error(error: ServerError) -> Tuple[Any, Union[int, str]]:

multinet/downloaders/__init__.py

@@ -0,0 +1,2 @@
+"""Downloader blueprints for various filetypes."""
+from . import csv, d3_json  # noqa: F401

multinet/downloaders/csv.py

@@ -0,0 +1,48 @@
+"""Multinet uploader for CSV files."""
+import csv
+from flasgger import swag_from
+from io import StringIO
+
+from multinet.util import require_db, filter_unwanted_keys, generate_filtered_docs
+from multinet.db import get_workspace_db, workspace_table
+from multinet.errors import NotFound
+
+from flask import Blueprint, make_response
+
+# Import types
+from typing import Any
+
+
+bp = Blueprint("download_csv", __name__)
+bp.before_request(require_db)
+
+
+@bp.route("/workspaces/<workspace>/tables/<table>/download", methods=["GET"])
+@swag_from("swagger/csv.yaml")
+def download(workspace: str, table: str) -> Any:
+    """
+    Download a table from the database as a CSV file.
+
+    `workspace` - the target workspace
+    `table` - the target table
+    """
+    space = get_workspace_db(workspace)
+    if not space.has_collection(table):
+        raise NotFound("table", table)
+
+    limit = workspace_table(workspace, table, 0, 0)["count"]
+    table_rows = workspace_table(workspace, table, 0, limit)["rows"]
+    fields = filter_unwanted_keys(table_rows[0]).keys()
+
+    io = StringIO()
+    writer = csv.DictWriter(io, fieldnames=fields)
+
+    writer.writeheader()
+    for row in generate_filtered_docs(table_rows):
+        writer.writerow(row)
+
+    output = make_response(io.getvalue())
+    output.headers["Content-Disposition"] = f"attachment; filename={table}.csv"
+    output.headers["Content-type"] = "text/csv"
+
+    return output

multinet/downloaders/d3_json.py

@@ -0,0 +1,77 @@
+"""Multinet downloader for nested JSON files."""
+import re
+
+from flasgger import swag_from
+
+from multinet.util import require_db, generate_filtered_docs
+from multinet.db import get_workspace_db
+from multinet.errors import GraphNotFound
+
+from flask import Blueprint, make_response
+
+# Import types
+from typing import Any
+
+bp = Blueprint("download_d3_json", __name__)
+bp.before_request(require_db)
+
+
+@bp.route("/workspaces/<workspace>/graphs/<graph>/download", methods=["GET"])
+@swag_from("swagger/d3_json.yaml")
+def download(workspace: str, graph: str) -> Any:
+    """Return a graph as a d3 json-encoded graph.
+
+    `workspace` - the target workspace
+    `graph` - the target graph
+    """
+
+    nodes = []
+    links = []
+
+    space = get_workspace_db(workspace)
+    if not space.has_graph(graph):
+        raise GraphNotFound(workspace, graph)
+
+    loaded_graph = space.graph(graph)
+
+    node_tables = loaded_graph.vertex_collections()
+    for node_table in node_tables:
+        table_nodes = loaded_graph.vertex_collection(node_table).all()
+
+        for node in table_nodes:
+            node["id"] = node["_key"]
+            del node["_key"]
+            nodes.append(node)
+
+    pattern = re.compile(r"^([^\d_]\w+)_nodes(/.+)")
+    edge_tables = [edef["edge_collection"] for edef in loaded_graph.edge_definitions()]
+    for edge_table in edge_tables:
+        edges = loaded_graph.edge_collection(edge_table).all()
+
+        for edge in edges:
+            source = edge["_from"]
+            target = edge["_to"]
+            source_match = pattern.search(source)
+            target_match = pattern.search(target)
+
+            if source_match and target_match:
+                source = "".join(source_match.groups())
+                target = "".join(target_match.groups())
+
+            edge["source"] = source
+            edge["target"] = target
+            del edge["_from"]
+            del edge["_to"]
+
+            links.append(edge)
+
+    response = make_response(
+        dict(
+            nodes=list(generate_filtered_docs(nodes)),
+            links=list(generate_filtered_docs(links)),
+        )
+    )
+    response.headers["Content-Disposition"] = f"attachment; filename={graph}.json"
+    response.headers["Content-type"] = "application/json"
+
+    return response

multinet/downloaders/swagger/csv.yaml

@@ -0,0 +1,21 @@
+Download a table as a CSV file
+---
+consumes:
+  - text/csv
+
+parameters:
+  - $ref: "#/parameters/workspace"
+  - $ref: "#/parameters/table"
+
+responses:
+  200:
+    description: CSV returned
+
+  404:
+    description: Workspace/Table Not Found
+    schema:
+      type: string
+      example:
+        "table_name"
+tags:
+  - table

multinet/downloaders/swagger/d3_json.yaml

@@ -0,0 +1,19 @@
+Download a graph in D3 JSON format
+---
+consumes:
+  - text/plain
+
+parameters:
+  - $ref: "#/parameters/workspace"
+  - $ref: "#/parameters/graph"
+
+
+responses:
+  200:
+    description: D3 data uploaded to tables
+
+  404:
+    description: Graph not found
+
+tags:
+  - graph

multinet/util.py

@@ -13,6 +13,19 @@
 TEST_DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../test/data"))
 
 
+def filter_unwanted_keys(row: Dict) -> Dict:
+    """Remove any unwanted keys from a document."""
+    unwanted = {"_rev", "_id"}
+    return {k: v for k, v in row.items() if k not in unwanted}
+
+
+def generate_filtered_docs(rows: Sequence[Dict]) -> Generator[Dict, None, None]:
+    """Filter unwanted keys from all documents with a generator."""
+
+    for row in rows:
+        yield filter_unwanted_keys(row)
+
+
 def get_edge_table_properties(workspace: str, edge_table: str) -> EdgeTableProperties:
     """
     Return extracted information about an edge table.